atlas-iac/services/ai-llm/deployment.yaml

# services/ai-llm/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ollama
  namespace: ai
spec:
  replicas: 1
  revisionHistoryLimit: 2
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 0
      maxUnavailable: 1
  selector:
    matchLabels:
      app: ollama
  template:
    metadata:
      labels:
        app: ollama
      annotations:
        ai.bstein.dev/model: qwen2.5:3b-instruct-q4_0,qwen2.5:14b-instruct-q4_0
        ai.bstein.dev/gpu: titan-20 shared routing GPU
        ai.bstein.dev/restartedAt: "2026-08-11T08:00:00Z"
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/hostname
                    operator: In
                    values:
                      - titan-20
      runtimeClassName: nvidia
      volumes:
        - name: models
          # The pod is pinned to titan-20. A local-path claim preserves the
          # 10 GiB model cache across rollouts without involving Longhorn.
          persistentVolumeClaim:
            claimName: ollama-models-titan20
      initContainers:
        - name: warm-model
          image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
          env:
            - name: OLLAMA_HOST
              value: 0.0.0.0
            - name: NVIDIA_VISIBLE_DEVICES
              value: all
            - name: NVIDIA_DRIVER_CAPABILITIES
              value: compute,utility
            - name: OLLAMA_MODELS
              value: /root/.ollama
            - name: OLLAMA_MODEL
              value: qwen2.5:14b-instruct-q4_0
            - name: OLLAMA_FAST_MODEL
              value: qwen2.5:3b-instruct-q4_0
            - name: OLLAMA_CONTEXT_LENGTH
              value: "512"
            - name: JETSON_JETPACK
              value: "5"
            - name: OLLAMA_LLM_LIBRARY
              value: cuda_jetpack5
            - name: LD_LIBRARY_PATH
              value: /lib/ollama:/lib/ollama/cuda_jetpack5:/usr/lib/aarch64-linux-gnu/tegra
          command:
            - /bin/sh
            - -c
            - |
              set -e
              ollama serve >/tmp/ollama.log 2>&1 &
              sleep 6
              for model in "${OLLAMA_FAST_MODEL}" "${OLLAMA_MODEL}"; do
                ollama pull "${model}"
              done
              pkill ollama || true
          volumeMounts:
            - name: models
              mountPath: /root/.ollama
          resources:
            requests:
              cpu: 500m
              memory: 2Gi
              nvidia.com/gpu.shared: 1
            limits:
              cpu: "2"
              memory: 4Gi
              nvidia.com/gpu.shared: 1
      containers:
        - name: ollama
          image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
          imagePullPolicy: IfNotPresent
          ports:
            - name: http
              containerPort: 11434
          env:
            - name: OLLAMA_HOST
              value: 0.0.0.0
            - name: OLLAMA_FAST_MODEL
              value: qwen2.5:3b-instruct-q4_0
            - name: OLLAMA_CONTEXT_LENGTH
              value: "8192"
            - name: OLLAMA_KEEP_ALIVE
              value: "-1"
            # The Xavier has 16 GiB of unified memory. Qwen 3B + Qwen 14B
            # weights alone use about 10.3 GB, so concurrent residency leaves
            # no safe room for KV cache, CUDA, kubelet, or the OS. Serialize
            # model residency; the router re-warms the classifier after a
            # local 14B answer.
            - name: OLLAMA_MAX_LOADED_MODELS
              value: "1"
            - name: OLLAMA_NUM_PARALLEL
              value: "1"
            - name: OLLAMA_FLASH_ATTENTION
              value: "1"
            - name: OLLAMA_KV_CACHE_TYPE
              value: q8_0
            - name: OLLAMA_MODELS
              value: /root/.ollama
            - name: NVIDIA_VISIBLE_DEVICES
              value: all
            - name: NVIDIA_DRIVER_CAPABILITIES
              value: compute,utility
            - name: JETSON_JETPACK
              value: "5"
            - name: OLLAMA_LLM_LIBRARY
              value: cuda_jetpack5
            - name: LD_LIBRARY_PATH
              value: /lib/ollama:/lib/ollama/cuda_jetpack5:/usr/lib/aarch64-linux-gnu/tegra
          command:
            - /bin/sh
            - -c
            - |
              set -e
              ollama serve &
              pid="$!"
              trap 'kill -TERM "$pid"; wait "$pid"' TERM INT
              sleep 6
              if ! timeout 180s ollama run "${OLLAMA_FAST_MODEL}" "reply with just pong" >/tmp/ollama-fast-warm.log 2>&1; then
                cat /tmp/ollama-fast-warm.log >&2
                exit 1
              fi
              touch /tmp/ollama-fast-ready
              wait "$pid"
          volumeMounts:
            - name: models
              mountPath: /root/.ollama
          readinessProbe:
            exec:
              command:
                - /bin/sh
                - -c
                - test -f /tmp/ollama-fast-ready && ollama list >/dev/null 2>&1
            initialDelaySeconds: 15
            periodSeconds: 10
            timeoutSeconds: 5
          resources:
            requests:
              cpu: "4"
              memory: 10Gi
              nvidia.com/gpu.shared: 1
            limits:
              cpu: "8"
              memory: 14Gi
              nvidia.com/gpu.shared: 1