titan-iac/services/hermes/ollama-deployment.yaml

# services/hermes/ollama-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: hermes-ollama
  namespace: hermes
  labels:
    app: hermes-ollama
spec:
  revisionHistoryLimit: 2
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app: hermes-ollama
  template:
    metadata:
      labels:
        app: hermes-ollama
      annotations:
        ai.bstein.dev/model: gpt-oss:20b
        ai.bstein.dev/gpu: titan-24 local-first lane
    spec:
      runtimeClassName: nvidia
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: kubernetes.io/hostname
                    operator: In
                    values:
                      - titan-24
      volumes:
        - name: models
          persistentVolumeClaim:
            claimName: hermes-models
      initContainers:
        - name: warm-model
          image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
          imagePullPolicy: IfNotPresent
          env:
            - name: OLLAMA_HOST
              value: 0.0.0.0
            - name: OLLAMA_MODELS
              value: /root/.ollama
            - name: OLLAMA_MODEL
              value: gpt-oss:20b
            - name: NVIDIA_VISIBLE_DEVICES
              value: all
            - name: NVIDIA_DRIVER_CAPABILITIES
              value: compute,utility
          command:
            - /bin/sh
            - -c
            - |
              set -e
              ollama serve >/tmp/ollama.log 2>&1 &
              sleep 6
              ollama pull "${OLLAMA_MODEL}"
              pkill ollama || true
          volumeMounts:
            - name: models
              mountPath: /root/.ollama
          resources:
            requests:
              cpu: "1"
              memory: 4Gi
              nvidia.com/gpu.shared: 1
            limits:
              cpu: "4"
              memory: 16Gi
              nvidia.com/gpu.shared: 1
      containers:
        - name: ollama
          image: ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d
          imagePullPolicy: IfNotPresent
          ports:
            - name: http
              containerPort: 11434
          env:
            - name: OLLAMA_HOST
              value: 0.0.0.0
            - name: OLLAMA_KEEP_ALIVE
              value: 6h
            - name: OLLAMA_CONTEXT_LENGTH
              value: "64000"
            - name: OLLAMA_FLASH_ATTENTION
              value: "1"
            - name: OLLAMA_KV_CACHE_TYPE
              value: q8_0
            - name: OLLAMA_MAX_LOADED_MODELS
              value: "1"
            - name: OLLAMA_NUM_PARALLEL
              value: "1"
            - name: OLLAMA_MODELS
              value: /root/.ollama
            - name: NVIDIA_VISIBLE_DEVICES
              value: all
            - name: NVIDIA_DRIVER_CAPABILITIES
              value: compute,utility
          volumeMounts:
            - name: models
              mountPath: /root/.ollama
          readinessProbe:
            httpGet:
              path: /api/tags
              port: 11434
            initialDelaySeconds: 15
            periodSeconds: 10
            timeoutSeconds: 5
          resources:
            requests:
              cpu: "8"
              memory: 24Gi
              nvidia.com/gpu.shared: 1
            limits:
              cpu: "16"
              memory: 40Gi
              nvidia.com/gpu.shared: 1