titan-iac/services/monitoring/grafana-dashboard-gpu.yaml

# services/monitoring/grafana-dashboard-gpu.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-dashboard-gpu
  labels:
    grafana_dashboard: "1"
data:
  atlas-gpu.json: |
    {
      "uid": "atlas-gpu",
      "title": "Atlas GPU",
      "folderUid": "atlas-internal",
      "editable": true,
      "panels": [
        {
          "id": 1,
          "type": "piechart",
          "title": "Namespace GPU Utilization",
          "datasource": {
            "type": "prometheus",
            "uid": "atlas-vm"
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 0
          },
          "targets": [
            {
              "expr": "((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) or (label_replace((sum(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) or on() vector(0)), \"namespace\", \"unattributed\", \"\", \"\") unless on() ((count(nvidia_gpu_device_utilization_percent) or on() vector(0)) > 0))) or (label_replace(clamp_min(((100 * count(nvidia_gpu_device_utilization_percent)) or (100 * count(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on() nvidia_gpu_device_utilization_percent) or on() vector(0)) - (sum((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) or (label_replace((sum(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) or on() vector(0)), \"namespace\", \"unattributed\", \"\", \"\") unless on() ((count(nvidia_gpu_device_utilization_percent) or on() vector(0)) > 0))) or on() vector(0)), 0), \"namespace\", \"unused\", \"\", \"\") and on() ((sum((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) or (label_replace((sum(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) or on() vector(0)), \"namespace\", \"unattributed\", \"\", \"\") unless on() ((count(nvidia_gpu_device_utilization_percent) or on() vector(0)) > 0))) or on() vector(0)) > 0)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) or (label_replace((sum(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) or on() vector(0)), \"namespace\", \"unattributed\", \"\", \"\") unless on() ((count(nvidia_gpu_device_utilization_percent) or on() vector(0)) > 0))) or on() vector(0)) == 0))",
              "refId": "A",
              "legendFormat": "{{namespace}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "percent",
              "color": {
                "mode": "palette-classic"
              }
            },
            "overrides": []
          },
          "options": {
            "legend": {
              "displayMode": "list",
              "placement": "right"
            },
            "pieType": "pie",
            "displayLabels": [],
            "tooltip": {
              "mode": "single"
            },
            "colorScheme": "interpolateSpectral",
            "colorBy": "value",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            }
          },
          "links": [
            {
              "title": "Workload namespaces only",
              "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%21~%22%5E%28kube-.%2A%7C.%2A-system%7Ctraefik%7Cmonitoring%7Clogging%7Ccert-manager%7Cmaintenance%7Cpostgres%29%24%22&var-namespace_scope_ram=${namespace_scope_ram}",
              "targetBlank": false
            },
            {
              "title": "All namespaces",
              "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%3D~%22.%2A%22&var-namespace_scope_ram=${namespace_scope_ram}",
              "targetBlank": false
            },
            {
              "title": "Infrastructure namespaces only",
              "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%3D~%22%5E%28kube-.%2A%7C.%2A-system%7Ctraefik%7Cmonitoring%7Clogging%7Ccert-manager%7Cmaintenance%7Cpostgres%29%24%22&var-namespace_scope_ram=${namespace_scope_ram}",
              "targetBlank": false
            }
          ],
          "description": "Current NVIDIA process-level GPU utilization by namespace. Host covers non-Kubernetes processes; unused fills remaining capacity while active; idle appears only at zero activity."
        },
        {
          "id": 2,
          "type": "timeseries",
          "title": "GPU Process Util by Namespace",
          "datasource": {
            "type": "prometheus",
            "uid": "atlas-vm"
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 0
          },
          "targets": [
            {
              "expr": "sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})",
              "refId": "A",
              "legendFormat": "{{namespace}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "options": {
            "legend": {
              "displayMode": "table",
              "placement": "right"
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "description": "NVML process-level SM utilization by namespace. Host covers GPU work outside Kubernetes pods."
        },
        {
          "id": 3,
          "type": "timeseries",
          "title": "GPU Util by Node",
          "datasource": {
            "type": "prometheus",
            "uid": "atlas-vm"
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 8
          },
          "targets": [
            {
              "expr": "label_replace(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")",
              "refId": "A",
              "legendFormat": "{{Hostname}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "options": {
            "legend": {
              "displayMode": "table",
              "placement": "right"
            },
            "tooltip": {
              "mode": "multi"
            }
          }
        },
        {
          "id": 4,
          "type": "table",
          "title": "GPU Pods Reporting Device Util",
          "datasource": {
            "type": "prometheus",
            "uid": "atlas-vm"
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 8
          },
          "targets": [
            {
              "expr": "topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=\"\"}) by (namespace,pod,Hostname))",
              "refId": "A"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "percent",
              "custom": {
                "filterable": true
              }
            },
            "overrides": []
          },
          "options": {
            "showHeader": true,
            "columnFilters": false
          },
          "transformations": [
            {
              "id": "labelsToFields",
              "options": {}
            }
          ],
          "description": "DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value."
        }
      ],
      "time": {
        "from": "now-12h",
        "to": "now"
      },
      "annotations": {
        "list": []
      },
      "schemaVersion": 39,
      "style": "dark",
      "tags": [
        "atlas",
        "gpu"
      ],
      "templating": {
        "list": [
          {
            "name": "namespace_scope_cpu",
            "label": "CPU namespace filter",
            "type": "custom",
            "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "current": {
              "text": "workload namespaces only",
              "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
              "selected": true
            },
            "options": [
              {
                "text": "workload namespaces only",
                "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": true
              },
              {
                "text": "all namespaces",
                "value": "namespace=~\".*\"",
                "selected": false
              },
              {
                "text": "infrastructure namespaces only",
                "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": false
              }
            ],
            "hide": 2,
            "multi": false,
            "includeAll": false,
            "refresh": 1,
            "sort": 0,
            "skipUrlSync": false
          },
          {
            "name": "namespace_scope_gpu",
            "label": "GPU namespace filter",
            "type": "custom",
            "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "current": {
              "text": "workload namespaces only",
              "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
              "selected": true
            },
            "options": [
              {
                "text": "workload namespaces only",
                "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": true
              },
              {
                "text": "all namespaces",
                "value": "namespace=~\".*\"",
                "selected": false
              },
              {
                "text": "infrastructure namespaces only",
                "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": false
              }
            ],
            "hide": 2,
            "multi": false,
            "includeAll": false,
            "refresh": 1,
            "sort": 0,
            "skipUrlSync": false
          },
          {
            "name": "namespace_scope_ram",
            "label": "RAM namespace filter",
            "type": "custom",
            "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "current": {
              "text": "workload namespaces only",
              "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
              "selected": true
            },
            "options": [
              {
                "text": "workload namespaces only",
                "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": true
              },
              {
                "text": "all namespaces",
                "value": "namespace=~\".*\"",
                "selected": false
              },
              {
                "text": "infrastructure namespaces only",
                "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": false
              }
            ],
            "hide": 2,
            "multi": false,
            "includeAll": false,
            "refresh": 1,
            "sort": 0,
            "skipUrlSync": false
          }
        ]
      }
    }