titan-iac/services/monitoring/grafana-dashboard-gpu.yaml

# services/monitoring/grafana-dashboard-gpu.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-dashboard-gpu
  labels:
    grafana_dashboard: "1"
data:
  atlas-gpu.json: |
    {
      "uid": "atlas-gpu",
      "title": "Atlas GPU",
      "folderUid": "atlas-internal",
      "editable": true,
      "panels": [
        {
          "id": 1,
          "type": "piechart",
          "title": "Namespace GPU Utilization",
          "datasource": {
            "type": "prometheus",
            "uid": "atlas-vm"
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 0
          },
          "targets": [
            {
              "expr": "(100 * ((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"}
              "refId": "A",
              "legendFormat": "{{namespace}}",
              "instant": true
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "percent",
              "color": {
                "mode": "palette-classic"
              }
            },
            "overrides": []
          },
          "options": {
            "legend": {
              "displayMode": "list",
              "placement": "right"
            },
            "pieType": "pie",
            "displayLabels": [],
            "tooltip": {
              "mode": "single"
            },
            "colorScheme": "interpolateSpectral",
            "colorBy": "value",
            "reduceOptions": {
              "calcs": [
                "lastNotNull"
              ],
              "fields": "",
              "values": false
            }
          },
          "links": [
            {
              "title": "Workload namespaces only",
              "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%21~%22%5E%28kube-.%2A%7C.%2A-system%7Ctraefik%7Cmonitoring%7Clogging%7Ccert-manager%7Cmaintenance%7Cpostgres%29%24%22&var-namespace_scope_ram=${namespace_scope_ram}",
              "targetBlank": false
            },
            {
              "title": "All namespaces",
              "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%3D~%22.%2A%22&var-namespace_scope_ram=${namespace_scope_ram}",
              "targetBlank": false
            },
            {
              "title": "Infrastructure namespaces only",
              "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%3D~%22%5E%28kube-.%2A%7C.%2A-system%7Ctraefik%7Cmonitoring%7Clogging%7Ccert-manager%7Cmaintenance%7Cpostgres%29%24%22&var-namespace_scope_ram=${namespace_scope_ram}",
              "targetBlank": false
            }
          ],
          "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero."
        },
        {
          "id": 2,
          "type": "timeseries",
          "title": "GPU Process Util by Namespace",
          "datasource": {
            "type": "prometheus",
            "uid": "atlas-vm"
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 0
          },
          "targets": [
            {
              "expr": "((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)",
              "refId": "A",
              "legendFormat": "{{namespace}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "options": {
            "legend": {
              "displayMode": "table",
              "placement": "right"
            },
            "tooltip": {
              "mode": "multi"
            }
          },
          "description": "NVML process-level SM utilization by namespace. Host covers GPU work outside Kubernetes pods."
        },
        {
          "id": 3,
          "type": "timeseries",
          "title": "GPU Util by Node",
          "datasource": {
            "type": "prometheus",
            "uid": "atlas-vm"
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 0,
            "y": 8
          },
          "targets": [
            {
              "expr": "label_replace(max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")",
              "refId": "A",
              "legendFormat": "{{Hostname}}"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "percent"
            },
            "overrides": []
          },
          "options": {
            "legend": {
              "displayMode": "table",
              "placement": "right"
            },
            "tooltip": {
              "mode": "multi"
            }
          }
        },
        {
          "id": 4,
          "type": "table",
          "title": "GPU Processes by Pod",
          "datasource": {
            "type": "prometheus",
            "uid": "atlas-vm"
          },
          "gridPos": {
            "h": 8,
            "w": 12,
            "x": 12,
            "y": 8
          },
          "targets": [
            {
              "expr": "topk(10, sum by (namespace,pod,node,process) (nvidia_process_gpu_sm_util_percent{pod!=\"host\"}) > 0)",
              "refId": "A"
            }
          ],
          "fieldConfig": {
            "defaults": {
              "unit": "percent",
              "custom": {
                "filterable": true
              }
            },
            "overrides": []
          },
          "options": {
            "showHeader": true,
            "columnFilters": false
          },
          "transformations": [
            {
              "id": "labelsToFields",
              "options": {}
            }
          ],
          "description": "NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization."
        }
      ],
      "time": {
        "from": "now-12h",
        "to": "now"
      },
      "annotations": {
        "list": []
      },
      "schemaVersion": 39,
      "style": "dark",
      "tags": [
        "atlas",
        "gpu"
      ],
      "templating": {
        "list": [
          {
            "name": "namespace_scope_cpu",
            "label": "CPU namespace filter",
            "type": "custom",
            "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "current": {
              "text": "workload namespaces only",
              "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
              "selected": true
            },
            "options": [
              {
                "text": "workload namespaces only",
                "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": true
              },
              {
                "text": "all namespaces",
                "value": "namespace=~\".*\"",
                "selected": false
              },
              {
                "text": "infrastructure namespaces only",
                "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": false
              }
            ],
            "hide": 2,
            "multi": false,
            "includeAll": false,
            "refresh": 1,
            "sort": 0,
            "skipUrlSync": false
          },
          {
            "name": "namespace_scope_gpu",
            "label": "GPU namespace filter",
            "type": "custom",
            "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "current": {
              "text": "workload namespaces only",
              "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
              "selected": true
            },
            "options": [
              {
                "text": "workload namespaces only",
                "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": true
              },
              {
                "text": "all namespaces",
                "value": "namespace=~\".*\"",
                "selected": false
              },
              {
                "text": "infrastructure namespaces only",
                "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": false
              }
            ],
            "hide": 2,
            "multi": false,
            "includeAll": false,
            "refresh": 1,
            "sort": 0,
            "skipUrlSync": false
          },
          {
            "name": "namespace_scope_ram",
            "label": "RAM namespace filter",
            "type": "custom",
            "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "current": {
              "text": "workload namespaces only",
              "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
              "selected": true
            },
            "options": [
              {
                "text": "workload namespaces only",
                "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": true
              },
              {
                "text": "all namespaces",
                "value": "namespace=~\".*\"",
                "selected": false
              },
              {
                "text": "infrastructure namespaces only",
                "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
                "selected": false
              }
            ],
            "hide": 2,
            "multi": false,
            "includeAll": false,
            "refresh": 1,
            "sort": 0,
            "skipUrlSync": false
          }
        ]
      }
    }