{
  "uid": "atlas-gpu",
  "title": "Atlas GPU",
  "folderUid": "atlas-internal",
  "editable": true,
  "panels": [
    {
      "id": 1,
      "type": "piechart",
      "title": "Namespace GPU Utilization",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "targets": [
        {
          "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
          "refId": "A",
          "legendFormat": "{{namespace}}",
          "instant": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "color": {
            "mode": "palette-classic"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "right"
        },
        "pieType": "pie",
        "displayLabels": [],
        "tooltip": {
          "mode": "single"
        },
        "colorScheme": "interpolateSpectral",
        "colorBy": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      },
      "links": [
        {
          "title": "Workload namespaces only",
          "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%21~%22%5E%28kube-.%2A%7C.%2A-system%7Ctraefik%7Cmonitoring%7Clogging%7Ccert-manager%7Cmaintenance%7Cpostgres%29%24%22&var-namespace_scope_ram=${namespace_scope_ram}",
          "targetBlank": false
        },
        {
          "title": "All namespaces",
          "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%3D~%22.%2A%22&var-namespace_scope_ram=${namespace_scope_ram}",
          "targetBlank": false
        },
        {
          "title": "Infrastructure namespaces only",
          "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%3D~%22%5E%28kube-.%2A%7C.%2A-system%7Ctraefik%7Cmonitoring%7Clogging%7Ccert-manager%7Cmaintenance%7Cpostgres%29%24%22&var-namespace_scope_ram=${namespace_scope_ram}",
          "targetBlank": false
        }
      ],
      "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero."
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "GPU Process Util by Namespace",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 0
      },
      "targets": [
        {
          "expr": "((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)",
          "refId": "A",
          "legendFormat": "{{namespace}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "description": "NVML process-level SM utilization by namespace. Host covers GPU work outside Kubernetes pods."
    },
    {
      "id": 3,
      "type": "timeseries",
      "title": "GPU Util by Node",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 8
      },
      "targets": [
        {
          "expr": "label_replace(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")",
          "refId": "A",
          "legendFormat": "{{Hostname}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 4,
      "type": "table",
      "title": "GPU Pods Reporting Device Util",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 8
      },
      "targets": [
        {
          "expr": "topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=\"\"}) by (namespace,pod,Hostname))",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "custom": {
            "filterable": true
          }
        },
        "overrides": []
      },
      "options": {
        "showHeader": true,
        "columnFilters": false
      },
      "transformations": [
        {
          "id": "labelsToFields",
          "options": {}
        }
      ],
      "description": "DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value."
    }
  ],
  "time": {
    "from": "now-12h",
    "to": "now"
  },
  "annotations": {
    "list": []
  },
  "schemaVersion": 39,
  "style": "dark",
  "tags": [
    "atlas",
    "gpu"
  ],
  "templating": {
    "list": [
      {
        "name": "namespace_scope_cpu",
        "label": "CPU namespace filter",
        "type": "custom",
        "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
        "current": {
          "text": "workload namespaces only",
          "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
          "selected": true
        },
        "options": [
          {
            "text": "workload namespaces only",
            "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": true
          },
          {
            "text": "all namespaces",
            "value": "namespace=~\".*\"",
            "selected": false
          },
          {
            "text": "infrastructure namespaces only",
            "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": false
          }
        ],
        "hide": 2,
        "multi": false,
        "includeAll": false,
        "refresh": 1,
        "sort": 0,
        "skipUrlSync": false
      },
      {
        "name": "namespace_scope_gpu",
        "label": "GPU namespace filter",
        "type": "custom",
        "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
        "current": {
          "text": "workload namespaces only",
          "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
          "selected": true
        },
        "options": [
          {
            "text": "workload namespaces only",
            "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": true
          },
          {
            "text": "all namespaces",
            "value": "namespace=~\".*\"",
            "selected": false
          },
          {
            "text": "infrastructure namespaces only",
            "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": false
          }
        ],
        "hide": 2,
        "multi": false,
        "includeAll": false,
        "refresh": 1,
        "sort": 0,
        "skipUrlSync": false
      },
      {
        "name": "namespace_scope_ram",
        "label": "RAM namespace filter",
        "type": "custom",
        "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
        "current": {
          "text": "workload namespaces only",
          "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
          "selected": true
        },
        "options": [
          {
            "text": "workload namespaces only",
            "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": true
          },
          {
            "text": "all namespaces",
            "value": "namespace=~\".*\"",
            "selected": false
          },
          {
            "text": "infrastructure namespaces only",
            "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": false
          }
        ],
        "hide": 2,
        "multi": false,
        "includeAll": false,
        "refresh": 1,
        "sort": 0,
        "skipUrlSync": false
      }
    ]
  }
}
