titan-iac/services/monitoring/dashboards/atlas-gpu.json

{
  "uid": "atlas-gpu",
  "title": "Atlas GPU",
  "folderUid": "atlas-internal",
  "editable": true,
  "panels": [
    {
      "id": 1,
      "type": "piechart",
      "title": "Namespace GPU Utilization",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "targets": [
        {
          "expr": "(100 * ((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() max by (node) (jetson_gr3d_freq_percent{node!=\"\"}))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (max by (node) (jetson_gr3d_freq_percent{node!=\"\"})))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((sum by (namespace) ((label_replace(((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * o
          "refId": "A",
          "legendFormat": "{{namespace}}",
          "instant": true
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "color": {
            "mode": "palette-classic"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "right"
        },
        "pieType": "pie",
        "displayLabels": [],
        "tooltip": {
          "mode": "single"
        },
        "colorScheme": "interpolateSpectral",
        "colorBy": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      },
      "links": [
        {
          "title": "Workload namespaces only",
          "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%21~%22%5E%28kube-.%2A%7C.%2A-system%7Ctraefik%7Cmonitoring%7Clogging%7Ccert-manager%7Cmaintenance%7Cpostgres%29%24%22&var-namespace_scope_ram=${namespace_scope_ram}",
          "targetBlank": false
        },
        {
          "title": "All namespaces",
          "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%3D~%22.%2A%22&var-namespace_scope_ram=${namespace_scope_ram}",
          "targetBlank": false
        },
        {
          "title": "Infrastructure namespaces only",
          "url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%3D~%22%5E%28kube-.%2A%7C.%2A-system%7Ctraefik%7Cmonitoring%7Clogging%7Ccert-manager%7Cmaintenance%7Cpostgres%29%24%22&var-namespace_scope_ram=${namespace_scope_ram}",
          "targetBlank": false
        }
      ],
      "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero."
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "GPU Process Util by Namespace",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 0
      },
      "targets": [
        {
          "expr": "((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)",
          "refId": "A",
          "legendFormat": "{{namespace}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right"
        },
        "tooltip": {
          "mode": "multi"
        }
      },
      "description": "NVML process-level SM utilization by namespace. Host covers GPU work outside Kubernetes pods."
    },
    {
      "id": 3,
      "type": "timeseries",
      "title": "GPU Util by Node",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 8
      },
      "targets": [
        {
          "expr": "label_replace(max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")",
          "refId": "A",
          "legendFormat": "{{Hostname}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 4,
      "type": "table",
      "title": "GPU Processes by Pod",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 8
      },
      "targets": [
        {
          "expr": "topk(10, sum by (namespace,pod,node,process) (nvidia_process_gpu_sm_util_percent{pod!=\"host\"}) > 0)",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "custom": {
            "filterable": true
          }
        },
        "overrides": []
      },
      "options": {
        "showHeader": true,
        "columnFilters": false
      },
      "transformations": [
        {
          "id": "labelsToFields",
          "options": {}
        }
      ],
      "description": "NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization."
    }
  ],
  "time": {
    "from": "now-12h",
    "to": "now"
  },
  "annotations": {
    "list": []
  },
  "schemaVersion": 39,
  "style": "dark",
  "tags": [
    "atlas",
    "gpu"
  ],
  "templating": {
    "list": [
      {
        "name": "namespace_scope_cpu",
        "label": "CPU namespace filter",
        "type": "custom",
        "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
        "current": {
          "text": "workload namespaces only",
          "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
          "selected": true
        },
        "options": [
          {
            "text": "workload namespaces only",
            "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": true
          },
          {
            "text": "all namespaces",
            "value": "namespace=~\".*\"",
            "selected": false
          },
          {
            "text": "infrastructure namespaces only",
            "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": false
          }
        ],
        "hide": 2,
        "multi": false,
        "includeAll": false,
        "refresh": 1,
        "sort": 0,
        "skipUrlSync": false
      },
      {
        "name": "namespace_scope_gpu",
        "label": "GPU namespace filter",
        "type": "custom",
        "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
        "current": {
          "text": "workload namespaces only",
          "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
          "selected": true
        },
        "options": [
          {
            "text": "workload namespaces only",
            "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": true
          },
          {
            "text": "all namespaces",
            "value": "namespace=~\".*\"",
            "selected": false
          },
          {
            "text": "infrastructure namespaces only",
            "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": false
          }
        ],
        "hide": 2,
        "multi": false,
        "includeAll": false,
        "refresh": 1,
        "sort": 0,
        "skipUrlSync": false
      },
      {
        "name": "namespace_scope_ram",
        "label": "RAM namespace filter",
        "type": "custom",
        "query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
        "current": {
          "text": "workload namespaces only",
          "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
          "selected": true
        },
        "options": [
          {
            "text": "workload namespaces only",
            "value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": true
          },
          {
            "text": "all namespaces",
            "value": "namespace=~\".*\"",
            "selected": false
          },
          {
            "text": "infrastructure namespaces only",
            "value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
            "selected": false
          }
        ],
        "hide": 2,
        "multi": false,
        "includeAll": false,
        "refresh": 1,
        "sort": 0,
        "skipUrlSync": false
      }
    ]
  }
}