titan-iac/services/monitoring/dashboards/atlas-gpu.json

{
  "uid": "atlas-gpu",
  "title": "Atlas GPU",
  "folderUid": "atlas-internal",
  "editable": true,
  "panels": [
    {
      "id": 1,
      "type": "piechart",
      "title": "Namespace GPU Share",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "targets": [
        {
          "expr": "(100 * (sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope}) by (namespace)) / clamp_min((sum(sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope}) by (namespace)) or on() vector(0)), 1)) or label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum(sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope}) by (namespace)) or on() vector(0)) == 0)",
          "refId": "A",
          "legendFormat": "{{namespace}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "color": {
            "mode": "palette-classic"
          }
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "list",
          "placement": "right"
        },
        "pieType": "pie",
        "displayLabels": [],
        "tooltip": {
          "mode": "single"
        },
        "colorScheme": "interpolateSpectral",
        "colorBy": "value",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        }
      }
    },
    {
      "id": 2,
      "type": "timeseries",
      "title": "GPU Util by Namespace",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 0
      },
      "targets": [
        {
          "expr": "sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope}) by (namespace)",
          "refId": "A",
          "legendFormat": "{{namespace}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 3,
      "type": "timeseries",
      "title": "GPU Util by Node",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 8
      },
      "targets": [
        {
          "expr": "sum by (Hostname) (DCGM_FI_DEV_GPU_UTIL{pod!=\"\"})",
          "refId": "A",
          "legendFormat": "{{Hostname}}"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent"
        },
        "overrides": []
      },
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right"
        },
        "tooltip": {
          "mode": "multi"
        }
      }
    },
    {
      "id": 4,
      "type": "table",
      "title": "Top Pods by GPU Util",
      "datasource": {
        "type": "prometheus",
        "uid": "atlas-vm"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 8
      },
      "targets": [
        {
          "expr": "topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=\"\"}) by (namespace,pod,Hostname))",
          "refId": "A"
        }
      ],
      "fieldConfig": {
        "defaults": {
          "unit": "percent",
          "custom": {
            "filterable": true
          }
        },
        "overrides": []
      },
      "options": {
        "showHeader": true,
        "columnFilters": false
      },
      "transformations": [
        {
          "id": "labelsToFields",
          "options": {}
        }
      ]
    }
  ],
  "time": {
    "from": "now-12h",
    "to": "now"
  },
  "annotations": {
    "list": []
  },
  "schemaVersion": 39,
  "style": "dark",
  "tags": [
    "atlas",
    "gpu"
  ],
  "templating": {
    "list": [
      {
        "name": "namespace_scope",
        "label": "Namespace filter",
        "type": "custom",
        "query": "workload namespaces only : namespace!~\"(^kube.*|.*-system$|^traefik$)\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"(^kube.*|.*-system$|^traefik$)\"",
        "current": {
          "text": "workload namespaces only",
          "value": "namespace!~\"(^kube.*|.*-system$|^traefik$)\"",
          "selected": true
        },
        "options": [
          {
            "text": "workload namespaces only",
            "value": "namespace!~\"(^kube.*|.*-system$|^traefik$)\"",
            "selected": true
          },
          {
            "text": "all namespaces",
            "value": "namespace=~\".*\"",
            "selected": false
          },
          {
            "text": "infrastructure namespaces only",
            "value": "namespace=~\"(^kube.*|.*-system$|^traefik$)\"",
            "selected": false
          }
        ],
        "hide": 0,
        "multi": false,
        "includeAll": false,
        "refresh": 1,
        "sort": 0,
        "skipUrlSync": false
      }
    ]
  }
}