monitoring(gpu): use process-level pod attribution

scripts/dashboards_render_atlas.py

@@ -5535,12 +5535,13 @@ def build_gpu_dashboard():
     panels.append(
         table_panel(
             4,
-            "GPU Pods Reporting Device Util",
+            "GPU Processes by Pod",
-            'topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=""}) by (namespace,pod,Hostname))',
+            'topk(10, sum by (namespace,pod,node,process) '
+            '(nvidia_process_gpu_sm_util_percent{pod!="host"}) > 0)',
             {"h": 8, "w": 12, "x": 12, "y": 8},
             unit="percent",
             transformations=[{"id": "labelsToFields", "options": {}}],
-            description="DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value.",
+            description="NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization.",
         )
     )
     return {

scripts/tests/test_dashboards_render_atlas.py

@@ -185,6 +185,10 @@ def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_famili
     assert "unless on(node)" in node_expr
     assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
 
+    process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"]
+    assert "nvidia_process_gpu_sm_util_percent" in process_expr
+    assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr
+
 
 def test_overview_and_testing_panels_all_have_concise_descriptions():
     mod = load_module()

services/monitoring/dashboards/atlas-gpu.json

@@ -152,7 +152,7 @@
     {
       "id": 4,
       "type": "table",
-      "title": "GPU Pods Reporting Device Util",
+      "title": "GPU Processes by Pod",
       "datasource": {
         "type": "prometheus",
         "uid": "atlas-vm"
@@ -165,7 +165,7 @@
       },
       "targets": [
         {
-          "expr": "topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=\"\"}) by (namespace,pod,Hostname))",
+          "expr": "topk(10, sum by (namespace,pod,node,process) (nvidia_process_gpu_sm_util_percent{pod!=\"host\"}) > 0)",
           "refId": "A"
         }
       ],
@@ -188,7 +188,7 @@
           "options": {}
         }
       ],
-      "description": "DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value."
+      "description": "NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization."
     }
   ],
   "time": {

services/monitoring/dcgm-exporter.yaml

@@ -53,13 +53,6 @@ spec:
           ports:
             - name: metrics
               containerPort: 9400
-          env:
-            - name: DCGM_EXPORTER_KUBERNETES
-              value: "true"
-            - name: KUBERNETES_VIRTUAL_GPUS
-              value: "true"
-            - name: NVIDIA_RESOURCE_NAMES
-              value: nvidia.com/gpu.shared
           securityContext:
             privileged: true
           resources:
@@ -69,14 +62,6 @@ spec:
             limits:
               cpu: 500m
               memory: 1Gi
-          volumeMounts:
-            - name: pod-resources
-              mountPath: /var/lib/kubelet/pod-resources
-      volumes:
-        - name: pod-resources
-          hostPath:
-            path: /var/lib/kubelet/pod-resources
-            type: Directory
 ---
 apiVersion: v1
 kind: Service

services/monitoring/grafana-dashboard-gpu.yaml

@@ -161,7 +161,7 @@ data:
         {
           "id": 4,
           "type": "table",
-          "title": "GPU Pods Reporting Device Util",
+          "title": "GPU Processes by Pod",
           "datasource": {
             "type": "prometheus",
             "uid": "atlas-vm"
@@ -174,7 +174,7 @@ data:
           },
           "targets": [
             {
-              "expr": "topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=\"\"}) by (namespace,pod,Hostname))",
+              "expr": "topk(10, sum by (namespace,pod,node,process) (nvidia_process_gpu_sm_util_percent{pod!=\"host\"}) > 0)",
               "refId": "A"
             }
           ],
@@ -197,7 +197,7 @@ data:
               "options": {}
             }
           ],
-          "description": "DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value."
+          "description": "NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization."
         }
       ],
       "time": {