diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index 64dab6688..edbecc8f3 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -5535,12 +5535,13 @@ def build_gpu_dashboard(): panels.append( table_panel( 4, - "GPU Pods Reporting Device Util", - 'topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=""}) by (namespace,pod,Hostname))', + "GPU Processes by Pod", + 'topk(10, sum by (namespace,pod,node,process) ' + '(nvidia_process_gpu_sm_util_percent{pod!="host"}) > 0)', {"h": 8, "w": 12, "x": 12, "y": 8}, unit="percent", transformations=[{"id": "labelsToFields", "options": {}}], - description="DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value.", + description="NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization.", ) ) return { diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index 3269adaf4..51273ff32 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -185,6 +185,10 @@ def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_famili assert "unless on(node)" in node_expr assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"] + process_expr = panels_by_title["GPU Processes by Pod"]["targets"][0]["expr"] + assert "nvidia_process_gpu_sm_util_percent" in process_expr + assert "DCGM_FI_DEV_GPU_UTIL" not in process_expr + def test_overview_and_testing_panels_all_have_concise_descriptions(): mod = load_module() diff --git a/services/monitoring/dashboards/atlas-gpu.json b/services/monitoring/dashboards/atlas-gpu.json index 1e56b6ba8..d267bd678 100644 --- a/services/monitoring/dashboards/atlas-gpu.json +++ b/services/monitoring/dashboards/atlas-gpu.json @@ -152,7 +152,7 @@ { "id": 4, "type": "table", - "title": "GPU Pods Reporting Device Util", + "title": "GPU Processes by Pod", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -165,7 +165,7 @@ }, "targets": [ { - "expr": "topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=\"\"}) by (namespace,pod,Hostname))", + "expr": "topk(10, sum by (namespace,pod,node,process) (nvidia_process_gpu_sm_util_percent{pod!=\"host\"}) > 0)", "refId": "A" } ], @@ -188,7 +188,7 @@ "options": {} } ], - "description": "DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value." + "description": "NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization." } ], "time": { diff --git a/services/monitoring/dcgm-exporter.yaml b/services/monitoring/dcgm-exporter.yaml index 74885e18b..c57af0679 100644 --- a/services/monitoring/dcgm-exporter.yaml +++ b/services/monitoring/dcgm-exporter.yaml @@ -53,13 +53,6 @@ spec: ports: - name: metrics containerPort: 9400 - env: - - name: DCGM_EXPORTER_KUBERNETES - value: "true" - - name: KUBERNETES_VIRTUAL_GPUS - value: "true" - - name: NVIDIA_RESOURCE_NAMES - value: nvidia.com/gpu.shared securityContext: privileged: true resources: @@ -69,14 +62,6 @@ spec: limits: cpu: 500m memory: 1Gi - volumeMounts: - - name: pod-resources - mountPath: /var/lib/kubelet/pod-resources - volumes: - - name: pod-resources - hostPath: - path: /var/lib/kubelet/pod-resources - type: Directory --- apiVersion: v1 kind: Service diff --git a/services/monitoring/grafana-dashboard-gpu.yaml b/services/monitoring/grafana-dashboard-gpu.yaml index f57ac7138..94cf3b1ad 100644 --- a/services/monitoring/grafana-dashboard-gpu.yaml +++ b/services/monitoring/grafana-dashboard-gpu.yaml @@ -161,7 +161,7 @@ data: { "id": 4, "type": "table", - "title": "GPU Pods Reporting Device Util", + "title": "GPU Processes by Pod", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -174,7 +174,7 @@ data: }, "targets": [ { - "expr": "topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=\"\"}) by (namespace,pod,Hostname))", + "expr": "topk(10, sum by (namespace,pod,node,process) (nvidia_process_gpu_sm_util_percent{pod!=\"host\"}) > 0)", "refId": "A" } ], @@ -197,7 +197,7 @@ data: "options": {} } ], - "description": "DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value." + "description": "NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization." } ], "time": {