From e794ccf254e94b48f9137df6fcd72ae15aa4f9d4 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 2 Aug 2026 03:08:20 -0300 Subject: [PATCH] monitoring(gpu): report time-weighted namespace usage --- scripts/dashboards_render_atlas.py | 51 +++++++++++++++++-- scripts/tests/test_dashboards_render_atlas.py | 19 +++++++ services/monitoring/dashboards/atlas-gpu.json | 6 +-- .../monitoring/dashboards/atlas-overview.json | 4 +- services/monitoring/dcgm-exporter.yaml | 5 +- .../monitoring/grafana-dashboard-gpu.yaml | 6 +-- .../grafana-dashboard-overview.yaml | 4 +- 7 files changed, 81 insertions(+), 14 deletions(-) diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index 1eece5ab3..64dab6688 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -284,8 +284,14 @@ def dcgm_gpu_util_by_node(): ) +def nvidia_gpu_util_by_node(): + return "max by (node) (nvidia_gpu_device_utilization_percent)" + + def gpu_util_by_node(): - return f"{dcgm_gpu_util_by_node()} or {jetson_gpu_util_by_node()}" + process_exporter = nvidia_gpu_util_by_node() + dcgm_fallback = f"({dcgm_gpu_util_by_node()}) unless on(node) ({process_exporter})" + return f"{process_exporter} or {dcgm_fallback} or {jetson_gpu_util_by_node()}" def gpu_util_by_hostname(): @@ -357,6 +363,21 @@ def nvidia_process_gpu_usage_by_namespace(scope_var): return f"(({usage}) > 0)" +def nvidia_process_gpu_usage_by_namespace_range(scope_var, window="$__range"): + namespace_samples = ( + "sum_over_time(nvidia_namespace_gpu_sm_util_percent" + f"{{{namespace_gpu_selector(scope_var)}}}[{window}])" + ) + device_samples = ( + f"count_over_time(nvidia_gpu_device_utilization_percent[{window}])" + ) + averaged = ( + f"({namespace_samples}) / on(node,gpu,uuid,model) group_left() " + f"clamp_min({device_samples}, 1)" + ) + return f"((sum by (namespace) ({averaged})) > 0)" + + def nvidia_gpu_device_utilization(): return "max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))" @@ -390,17 +411,41 @@ def unattributed_gpu_usage(): ) +def legacy_gpu_util_range_without_process_exporter(window="$__range"): + dcgm_range = dcgm_gpu_util_by_node().replace( + "DCGM_FI_DEV_GPU_UTIL", f"avg_over_time(DCGM_FI_DEV_GPU_UTIL[{window}])" + ) + jetson_range = ( + f'max by (node) (avg_over_time(jetson_gr3d_freq_percent{{node!=""}}[{window}]))' + ) + process_exporter_nodes = ( + f"max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[{window}]))" + ) + return f"(({dcgm_range}) or ({jetson_range})) unless on(node) ({process_exporter_nodes})" + + +def unattributed_gpu_usage_range(window="$__range"): + total = f"(sum({legacy_gpu_util_range_without_process_exporter(window)}) or on() vector(0))" + return f'label_replace(({total} > 0), "namespace", "unattributed", "", "")' + + def gpu_utilization_raw(scope_var): return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})" +def gpu_utilization_range_raw(scope_var, window="$__range"): + attributed = nvidia_process_gpu_usage_by_namespace_range(scope_var, window) + unattributed = unattributed_gpu_usage_range(window) + return f"({attributed}) or ({unattributed})" + + def gpu_pool_used_expr(scope_var): raw_total = f"(sum({gpu_utilization_raw(scope_var)}) or on() vector(0))" return f"100 * {raw_total} / clamp_min({gpu_capacity_percent()}, 1)" def namespace_gpu_share_expr(scope_var): - activity = gpu_utilization_raw(scope_var) + activity = gpu_utilization_range_raw(scope_var) total = f"(sum({activity}) or on() vector(0))" share = f"100 * ({activity}) / clamp_min({total}, 1)" idle = ( @@ -1910,7 +1955,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = { "Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.", "Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.", "Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.", - "Namespace GPU Utilization": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.", + "Namespace GPU Utilization": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero.", "Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.", "Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.", "Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.", diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index 3740589e4..3269adaf4 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -157,6 +157,11 @@ def test_overview_uses_readable_quality_power_and_gitops_panels(): gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"] assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr assert "nvidia_gpu_device_utilization_percent" in gpu_expr + assert "sum_over_time" in gpu_expr + assert "count_over_time" in gpu_expr + assert "avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range])" in gpu_expr + assert "avg_over_time(jetson_gr3d_freq_percent" in gpu_expr + assert "$__range" in gpu_expr assert "sum by (namespace)" in gpu_expr assert 'namespace", "shared"' not in gpu_expr assert "kube_node_labels" not in gpu_expr @@ -165,6 +170,20 @@ def test_overview_uses_readable_quality_power_and_gitops_panels(): assert 'namespace", "unattributed"' in gpu_expr assert 'namespace", "idle"' in gpu_expr assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True + assert "Time-weighted share" in panels_by_title["Namespace GPU Utilization"]["description"] + + +def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families(): + mod = load_module() + dashboard = mod.build_gpu_dashboard() + panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])} + + node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"] + assert "nvidia_gpu_device_utilization_percent" in node_expr + assert "DCGM_FI_DEV_GPU_UTIL" in node_expr + assert "jetson_gr3d_freq_percent" in node_expr + assert "unless on(node)" in node_expr + assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"] def test_overview_and_testing_panels_all_have_concise_descriptions(): diff --git a/services/monitoring/dashboards/atlas-gpu.json b/services/monitoring/dashboards/atlas-gpu.json index 6bc000fdc..1e56b6ba8 100644 --- a/services/monitoring/dashboards/atlas-gpu.json +++ b/services/monitoring/dashboards/atlas-gpu.json @@ -20,7 +20,7 @@ }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -72,7 +72,7 @@ "targetBlank": false } ], - "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero." + "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero." }, { "id": 2, @@ -128,7 +128,7 @@ }, "targets": [ { - "expr": "label_replace(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{Hostname}}" } diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index 9728f5767..ecac87845 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -3728,7 +3728,7 @@ }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -3780,7 +3780,7 @@ "targetBlank": false } ], - "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero." + "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero." }, { "id": 13, diff --git a/services/monitoring/dcgm-exporter.yaml b/services/monitoring/dcgm-exporter.yaml index e8c3b03da..74885e18b 100644 --- a/services/monitoring/dcgm-exporter.yaml +++ b/services/monitoring/dcgm-exporter.yaml @@ -65,7 +65,10 @@ spec: resources: requests: cpu: 50m - memory: 64Mi + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi volumeMounts: - name: pod-resources mountPath: /var/lib/kubelet/pod-resources diff --git a/services/monitoring/grafana-dashboard-gpu.yaml b/services/monitoring/grafana-dashboard-gpu.yaml index 40692ea77..f57ac7138 100644 --- a/services/monitoring/grafana-dashboard-gpu.yaml +++ b/services/monitoring/grafana-dashboard-gpu.yaml @@ -29,7 +29,7 @@ data: }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -81,7 +81,7 @@ data: "targetBlank": false } ], - "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero." + "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero." }, { "id": 2, @@ -137,7 +137,7 @@ data: }, "targets": [ { - "expr": "label_replace(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")", + "expr": "label_replace(max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")", "refId": "A", "legendFormat": "{{Hostname}}" } diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index 55bcaef75..bf6ce496e 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -3737,7 +3737,7 @@ data: }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -3789,7 +3789,7 @@ data: "targetBlank": false } ], - "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero." + "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero." }, { "id": 13,