monitoring(gpu): report time-weighted namespace usage

scripts/dashboards_render_atlas.py

@@ -284,8 +284,14 @@ def dcgm_gpu_util_by_node():
     )
 
 
+def nvidia_gpu_util_by_node():
+    return "max by (node) (nvidia_gpu_device_utilization_percent)"
+
+
 def gpu_util_by_node():
-    return f"{dcgm_gpu_util_by_node()} or {jetson_gpu_util_by_node()}"
+    process_exporter = nvidia_gpu_util_by_node()
+    dcgm_fallback = f"({dcgm_gpu_util_by_node()}) unless on(node) ({process_exporter})"
+    return f"{process_exporter} or {dcgm_fallback} or {jetson_gpu_util_by_node()}"
 
 
 def gpu_util_by_hostname():
@@ -357,6 +363,21 @@ def nvidia_process_gpu_usage_by_namespace(scope_var):
     return f"(({usage}) > 0)"
 
 
+def nvidia_process_gpu_usage_by_namespace_range(scope_var, window="$__range"):
+    namespace_samples = (
+        "sum_over_time(nvidia_namespace_gpu_sm_util_percent"
+        f"{{{namespace_gpu_selector(scope_var)}}}[{window}])"
+    )
+    device_samples = (
+        f"count_over_time(nvidia_gpu_device_utilization_percent[{window}])"
+    )
+    averaged = (
+        f"({namespace_samples}) / on(node,gpu,uuid,model) group_left() "
+        f"clamp_min({device_samples}, 1)"
+    )
+    return f"((sum by (namespace) ({averaged})) > 0)"
+
+
 def nvidia_gpu_device_utilization():
     return "max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))"
 
@@ -390,17 +411,41 @@ def unattributed_gpu_usage():
     )
 
 
+def legacy_gpu_util_range_without_process_exporter(window="$__range"):
+    dcgm_range = dcgm_gpu_util_by_node().replace(
+        "DCGM_FI_DEV_GPU_UTIL", f"avg_over_time(DCGM_FI_DEV_GPU_UTIL[{window}])"
+    )
+    jetson_range = (
+        f'max by (node) (avg_over_time(jetson_gr3d_freq_percent{{node!=""}}[{window}]))'
+    )
+    process_exporter_nodes = (
+        f"max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[{window}]))"
+    )
+    return f"(({dcgm_range}) or ({jetson_range})) unless on(node) ({process_exporter_nodes})"
+
+
+def unattributed_gpu_usage_range(window="$__range"):
+    total = f"(sum({legacy_gpu_util_range_without_process_exporter(window)}) or on() vector(0))"
+    return f'label_replace(({total} > 0), "namespace", "unattributed", "", "")'
+
+
 def gpu_utilization_raw(scope_var):
     return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})"
 
 
+def gpu_utilization_range_raw(scope_var, window="$__range"):
+    attributed = nvidia_process_gpu_usage_by_namespace_range(scope_var, window)
+    unattributed = unattributed_gpu_usage_range(window)
+    return f"({attributed}) or ({unattributed})"
+
+
 def gpu_pool_used_expr(scope_var):
     raw_total = f"(sum({gpu_utilization_raw(scope_var)}) or on() vector(0))"
     return f"100 * {raw_total} / clamp_min({gpu_capacity_percent()}, 1)"
 
 
 def namespace_gpu_share_expr(scope_var):
-    activity = gpu_utilization_raw(scope_var)
+    activity = gpu_utilization_range_raw(scope_var)
     total = f"(sum({activity}) or on() vector(0))"
     share = f"100 * ({activity}) / clamp_min({total}, 1)"
     idle = (
@@ -1910,7 +1955,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
     "Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.",
     "Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.",
     "Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.",
-    "Namespace GPU Utilization": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.",
+    "Namespace GPU Utilization": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero.",
     "Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.",
     "Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.",
     "Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.",

scripts/tests/test_dashboards_render_atlas.py

@@ -157,6 +157,11 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
     assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
     assert "nvidia_gpu_device_utilization_percent" in gpu_expr
+    assert "sum_over_time" in gpu_expr
+    assert "count_over_time" in gpu_expr
+    assert "avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range])" in gpu_expr
+    assert "avg_over_time(jetson_gr3d_freq_percent" in gpu_expr
+    assert "$__range" in gpu_expr
     assert "sum by (namespace)" in gpu_expr
     assert 'namespace", "shared"' not in gpu_expr
     assert "kube_node_labels" not in gpu_expr
@@ -165,6 +170,20 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     assert 'namespace", "unattributed"' in gpu_expr
     assert 'namespace", "idle"' in gpu_expr
     assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
+    assert "Time-weighted share" in panels_by_title["Namespace GPU Utilization"]["description"]
+
+
+def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():
+    mod = load_module()
+    dashboard = mod.build_gpu_dashboard()
+    panels_by_title = {panel["title"]: panel for panel in flatten_panels(dashboard["panels"])}
+
+    node_expr = panels_by_title["GPU Util by Node"]["targets"][0]["expr"]
+    assert "nvidia_gpu_device_utilization_percent" in node_expr
+    assert "DCGM_FI_DEV_GPU_UTIL" in node_expr
+    assert "jetson_gr3d_freq_percent" in node_expr
+    assert "unless on(node)" in node_expr
+    assert mod.GPU_NODES == ["titan-20", "titan-21", "titan-22", "titan-24"]
 
 
 def test_overview_and_testing_panels_all_have_concise_descriptions():

services/monitoring/dashboards/atlas-gpu.json

@@ -20,7 +20,7 @@
       },
       "targets": [
         {
-          "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
+          "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
           "refId": "A",
           "legendFormat": "{{namespace}}",
           "instant": true
@@ -72,7 +72,7 @@
           "targetBlank": false
         }
       ],
-      "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero."
+      "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
     },
     {
       "id": 2,
@@ -128,7 +128,7 @@
       },
       "targets": [
         {
-          "expr": "label_replace(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")",
+          "expr": "label_replace(max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")",
           "refId": "A",
           "legendFormat": "{{Hostname}}"
         }

services/monitoring/dashboards/atlas-overview.json

@@ -3728,7 +3728,7 @@
       },
       "targets": [
         {
-          "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
+          "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
           "refId": "A",
           "legendFormat": "{{namespace}}",
           "instant": true
@@ -3780,7 +3780,7 @@
           "targetBlank": false
         }
       ],
-      "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero."
+      "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
     },
     {
       "id": 13,

services/monitoring/dcgm-exporter.yaml

@@ -65,7 +65,10 @@ spec:
           resources:
             requests:
               cpu: 50m
-              memory: 64Mi
+              memory: 512Mi
+            limits:
+              cpu: 500m
+              memory: 1Gi
           volumeMounts:
             - name: pod-resources
               mountPath: /var/lib/kubelet/pod-resources

services/monitoring/grafana-dashboard-gpu.yaml

@@ -29,7 +29,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
+              "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
               "refId": "A",
               "legendFormat": "{{namespace}}",
               "instant": true
@@ -81,7 +81,7 @@ data:
               "targetBlank": false
             }
           ],
-          "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero."
+          "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
         },
         {
           "id": 2,
@@ -137,7 +137,7 @@ data:
           },
           "targets": [
             {
-              "expr": "label_replace(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")",
+              "expr": "label_replace(max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")",
               "refId": "A",
               "legendFormat": "{{Hostname}}"
             }

services/monitoring/grafana-dashboard-overview.yaml

@@ -3737,7 +3737,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
+              "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
               "refId": "A",
               "legendFormat": "{{namespace}}",
               "instant": true
@@ -3789,7 +3789,7 @@ data:
               "targetBlank": false
             }
           ],
-          "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero."
+          "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
         },
         {
           "id": 13,