monitoring(gpu): attribute Jetson activity by allocation

scripts/dashboards_render_atlas.py

@@ -301,16 +301,11 @@ def gpu_util_by_hostname():
 GPU_RESOURCE_REGEX = "nvidia(_com_|[.]com/)gpu.*"
 
 
-def gpu_node_labels():
-    return f'max by (node) (kube_node_status_allocatable{{resource=~"{GPU_RESOURCE_REGEX}"}} > bool 0)'
-
-
 def gpu_requests_by_namespace_node(scope_var):
     return (
         "sum by (namespace,node) ("
         f'kube_pod_container_resource_requests{{resource=~"{GPU_RESOURCE_REGEX}",{scope_var}}} '
         "* on(namespace,pod) group_left(node) kube_pod_info "
-        f"* on(node) group_left() ({gpu_node_labels()})"
         ")"
     )
 
@@ -332,7 +327,8 @@ def gpu_usage_by_namespace(scope_var):
 
 def jetson_gpu_usage_by_namespace(scope_var):
     requests_by_ns = gpu_requests_by_namespace_node(scope_var)
-    total_by_node = f"sum by (node) ({requests_by_ns})"
+    all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
+    total_by_node = f"sum by (node) ({all_requests})"
     return (
         "sum by (namespace) ("
         f"({requests_by_ns}) / on(node) group_left() clamp_min({total_by_node}, 1) "
@@ -341,6 +337,12 @@ def jetson_gpu_usage_by_namespace(scope_var):
     )
 
 
+def jetson_gpu_requested_nodes():
+    all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
+    requested = f"(sum by (node) ({all_requests}) > 0)"
+    return f"({requested}) and on(node) ({jetson_gpu_util_by_node()})"
+
+
 def namespace_share_expr(resource_expr):
     total = f"clamp_min(sum( {resource_expr} ), 1)"
     return f"100 * ( {resource_expr} ) / {total}"
@@ -390,14 +392,29 @@ def gpu_total_devices_expr():
 
 
 def unattributed_gpu_usage():
-    legacy_total = f"(sum({legacy_gpu_util_without_process_exporter()}) or on() vector(0))"
+    unresolved = (
+        f"({legacy_gpu_util_without_process_exporter()}) "
+        f"unless on(node) ({jetson_gpu_requested_nodes()})"
+    )
+    legacy_total = f"(sum({unresolved}) or on() vector(0))"
     return (
         f'label_replace(({legacy_total} > 0), "namespace", "unattributed", "", "")'
     )
 
 
 def gpu_utilization_raw(scope_var):
-    return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})"
+    nvidia = (
+        'label_replace('
+        f'{nvidia_process_gpu_usage_by_namespace(scope_var)}, '
+        '"gpu_source", "nvidia", "", "")'
+    )
+    jetson = (
+        'label_replace('
+        f'(({jetson_gpu_usage_by_namespace(scope_var)}) > 0), '
+        '"gpu_source", "jetson", "", "")'
+    )
+    attributed = f"sum by (namespace) (({nvidia}) or ({jetson}))"
+    return f"({attributed}) or ({unattributed_gpu_usage()})"
 
 
 def gpu_pool_used_expr(scope_var):
@@ -1916,7 +1933,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
     "Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.",
     "Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.",
     "Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.",
-    "Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.",
+    "Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.",
     "Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.",
     "Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.",
     "Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.",

scripts/tests/test_dashboards_render_atlas.py

@@ -163,7 +163,10 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     assert "$__range" not in gpu_expr
     assert "sum by (namespace)" in gpu_expr
     assert 'namespace", "shared"' not in gpu_expr
-    assert "kube_node_labels" not in gpu_expr
+    assert "kube_pod_container_resource_requests" in gpu_expr
+    assert mod.GPU_RESOURCE_REGEX in gpu_expr
+    assert '"gpu_source", "nvidia"' in gpu_expr
+    assert '"gpu_source", "jetson"' in gpu_expr
     assert "100 *" in gpu_expr
     assert "100 -" not in gpu_expr
     assert 'namespace", "unattributed"' in gpu_expr
@@ -171,6 +174,7 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
     assert "Current proportional share" in panels_by_title["Namespace GPU Utilization"]["description"]
     assert "independent of the selected dashboard time range" in panels_by_title["Namespace GPU Utilization"]["description"]
+    assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
 
 
 def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():