2185a90825
Evidence-led capacity/QoS audit of titan-20 (Hermes LLM fallback/ classifier), titan-21 (STT/TTS), and titan-22 (Jellyfin media-primary, restored) for t_26da4c88. titan-20/21 are CPU-committed with no safe headroom (titan-20 at 227 MiB free memory at its 24h worst point); titan-22 has real idle CPU/RAM but its shared-GPU time-slicing has no VRAM/engine isolation, so no workload is relocated. Adds alerting for the sharpest gaps found (titan-20 memory exhaustion, titan-22 CPU/RAM/ GPU-VRAM pressure, Jellyfin CPU throttling, titan-21 CPU pressure) and two Atlas GPU dashboard panels (VRAM, NVENC/NVDEC utilization) so a future opportunistic-workload PR or a live transcode incident is visible without a promql session. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
397 lines
18 KiB
JSON
397 lines
18 KiB
JSON
{
|
|
"uid": "atlas-gpu",
|
|
"title": "Atlas GPU",
|
|
"folderUid": "atlas-internal",
|
|
"editable": true,
|
|
"panels": [
|
|
{
|
|
"id": 1,
|
|
"type": "piechart",
|
|
"title": "Namespace GPU Utilization",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "atlas-vm"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "(100 * ((sum by (namespace) ((label_replace(((sum by (namespace) (avg_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range]))) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() (100 * sum by (node) (increase(jetson_gr3d_active_seconds_total{node!=\"\"}[$__range])) / $__range_s))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (100 * sum by (node) (increase(jetson_gr3d_active_seconds_total{node!=\"\"}[$__range])) / $__range_s))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((sum by (namespace) ((label_replace(((sum by (namespace) (avg_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range]))) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() (100 * sum by (node) (increase(jetson_gr3d_active_seconds_total{node!=\"\"}[$__range])) / $__range_s))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (100 * sum by (node) (increase(jetson_gr3d_active_seconds_total{node!=\"\"}[$__range])) / $__range_s))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((sum by (namespace) ((label_replace(((sum by (namespace) (avg_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range]))) > 0), \"gpu_source\", \"nvidia\", \"\", \"\")) or (label_replace(((sum by (namespace) ((sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",$namespace_scope_gpu} * on(namespace,pod) group_left(node) kube_pod_info )) / on(node) group_left() clamp_min(sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )), 1) * on(node) group_left() (100 * sum by (node) (increase(jetson_gr3d_active_seconds_total{node!=\"\"}[$__range])) / $__range_s))) > 0), \"gpu_source\", \"jetson\", \"\", \"\")))) or (label_replace(((sum((((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) unless on(node) (((sum by (node) (sum by (namespace,node) (kube_pod_container_resource_requests{resource=~\"nvidia(_com_|[.]com/)gpu.*\",namespace=~\".*\"} * on(namespace,pod) group_left(node) kube_pod_info )) > 0)) and on(node) (100 * sum by (node) (increase(jetson_gr3d_active_seconds_total{node!=\"\"}[$__range])) / $__range_s))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
|
|
"refId": "A",
|
|
"legendFormat": "{{namespace}}",
|
|
"instant": true
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "list",
|
|
"placement": "right"
|
|
},
|
|
"pieType": "pie",
|
|
"displayLabels": [],
|
|
"tooltip": {
|
|
"mode": "single"
|
|
},
|
|
"colorScheme": "interpolateSpectral",
|
|
"colorBy": "value",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
}
|
|
},
|
|
"links": [
|
|
{
|
|
"title": "Workload namespaces only",
|
|
"url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%21~%22%5E%28kube-.%2A%7C.%2A-system%7Ctraefik%7Cmonitoring%7Clogging%7Ccert-manager%7Cmaintenance%7Cpostgres%29%24%22&var-namespace_scope_ram=${namespace_scope_ram}",
|
|
"targetBlank": false
|
|
},
|
|
{
|
|
"title": "All namespaces",
|
|
"url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%3D~%22.%2A%22&var-namespace_scope_ram=${namespace_scope_ram}",
|
|
"targetBlank": false
|
|
},
|
|
{
|
|
"title": "Infrastructure namespaces only",
|
|
"url": "?var-namespace_scope_cpu=${namespace_scope_cpu}&var-namespace_scope_gpu=namespace%3D~%22%5E%28kube-.%2A%7C.%2A-system%7Ctraefik%7Cmonitoring%7Clogging%7Ccert-manager%7Cmaintenance%7Cpostgres%29%24%22&var-namespace_scope_ram=${namespace_scope_ram}",
|
|
"targetBlank": false
|
|
}
|
|
],
|
|
"description": "Proportional share of GPU compute activity observed over the selected time range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 activity is continuously sampled and assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of observed compute; idle appears only when the selected range contains no activity."
|
|
},
|
|
{
|
|
"id": 2,
|
|
"type": "timeseries",
|
|
"title": "GPU Process Util by Namespace",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "atlas-vm"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "((sum by (namespace) (avg_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range]))) > 0)",
|
|
"refId": "A",
|
|
"legendFormat": "{{namespace}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
},
|
|
"description": "NVML process-level SM utilization by namespace. Host covers GPU work outside Kubernetes pods."
|
|
},
|
|
{
|
|
"id": 3,
|
|
"type": "timeseries",
|
|
"title": "GPU Util by Node",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "atlas-vm"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "label_replace(max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")",
|
|
"refId": "A",
|
|
"legendFormat": "{{Hostname}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent"
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"id": 4,
|
|
"type": "table",
|
|
"title": "GPU Processes by Pod",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "atlas-vm"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 8
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "topk(10, sum by (namespace,pod,node,process) (nvidia_process_gpu_sm_util_percent{pod!=\"host\"}) > 0)",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"custom": {
|
|
"filterable": true
|
|
}
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"showHeader": true,
|
|
"columnFilters": false
|
|
},
|
|
"transformations": [
|
|
{
|
|
"id": "labelsToFields",
|
|
"options": {}
|
|
}
|
|
],
|
|
"description": "NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization."
|
|
},
|
|
{
|
|
"id": 5,
|
|
"type": "timeseries",
|
|
"title": "GPU Memory (VRAM) Used by Node",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "atlas-vm"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 16
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "avg by (node) (label_replace(label_replace((DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE) * 100), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})",
|
|
"refId": "A",
|
|
"legendFormat": "{{node}}"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
},
|
|
"description": "DCGM frame-buffer used as a percent of total VRAM, per DCGM-instrumented node (titan-22, titan-24). Time-sliced GPU sharing gives no VRAM isolation between tenants, so this is the most direct signal of contention risk for an active Jellyfin transcode on titan-22."
|
|
},
|
|
{
|
|
"id": 6,
|
|
"type": "timeseries",
|
|
"title": "GPU Encoder/Decoder Utilization by Node",
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "atlas-vm"
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 16
|
|
},
|
|
"targets": [
|
|
{
|
|
"expr": "avg by (node) (label_replace(label_replace((DCGM_FI_DEV_ENC_UTIL), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})",
|
|
"refId": "A",
|
|
"legendFormat": "{{node}} encoder"
|
|
},
|
|
{
|
|
"expr": "avg by (node) (label_replace(label_replace((DCGM_FI_DEV_DEC_UTIL), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})",
|
|
"refId": "B",
|
|
"legendFormat": "{{node}} decoder"
|
|
}
|
|
],
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"unit": "percent",
|
|
"max": 100
|
|
},
|
|
"overrides": []
|
|
},
|
|
"options": {
|
|
"legend": {
|
|
"displayMode": "table",
|
|
"placement": "right"
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi"
|
|
}
|
|
},
|
|
"description": "DCGM NVENC/NVDEC engine utilization, per DCGM-instrumented node. This is the engine Jellyfin hardware transcode/playback actually contends on; the shared-GPU device plugin (nvidia.com/gpu.shared) does not reserve or prioritize it."
|
|
}
|
|
],
|
|
"time": {
|
|
"from": "now-12h",
|
|
"to": "now"
|
|
},
|
|
"annotations": {
|
|
"list": []
|
|
},
|
|
"schemaVersion": 39,
|
|
"style": "dark",
|
|
"tags": [
|
|
"atlas",
|
|
"gpu"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"name": "namespace_scope_cpu",
|
|
"label": "CPU namespace filter",
|
|
"type": "custom",
|
|
"query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"current": {
|
|
"text": "workload namespaces only",
|
|
"value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"selected": true
|
|
},
|
|
"options": [
|
|
{
|
|
"text": "workload namespaces only",
|
|
"value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"selected": true
|
|
},
|
|
{
|
|
"text": "all namespaces",
|
|
"value": "namespace=~\".*\"",
|
|
"selected": false
|
|
},
|
|
{
|
|
"text": "infrastructure namespaces only",
|
|
"value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"selected": false
|
|
}
|
|
],
|
|
"hide": 2,
|
|
"multi": false,
|
|
"includeAll": false,
|
|
"refresh": 1,
|
|
"sort": 0,
|
|
"skipUrlSync": false
|
|
},
|
|
{
|
|
"name": "namespace_scope_gpu",
|
|
"label": "GPU namespace filter",
|
|
"type": "custom",
|
|
"query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"current": {
|
|
"text": "workload namespaces only",
|
|
"value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"selected": true
|
|
},
|
|
"options": [
|
|
{
|
|
"text": "workload namespaces only",
|
|
"value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"selected": true
|
|
},
|
|
{
|
|
"text": "all namespaces",
|
|
"value": "namespace=~\".*\"",
|
|
"selected": false
|
|
},
|
|
{
|
|
"text": "infrastructure namespaces only",
|
|
"value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"selected": false
|
|
}
|
|
],
|
|
"hide": 2,
|
|
"multi": false,
|
|
"includeAll": false,
|
|
"refresh": 1,
|
|
"sort": 0,
|
|
"skipUrlSync": false
|
|
},
|
|
{
|
|
"name": "namespace_scope_ram",
|
|
"label": "RAM namespace filter",
|
|
"type": "custom",
|
|
"query": "workload namespaces only : namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\",all namespaces : namespace=~\".*\",infrastructure namespaces only : namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"current": {
|
|
"text": "workload namespaces only",
|
|
"value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"selected": true
|
|
},
|
|
"options": [
|
|
{
|
|
"text": "workload namespaces only",
|
|
"value": "namespace!~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"selected": true
|
|
},
|
|
{
|
|
"text": "all namespaces",
|
|
"value": "namespace=~\".*\"",
|
|
"selected": false
|
|
},
|
|
{
|
|
"text": "infrastructure namespaces only",
|
|
"value": "namespace=~\"^(kube-.*|.*-system|traefik|monitoring|logging|cert-manager|maintenance|postgres)$\"",
|
|
"selected": false
|
|
}
|
|
],
|
|
"hide": 2,
|
|
"multi": false,
|
|
"includeAll": false,
|
|
"refresh": 1,
|
|
"sort": 0,
|
|
"skipUrlSync": false
|
|
}
|
|
]
|
|
}
|
|
}
|