From 4991493d3be67a922310849e5e920941d074831c Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 2 Aug 2026 03:59:32 -0300 Subject: [PATCH] ai(hermes): add operator guide and current GPU shares --- scripts/dashboards_render_atlas.py | 43 +------- scripts/tests/test_dashboards_render_atlas.py | 12 +-- scripts/tests/test_nvidia_process_exporter.py | 99 +++++++++++++++++++ services/hermes/configmap.yaml | 4 + services/hermes/deployment.yaml | 17 +++- services/hermes/kustomization.yaml | 9 ++ .../dashboard/dist/index.js | 88 +++++++++++++++++ .../dashboard/dist/style.css | 64 ++++++++++++ .../dashboard/manifest.json | 16 +++ .../skills/master-hermes-on-atlas/SKILL.md | 14 ++- .../master-hermes-on-atlas/agents/openai.yaml | 2 +- services/monitoring/dashboards/atlas-gpu.json | 4 +- .../monitoring/dashboards/atlas-overview.json | 4 +- .../monitoring/grafana-dashboard-gpu.yaml | 4 +- .../grafana-dashboard-overview.yaml | 4 +- .../monitoring/nvidia-process-exporter.yaml | 1 + .../scripts/nvidia_process_exporter.py | 34 ++++++- 17 files changed, 352 insertions(+), 67 deletions(-) create mode 100644 scripts/tests/test_nvidia_process_exporter.py create mode 100644 services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js create mode 100644 services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css create mode 100644 services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index edbecc8f3..68c2d6317 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -363,21 +363,6 @@ def nvidia_process_gpu_usage_by_namespace(scope_var): return f"(({usage}) > 0)" -def nvidia_process_gpu_usage_by_namespace_range(scope_var, window="$__range"): - namespace_samples = ( - "sum_over_time(nvidia_namespace_gpu_sm_util_percent" - f"{{{namespace_gpu_selector(scope_var)}}}[{window}])" - ) - device_samples = ( - f"count_over_time(nvidia_gpu_device_utilization_percent[{window}])" - ) - averaged = ( - f"({namespace_samples}) / on(node,gpu,uuid,model) group_left() " - f"clamp_min({device_samples}, 1)" - ) - return f"((sum by (namespace) ({averaged})) > 0)" - - def nvidia_gpu_device_utilization(): return "max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))" @@ -411,41 +396,17 @@ def unattributed_gpu_usage(): ) -def legacy_gpu_util_range_without_process_exporter(window="$__range"): - dcgm_range = dcgm_gpu_util_by_node().replace( - "DCGM_FI_DEV_GPU_UTIL", f"avg_over_time(DCGM_FI_DEV_GPU_UTIL[{window}])" - ) - jetson_range = ( - f'max by (node) (avg_over_time(jetson_gr3d_freq_percent{{node!=""}}[{window}]))' - ) - process_exporter_nodes = ( - f"max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[{window}]))" - ) - return f"(({dcgm_range}) or ({jetson_range})) unless on(node) ({process_exporter_nodes})" - - -def unattributed_gpu_usage_range(window="$__range"): - total = f"(sum({legacy_gpu_util_range_without_process_exporter(window)}) or on() vector(0))" - return f'label_replace(({total} > 0), "namespace", "unattributed", "", "")' - - def gpu_utilization_raw(scope_var): return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})" -def gpu_utilization_range_raw(scope_var, window="$__range"): - attributed = nvidia_process_gpu_usage_by_namespace_range(scope_var, window) - unattributed = unattributed_gpu_usage_range(window) - return f"({attributed}) or ({unattributed})" - - def gpu_pool_used_expr(scope_var): raw_total = f"(sum({gpu_utilization_raw(scope_var)}) or on() vector(0))" return f"100 * {raw_total} / clamp_min({gpu_capacity_percent()}, 1)" def namespace_gpu_share_expr(scope_var): - activity = gpu_utilization_range_raw(scope_var) + activity = gpu_utilization_raw(scope_var) total = f"(sum({activity}) or on() vector(0))" share = f"100 * ({activity}) / clamp_min({total}, 1)" idle = ( @@ -1955,7 +1916,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = { "Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.", "Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.", "Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.", - "Namespace GPU Utilization": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero.", + "Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.", "Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.", "Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.", "Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.", diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index 51273ff32..99df8288c 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -157,11 +157,10 @@ def test_overview_uses_readable_quality_power_and_gitops_panels(): gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"] assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr assert "nvidia_gpu_device_utilization_percent" in gpu_expr - assert "sum_over_time" in gpu_expr - assert "count_over_time" in gpu_expr - assert "avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range])" in gpu_expr - assert "avg_over_time(jetson_gr3d_freq_percent" in gpu_expr - assert "$__range" in gpu_expr + assert "sum_over_time" not in gpu_expr + assert "count_over_time" not in gpu_expr + assert "avg_over_time" not in gpu_expr + assert "$__range" not in gpu_expr assert "sum by (namespace)" in gpu_expr assert 'namespace", "shared"' not in gpu_expr assert "kube_node_labels" not in gpu_expr @@ -170,7 +169,8 @@ def test_overview_uses_readable_quality_power_and_gitops_panels(): assert 'namespace", "unattributed"' in gpu_expr assert 'namespace", "idle"' in gpu_expr assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True - assert "Time-weighted share" in panels_by_title["Namespace GPU Utilization"]["description"] + assert "Current proportional share" in panels_by_title["Namespace GPU Utilization"]["description"] + assert "independent of the selected dashboard time range" in panels_by_title["Namespace GPU Utilization"]["description"] def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families(): diff --git a/scripts/tests/test_nvidia_process_exporter.py b/scripts/tests/test_nvidia_process_exporter.py new file mode 100644 index 000000000..3a01ee91a --- /dev/null +++ b/scripts/tests/test_nvidia_process_exporter.py @@ -0,0 +1,99 @@ +import importlib.util +from pathlib import Path +import sys +import types + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +EXPORTER_PATH = ROOT / "services" / "monitoring" / "scripts" / "nvidia_process_exporter.py" + + +def load_exporter(monkeypatch): + """Load the exporter without requiring an NVIDIA driver on the test host.""" + + pynvml = types.ModuleType("pynvml") + + class NVMLError(Exception): + pass + + class NVMLErrorNotFound(NVMLError): + pass + + class NVMLErrorNotSupported(NVMLError): + pass + + pynvml.NVMLError = NVMLError + pynvml.NVMLError_NotFound = NVMLErrorNotFound + pynvml.NVMLError_NotSupported = NVMLErrorNotSupported + for name in ( + "nvmlDeviceGetComputeRunningProcesses_v3", + "nvmlDeviceGetCount", + "nvmlDeviceGetGraphicsRunningProcesses_v3", + "nvmlDeviceGetHandleByIndex", + "nvmlDeviceGetName", + "nvmlDeviceGetProcessUtilization", + "nvmlDeviceGetUUID", + "nvmlDeviceGetUtilizationRates", + "nvmlInit", + ): + setattr(pynvml, name, lambda *args, **kwargs: None) + + monkeypatch.setitem(sys.modules, "pynvml", pynvml) + spec = importlib.util.spec_from_file_location("nvidia_process_exporter_test", EXPORTER_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_process_sample_window_uses_nvml_microseconds(monkeypatch): + exporter = load_exporter(monkeypatch) + observed = [] + monkeypatch.setattr(exporter.time, "time", lambda: 1_700_000_000.0) + monkeypatch.setattr( + exporter, + "nvmlDeviceGetProcessUtilization", + lambda handle, since: observed.append(since) or [], + ) + + samples, supported = exporter.process_utilization_samples(object()) + + assert samples == {} + assert supported == 1 + assert observed == [1_700_000_000_000_000 - 30_000_000] + + +def test_namespace_attribution_scales_to_current_device_total(monkeypatch): + exporter = load_exporter(monkeypatch) + + result = exporter.reconcile_namespace_utilization( + {"game-stream": 20, "hermes": 10}, + device_util=3, + ) + + assert sum(result.values()) == pytest.approx(3) + assert result["game-stream"] == pytest.approx(2) + assert result["hermes"] == pytest.approx(1) + + +def test_namespace_attribution_assigns_unexplained_compute_to_host(monkeypatch): + exporter = load_exporter(monkeypatch) + + result = exporter.reconcile_namespace_utilization( + {"hermes": 1}, + device_util=3, + ) + + assert result == {"hermes": 1, "host": 2} + + +def test_zero_device_utilization_clears_stale_process_samples(monkeypatch): + exporter = load_exporter(monkeypatch) + + result = exporter.reconcile_namespace_utilization( + {"hermes": 40, "game-stream": 5}, + device_util=0, + ) + + assert result == {"hermes": 0, "game-stream": 0} diff --git a/services/hermes/configmap.yaml b/services/hermes/configmap.yaml index af663e113..c8818279b 100644 --- a/services/hermes/configmap.yaml +++ b/services/hermes/configmap.yaml @@ -28,6 +28,10 @@ data: external_dirs: - /opt/data/workspace/skills + plugins: + enabled: + - atlas-operator-guide + terminal: backend: local cwd: /opt/data/workspace diff --git a/services/hermes/deployment.yaml b/services/hermes/deployment.yaml index afc00aa65..51dd65068 100644 --- a/services/hermes/deployment.yaml +++ b/services/hermes/deployment.yaml @@ -23,7 +23,7 @@ spec: ai.bstein.dev/model: gpt-oss:20b with openai-codex fallback ai.bstein.dev/role: testing-triage ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred) - ai.bstein.dev/config-rev: "20260802-mastery-lab" + ai.bstein.dev/config-rev: "20260802-operator-guide" spec: serviceAccountName: hermes-triage automountServiceAccountToken: true @@ -197,6 +197,9 @@ spec: - name: mastery-skill mountPath: /opt/data/workspace/skills/master-hermes-on-atlas readOnly: true + - name: operator-guide-plugin + mountPath: /opt/data/plugins/atlas-operator-guide + readOnly: true readinessProbe: httpGet: path: /api/status @@ -251,3 +254,15 @@ spec: path: references/incident-drills.md - key: mastery-rubric.md path: references/mastery-rubric.md + - key: two-hour-proof-sprint.md + path: references/two-hour-proof-sprint.md + - name: operator-guide-plugin + configMap: + name: hermes-operator-guide-plugin + items: + - key: manifest.json + path: dashboard/manifest.json + - key: index.js + path: dashboard/dist/index.js + - key: style.css + path: dashboard/dist/style.css diff --git a/services/hermes/kustomization.yaml b/services/hermes/kustomization.yaml index fe8660d8b..0966dccc8 100644 --- a/services/hermes/kustomization.yaml +++ b/services/hermes/kustomization.yaml @@ -38,5 +38,14 @@ configMapGenerator: - curriculum.md=skills/master-hermes-on-atlas/references/curriculum.md - incident-drills.md=skills/master-hermes-on-atlas/references/incident-drills.md - mastery-rubric.md=skills/master-hermes-on-atlas/references/mastery-rubric.md + - two-hour-proof-sprint.md=skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md + options: + disableNameSuffixHash: true + - name: hermes-operator-guide-plugin + namespace: hermes + files: + - manifest.json=plugins/atlas-operator-guide/dashboard/manifest.json + - index.js=plugins/atlas-operator-guide/dashboard/dist/index.js + - style.css=plugins/atlas-operator-guide/dashboard/dist/style.css options: disableNameSuffixHash: true diff --git a/services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js b/services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js new file mode 100644 index 000000000..5a2ff8cee --- /dev/null +++ b/services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js @@ -0,0 +1,88 @@ +(function () { + "use strict"; + + var pluginName = "atlas-operator-guide"; + var storageKey = "atlas.operator-guide.hidden.v1"; + var prompt = [ + "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint.", + "I will perform every setup and evidence step myself.", + "Start with step 1 only, require live proof, and stop at every mutation or approval boundary.", + "First tell me to select openai-codex/gpt-5.4 if this session is using the local model." + ].join(" "); + + function OperatorGuide() { + var sdk = window.__HERMES_PLUGIN_SDK__; + var React = sdk.React; + var state = sdk.hooks.useState(function () { + return window.localStorage.getItem(storageKey) === "1"; + }); + var hidden = state[0]; + var setHidden = state[1]; + var copiedState = sdk.hooks.useState(false); + var copied = copiedState[0]; + var setCopied = copiedState[1]; + + if (hidden) { + return React.createElement( + "button", + { + className: "atlas-guide-restore", + onClick: function () { + window.localStorage.removeItem(storageKey); + setHidden(false); + }, + type: "button" + }, + "Show Atlas operator walkthrough" + ); + } + + function copyPrompt() { + navigator.clipboard.writeText(prompt).then(function () { + setCopied(true); + window.setTimeout(function () { setCopied(false); }, 2500); + }); + } + + return React.createElement( + "section", + { className: "atlas-guide-card", role: "region", "aria-label": "Atlas operator walkthrough" }, + React.createElement( + "div", + { className: "atlas-guide-heading" }, + React.createElement("div", null, + React.createElement("strong", null, "Start here: two-hour Hermes proof sprint"), + React.createElement("p", null, "You do the setup and evidence work. Hermes coaches, checks, and stops at approval boundaries.") + ), + React.createElement( + "button", + { + className: "atlas-guide-hide", + onClick: function () { + window.localStorage.setItem(storageKey, "1"); + setHidden(true); + }, + type: "button" + }, + "Hide" + ) + ), + React.createElement( + "ol", + { className: "atlas-guide-steps" }, + React.createElement("li", null, "Open Models and select openai-codex/gpt-5.4 for the coaching session."), + React.createElement("li", null, "Copy the sprint prompt, paste it into Chat, and press Enter."), + React.createElement("li", null, "Complete one live triage and build one writable skill yourself.") + ), + React.createElement( + "div", + { className: "atlas-guide-actions" }, + React.createElement("a", { className: "atlas-guide-button", href: "/models" }, "Open Models"), + React.createElement("button", { className: "atlas-guide-button", onClick: copyPrompt, type: "button" }, copied ? "Prompt copied" : "Copy sprint prompt"), + React.createElement("a", { className: "atlas-guide-link", href: "/skills" }, "Inspect skills") + ) + ); + } + + window.__HERMES_PLUGINS__.registerSlot(pluginName, "chat:top", OperatorGuide); +}()); diff --git a/services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css b/services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css new file mode 100644 index 000000000..ffecc67c8 --- /dev/null +++ b/services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css @@ -0,0 +1,64 @@ +.atlas-guide-card { + border: 1px solid rgba(247, 201, 72, 0.7); + background: rgba(247, 201, 72, 0.08); + color: inherit; + padding: 0.75rem 1rem; + font-family: "Mondwest", monospace; +} + +.atlas-guide-heading { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 1rem; +} + +.atlas-guide-heading strong { + color: #f7c948; + font-family: "Rules Expanded", sans-serif; + letter-spacing: 0.04em; + text-transform: uppercase; +} + +.atlas-guide-heading p { + margin: 0.25rem 0 0; + opacity: 0.8; +} + +.atlas-guide-steps { + margin: 0.6rem 0; + padding-left: 1.25rem; +} + +.atlas-guide-actions { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0.5rem; +} + +.atlas-guide-button, +.atlas-guide-hide, +.atlas-guide-restore { + border: 1px solid rgba(247, 201, 72, 0.7); + background: transparent; + color: #f7c948; + cursor: pointer; + padding: 0.35rem 0.6rem; + text-decoration: none; +} + +.atlas-guide-button:hover, +.atlas-guide-hide:hover, +.atlas-guide-restore:hover { + background: rgba(247, 201, 72, 0.14); +} + +.atlas-guide-hide, +.atlas-guide-restore { + font-size: 0.75rem; +} + +.atlas-guide-link { + color: #f7c948; +} diff --git a/services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json b/services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json new file mode 100644 index 000000000..440a16d0e --- /dev/null +++ b/services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json @@ -0,0 +1,16 @@ +{ + "name": "atlas-operator-guide", + "label": "Atlas Operator Guide", + "description": "Hands-on onboarding for supervised Atlas triage and Hermes skill building.", + "icon": "Sparkles", + "version": "1.0.0", + "tab": { + "path": "/atlas-operator-guide", + "hidden": true + }, + "slots": [ + "chat:top" + ], + "entry": "dist/index.js", + "css": "dist/style.css" +} diff --git a/services/hermes/skills/master-hermes-on-atlas/SKILL.md b/services/hermes/skills/master-hermes-on-atlas/SKILL.md index 136caad32..699dfa1c5 100644 --- a/services/hermes/skills/master-hermes-on-atlas/SKILL.md +++ b/services/hermes/skills/master-hermes-on-atlas/SKILL.md @@ -22,15 +22,19 @@ single-question review. 1. Read `references/architecture.md` before teaching deployment-specific facts. 2. Read `references/curriculum.md` to select the next lab. -3. Ask whether to assess, resume, or choose a lab. If no progress record exists, +3. When Brad asks for the two-hour proof sprint, read + `references/two-hour-proof-sprint.md` and run it in order. Brad performs + every required UI action and evidence check; coach and verify without doing + the learning-critical steps for him. +4. Ask whether to assess, resume, or choose a lab. If no progress record exists, default to a five-question assessment followed by the first weak area. -4. Give one bounded task at a time. State the goal, safety boundary, exact +5. Give one bounded task at a time. State the goal, safety boundary, exact success evidence, and at most one initial hint. -5. Wait for Brad's answer or observed command output before revealing the +6. Wait for Brad's answer or observed command output before revealing the explanation. -6. Grade with `references/mastery-rubric.md`. Separate demonstrated ability +7. Grade with `references/mastery-rubric.md`. Separate demonstrated ability from verbal familiarity. -7. Offer a concise progress update. Write it to +8. Offer a concise progress update. Write it to `/opt/data/workspace/hermes-training/progress.md` only after Brad explicitly approves the file change. diff --git a/services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml b/services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml index 9e39098b1..760f152d1 100644 --- a/services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml +++ b/services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Master Hermes on Atlas" short_description: "Practice Hermes safely on the live Atlas stack" - default_prompt: "Use $master-hermes-on-atlas to assess me and begin my next hands-on Hermes lab; first stop and tell me to select openai-codex/gpt-5.4 if this session is using the local model." + default_prompt: "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint. I will perform every setup and evidence step myself. Start with step 1 only, require live proof, and stop at every mutation or approval boundary. First tell me to select openai-codex/gpt-5.4 if this session is using the local model." diff --git a/services/monitoring/dashboards/atlas-gpu.json b/services/monitoring/dashboards/atlas-gpu.json index d267bd678..c50bc8f52 100644 --- a/services/monitoring/dashboards/atlas-gpu.json +++ b/services/monitoring/dashboards/atlas-gpu.json @@ -20,7 +20,7 @@ }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -72,7 +72,7 @@ "targetBlank": false } ], - "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero." + "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero." }, { "id": 2, diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index ecac87845..ff74cca2b 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -3728,7 +3728,7 @@ }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -3780,7 +3780,7 @@ "targetBlank": false } ], - "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero." + "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero." }, { "id": 13, diff --git a/services/monitoring/grafana-dashboard-gpu.yaml b/services/monitoring/grafana-dashboard-gpu.yaml index 94cf3b1ad..fee7e5385 100644 --- a/services/monitoring/grafana-dashboard-gpu.yaml +++ b/services/monitoring/grafana-dashboard-gpu.yaml @@ -29,7 +29,7 @@ data: }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -81,7 +81,7 @@ data: "targetBlank": false } ], - "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero." + "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero." }, { "id": 2, diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index bf6ce496e..77b128670 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -3737,7 +3737,7 @@ data: }, "targets": [ { - "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", + "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))", "refId": "A", "legendFormat": "{{namespace}}", "instant": true @@ -3789,7 +3789,7 @@ data: "targetBlank": false } ], - "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero." + "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero." }, { "id": 13, diff --git a/services/monitoring/nvidia-process-exporter.yaml b/services/monitoring/nvidia-process-exporter.yaml index 408e4c42a..325a03794 100644 --- a/services/monitoring/nvidia-process-exporter.yaml +++ b/services/monitoring/nvidia-process-exporter.yaml @@ -49,6 +49,7 @@ spec: annotations: prometheus.io/scrape: "true" prometheus.io/port: "9401" + monitoring.bstein.dev/restart-rev: "20260802-current-attribution" spec: serviceAccountName: nvidia-process-exporter imagePullSecrets: diff --git a/services/monitoring/scripts/nvidia_process_exporter.py b/services/monitoring/scripts/nvidia_process_exporter.py index a43b2b45a..136a199c3 100644 --- a/services/monitoring/scripts/nvidia_process_exporter.py +++ b/services/monitoring/scripts/nvidia_process_exporter.py @@ -134,7 +134,8 @@ def running_process_memory(handle): def process_utilization_samples(handle): try: - since = int(time.time() * 1000) - SAMPLE_WINDOW_MS + # NVML process sample timestamps are microseconds since the epoch. + since = int(time.time() * 1_000_000) - (SAMPLE_WINDOW_MS * 1000) samples = nvmlDeviceGetProcessUtilization(handle, since) except NVMLError_NotFound: return {}, 1 @@ -156,6 +157,32 @@ def process_utilization_samples(handle): return by_pid, 1 +def reconcile_namespace_utilization(namespace_sm, device_util): + """Make namespace attribution add up to the device's current utilization.""" + + current = max(float(device_util), 0.0) + reconciled = { + namespace: max(float(value), 0.0) + for namespace, value in namespace_sm.items() + } + attributed = sum(reconciled.values()) + + if current == 0: + return {namespace: 0.0 for namespace in reconciled} + + # Per-process NVML samples and the device gauge are collected on slightly + # different intervals. Scale a stale/overlapping process sum down to the + # device's current total while preserving the namespace proportions. + if attributed > current and attributed > 0: + scale = current / attributed + return {namespace: value * scale for namespace, value in reconciled.items()} + + residual = current - attributed + if residual > 0.1: + reconciled["host"] = reconciled.get("host", 0.0) + residual + return reconciled + + def collect_metrics(): nvmlInit() pods = load_pods() @@ -204,10 +231,7 @@ def collect_metrics(): lines.append(metric_line("nvidia_process_gpu_sm_util_percent", labels, sm_util)) lines.append(metric_line("nvidia_process_gpu_memory_used_bytes", labels, int(proc_info["memory"]))) - attributed = sum(namespace_sm.values()) - residual = max(device_util - attributed, 0.0) - if residual > 0.1: - namespace_sm["host"] = namespace_sm.get("host", 0.0) + residual + namespace_sm = reconcile_namespace_utilization(namespace_sm, device_util) for namespace, value in sorted(namespace_sm.items()): labels = {**base, "namespace": namespace, "pod": "__namespace_total__"}