ai(hermes): add operator guide and current GPU shares

scripts/dashboards_render_atlas.py

@@ -363,21 +363,6 @@ def nvidia_process_gpu_usage_by_namespace(scope_var):
     return f"(({usage}) > 0)"
 
 
-def nvidia_process_gpu_usage_by_namespace_range(scope_var, window="$__range"):
-    namespace_samples = (
-        "sum_over_time(nvidia_namespace_gpu_sm_util_percent"
-        f"{{{namespace_gpu_selector(scope_var)}}}[{window}])"
-    )
-    device_samples = (
-        f"count_over_time(nvidia_gpu_device_utilization_percent[{window}])"
-    )
-    averaged = (
-        f"({namespace_samples}) / on(node,gpu,uuid,model) group_left() "
-        f"clamp_min({device_samples}, 1)"
-    )
-    return f"((sum by (namespace) ({averaged})) > 0)"
-
-
 def nvidia_gpu_device_utilization():
     return "max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))"
 
@@ -411,41 +396,17 @@ def unattributed_gpu_usage():
     )
 
 
-def legacy_gpu_util_range_without_process_exporter(window="$__range"):
-    dcgm_range = dcgm_gpu_util_by_node().replace(
-        "DCGM_FI_DEV_GPU_UTIL", f"avg_over_time(DCGM_FI_DEV_GPU_UTIL[{window}])"
-    )
-    jetson_range = (
-        f'max by (node) (avg_over_time(jetson_gr3d_freq_percent{{node!=""}}[{window}]))'
-    )
-    process_exporter_nodes = (
-        f"max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[{window}]))"
-    )
-    return f"(({dcgm_range}) or ({jetson_range})) unless on(node) ({process_exporter_nodes})"
-
-
-def unattributed_gpu_usage_range(window="$__range"):
-    total = f"(sum({legacy_gpu_util_range_without_process_exporter(window)}) or on() vector(0))"
-    return f'label_replace(({total} > 0), "namespace", "unattributed", "", "")'
-
-
 def gpu_utilization_raw(scope_var):
     return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})"
 
 
-def gpu_utilization_range_raw(scope_var, window="$__range"):
-    attributed = nvidia_process_gpu_usage_by_namespace_range(scope_var, window)
-    unattributed = unattributed_gpu_usage_range(window)
-    return f"({attributed}) or ({unattributed})"
-
-
 def gpu_pool_used_expr(scope_var):
     raw_total = f"(sum({gpu_utilization_raw(scope_var)}) or on() vector(0))"
     return f"100 * {raw_total} / clamp_min({gpu_capacity_percent()}, 1)"
 
 
 def namespace_gpu_share_expr(scope_var):
-    activity = gpu_utilization_range_raw(scope_var)
+    activity = gpu_utilization_raw(scope_var)
     total = f"(sum({activity}) or on() vector(0))"
     share = f"100 * ({activity}) / clamp_min({total}, 1)"
     idle = (
@@ -1955,7 +1916,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
     "Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.",
     "Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.",
     "Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.",
-    "Namespace GPU Utilization": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero.",
+    "Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.",
     "Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.",
     "Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.",
     "Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.",

scripts/tests/test_dashboards_render_atlas.py

@@ -157,11 +157,10 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
     assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
     assert "nvidia_gpu_device_utilization_percent" in gpu_expr
-    assert "sum_over_time" in gpu_expr
+    assert "sum_over_time" not in gpu_expr
-    assert "count_over_time" in gpu_expr
+    assert "count_over_time" not in gpu_expr
-    assert "avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range])" in gpu_expr
+    assert "avg_over_time" not in gpu_expr
-    assert "avg_over_time(jetson_gr3d_freq_percent" in gpu_expr
+    assert "$__range" not in gpu_expr
-    assert "$__range" in gpu_expr
     assert "sum by (namespace)" in gpu_expr
     assert 'namespace", "shared"' not in gpu_expr
     assert "kube_node_labels" not in gpu_expr
@@ -170,7 +169,8 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     assert 'namespace", "unattributed"' in gpu_expr
     assert 'namespace", "idle"' in gpu_expr
     assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
-    assert "Time-weighted share" in panels_by_title["Namespace GPU Utilization"]["description"]
+    assert "Current proportional share" in panels_by_title["Namespace GPU Utilization"]["description"]
+    assert "independent of the selected dashboard time range" in panels_by_title["Namespace GPU Utilization"]["description"]
 
 
 def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():

scripts/tests/test_nvidia_process_exporter.py

@@ -0,0 +1,99 @@
+import importlib.util
+from pathlib import Path
+import sys
+import types
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[2]
+EXPORTER_PATH = ROOT / "services" / "monitoring" / "scripts" / "nvidia_process_exporter.py"
+
+
+def load_exporter(monkeypatch):
+    """Load the exporter without requiring an NVIDIA driver on the test host."""
+
+    pynvml = types.ModuleType("pynvml")
+
+    class NVMLError(Exception):
+        pass
+
+    class NVMLErrorNotFound(NVMLError):
+        pass
+
+    class NVMLErrorNotSupported(NVMLError):
+        pass
+
+    pynvml.NVMLError = NVMLError
+    pynvml.NVMLError_NotFound = NVMLErrorNotFound
+    pynvml.NVMLError_NotSupported = NVMLErrorNotSupported
+    for name in (
+        "nvmlDeviceGetComputeRunningProcesses_v3",
+        "nvmlDeviceGetCount",
+        "nvmlDeviceGetGraphicsRunningProcesses_v3",
+        "nvmlDeviceGetHandleByIndex",
+        "nvmlDeviceGetName",
+        "nvmlDeviceGetProcessUtilization",
+        "nvmlDeviceGetUUID",
+        "nvmlDeviceGetUtilizationRates",
+        "nvmlInit",
+    ):
+        setattr(pynvml, name, lambda *args, **kwargs: None)
+
+    monkeypatch.setitem(sys.modules, "pynvml", pynvml)
+    spec = importlib.util.spec_from_file_location("nvidia_process_exporter_test", EXPORTER_PATH)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_process_sample_window_uses_nvml_microseconds(monkeypatch):
+    exporter = load_exporter(monkeypatch)
+    observed = []
+    monkeypatch.setattr(exporter.time, "time", lambda: 1_700_000_000.0)
+    monkeypatch.setattr(
+        exporter,
+        "nvmlDeviceGetProcessUtilization",
+        lambda handle, since: observed.append(since) or [],
+    )
+
+    samples, supported = exporter.process_utilization_samples(object())
+
+    assert samples == {}
+    assert supported == 1
+    assert observed == [1_700_000_000_000_000 - 30_000_000]
+
+
+def test_namespace_attribution_scales_to_current_device_total(monkeypatch):
+    exporter = load_exporter(monkeypatch)
+
+    result = exporter.reconcile_namespace_utilization(
+        {"game-stream": 20, "hermes": 10},
+        device_util=3,
+    )
+
+    assert sum(result.values()) == pytest.approx(3)
+    assert result["game-stream"] == pytest.approx(2)
+    assert result["hermes"] == pytest.approx(1)
+
+
+def test_namespace_attribution_assigns_unexplained_compute_to_host(monkeypatch):
+    exporter = load_exporter(monkeypatch)
+
+    result = exporter.reconcile_namespace_utilization(
+        {"hermes": 1},
+        device_util=3,
+    )
+
+    assert result == {"hermes": 1, "host": 2}
+
+
+def test_zero_device_utilization_clears_stale_process_samples(monkeypatch):
+    exporter = load_exporter(monkeypatch)
+
+    result = exporter.reconcile_namespace_utilization(
+        {"hermes": 40, "game-stream": 5},
+        device_util=0,
+    )
+
+    assert result == {"hermes": 0, "game-stream": 0}

services/hermes/configmap.yaml

@@ -28,6 +28,10 @@ data:
       external_dirs:
         - /opt/data/workspace/skills
 
+    plugins:
+      enabled:
+        - atlas-operator-guide
+
     terminal:
       backend: local
       cwd: /opt/data/workspace

services/hermes/deployment.yaml

@@ -23,7 +23,7 @@ spec:
         ai.bstein.dev/model: gpt-oss:20b with openai-codex fallback
         ai.bstein.dev/role: testing-triage
         ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred)
-        ai.bstein.dev/config-rev: "20260802-mastery-lab"
+        ai.bstein.dev/config-rev: "20260802-operator-guide"
     spec:
       serviceAccountName: hermes-triage
       automountServiceAccountToken: true
@@ -197,6 +197,9 @@ spec:
             - name: mastery-skill
               mountPath: /opt/data/workspace/skills/master-hermes-on-atlas
               readOnly: true
+            - name: operator-guide-plugin
+              mountPath: /opt/data/plugins/atlas-operator-guide
+              readOnly: true
           readinessProbe:
             httpGet:
               path: /api/status
@@ -251,3 +254,15 @@ spec:
                 path: references/incident-drills.md
               - key: mastery-rubric.md
                 path: references/mastery-rubric.md
+              - key: two-hour-proof-sprint.md
+                path: references/two-hour-proof-sprint.md
+        - name: operator-guide-plugin
+          configMap:
+            name: hermes-operator-guide-plugin
+            items:
+              - key: manifest.json
+                path: dashboard/manifest.json
+              - key: index.js
+                path: dashboard/dist/index.js
+              - key: style.css
+                path: dashboard/dist/style.css

services/hermes/kustomization.yaml

@@ -38,5 +38,14 @@ configMapGenerator:
       - curriculum.md=skills/master-hermes-on-atlas/references/curriculum.md
       - incident-drills.md=skills/master-hermes-on-atlas/references/incident-drills.md
       - mastery-rubric.md=skills/master-hermes-on-atlas/references/mastery-rubric.md
+      - two-hour-proof-sprint.md=skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md
+    options:
+      disableNameSuffixHash: true
+  - name: hermes-operator-guide-plugin
+    namespace: hermes
+    files:
+      - manifest.json=plugins/atlas-operator-guide/dashboard/manifest.json
+      - index.js=plugins/atlas-operator-guide/dashboard/dist/index.js
+      - style.css=plugins/atlas-operator-guide/dashboard/dist/style.css
     options:
       disableNameSuffixHash: true

services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js

@@ -0,0 +1,88 @@
+(function () {
+  "use strict";
+
+  var pluginName = "atlas-operator-guide";
+  var storageKey = "atlas.operator-guide.hidden.v1";
+  var prompt = [
+    "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint.",
+    "I will perform every setup and evidence step myself.",
+    "Start with step 1 only, require live proof, and stop at every mutation or approval boundary.",
+    "First tell me to select openai-codex/gpt-5.4 if this session is using the local model."
+  ].join(" ");
+
+  function OperatorGuide() {
+    var sdk = window.__HERMES_PLUGIN_SDK__;
+    var React = sdk.React;
+    var state = sdk.hooks.useState(function () {
+      return window.localStorage.getItem(storageKey) === "1";
+    });
+    var hidden = state[0];
+    var setHidden = state[1];
+    var copiedState = sdk.hooks.useState(false);
+    var copied = copiedState[0];
+    var setCopied = copiedState[1];
+
+    if (hidden) {
+      return React.createElement(
+        "button",
+        {
+          className: "atlas-guide-restore",
+          onClick: function () {
+            window.localStorage.removeItem(storageKey);
+            setHidden(false);
+          },
+          type: "button"
+        },
+        "Show Atlas operator walkthrough"
+      );
+    }
+
+    function copyPrompt() {
+      navigator.clipboard.writeText(prompt).then(function () {
+        setCopied(true);
+        window.setTimeout(function () { setCopied(false); }, 2500);
+      });
+    }
+
+    return React.createElement(
+      "section",
+      { className: "atlas-guide-card", role: "region", "aria-label": "Atlas operator walkthrough" },
+      React.createElement(
+        "div",
+        { className: "atlas-guide-heading" },
+        React.createElement("div", null,
+          React.createElement("strong", null, "Start here: two-hour Hermes proof sprint"),
+          React.createElement("p", null, "You do the setup and evidence work. Hermes coaches, checks, and stops at approval boundaries.")
+        ),
+        React.createElement(
+          "button",
+          {
+            className: "atlas-guide-hide",
+            onClick: function () {
+              window.localStorage.setItem(storageKey, "1");
+              setHidden(true);
+            },
+            type: "button"
+          },
+          "Hide"
+        )
+      ),
+      React.createElement(
+        "ol",
+        { className: "atlas-guide-steps" },
+        React.createElement("li", null, "Open Models and select openai-codex/gpt-5.4 for the coaching session."),
+        React.createElement("li", null, "Copy the sprint prompt, paste it into Chat, and press Enter."),
+        React.createElement("li", null, "Complete one live triage and build one writable skill yourself.")
+      ),
+      React.createElement(
+        "div",
+        { className: "atlas-guide-actions" },
+        React.createElement("a", { className: "atlas-guide-button", href: "/models" }, "Open Models"),
+        React.createElement("button", { className: "atlas-guide-button", onClick: copyPrompt, type: "button" }, copied ? "Prompt copied" : "Copy sprint prompt"),
+        React.createElement("a", { className: "atlas-guide-link", href: "/skills" }, "Inspect skills")
+      )
+    );
+  }
+
+  window.__HERMES_PLUGINS__.registerSlot(pluginName, "chat:top", OperatorGuide);
+}());

services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css

@@ -0,0 +1,64 @@
+.atlas-guide-card {
+  border: 1px solid rgba(247, 201, 72, 0.7);
+  background: rgba(247, 201, 72, 0.08);
+  color: inherit;
+  padding: 0.75rem 1rem;
+  font-family: "Mondwest", monospace;
+}
+
+.atlas-guide-heading {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: 1rem;
+}
+
+.atlas-guide-heading strong {
+  color: #f7c948;
+  font-family: "Rules Expanded", sans-serif;
+  letter-spacing: 0.04em;
+  text-transform: uppercase;
+}
+
+.atlas-guide-heading p {
+  margin: 0.25rem 0 0;
+  opacity: 0.8;
+}
+
+.atlas-guide-steps {
+  margin: 0.6rem 0;
+  padding-left: 1.25rem;
+}
+
+.atlas-guide-actions {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: center;
+  gap: 0.5rem;
+}
+
+.atlas-guide-button,
+.atlas-guide-hide,
+.atlas-guide-restore {
+  border: 1px solid rgba(247, 201, 72, 0.7);
+  background: transparent;
+  color: #f7c948;
+  cursor: pointer;
+  padding: 0.35rem 0.6rem;
+  text-decoration: none;
+}
+
+.atlas-guide-button:hover,
+.atlas-guide-hide:hover,
+.atlas-guide-restore:hover {
+  background: rgba(247, 201, 72, 0.14);
+}
+
+.atlas-guide-hide,
+.atlas-guide-restore {
+  font-size: 0.75rem;
+}
+
+.atlas-guide-link {
+  color: #f7c948;
+}

services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json

@@ -0,0 +1,16 @@
+{
+  "name": "atlas-operator-guide",
+  "label": "Atlas Operator Guide",
+  "description": "Hands-on onboarding for supervised Atlas triage and Hermes skill building.",
+  "icon": "Sparkles",
+  "version": "1.0.0",
+  "tab": {
+    "path": "/atlas-operator-guide",
+    "hidden": true
+  },
+  "slots": [
+    "chat:top"
+  ],
+  "entry": "dist/index.js",
+  "css": "dist/style.css"
+}

services/hermes/skills/master-hermes-on-atlas/SKILL.md

@@ -22,15 +22,19 @@ single-question review.
 
 1. Read `references/architecture.md` before teaching deployment-specific facts.
 2. Read `references/curriculum.md` to select the next lab.
-3. Ask whether to assess, resume, or choose a lab. If no progress record exists,
+3. When Brad asks for the two-hour proof sprint, read
+   `references/two-hour-proof-sprint.md` and run it in order. Brad performs
+   every required UI action and evidence check; coach and verify without doing
+   the learning-critical steps for him.
+4. Ask whether to assess, resume, or choose a lab. If no progress record exists,
    default to a five-question assessment followed by the first weak area.
-4. Give one bounded task at a time. State the goal, safety boundary, exact
+5. Give one bounded task at a time. State the goal, safety boundary, exact
    success evidence, and at most one initial hint.
-5. Wait for Brad's answer or observed command output before revealing the
+6. Wait for Brad's answer or observed command output before revealing the
    explanation.
-6. Grade with `references/mastery-rubric.md`. Separate demonstrated ability
+7. Grade with `references/mastery-rubric.md`. Separate demonstrated ability
    from verbal familiarity.
-7. Offer a concise progress update. Write it to
+8. Offer a concise progress update. Write it to
    `/opt/data/workspace/hermes-training/progress.md` only after Brad explicitly
    approves the file change.
 

services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml

@@ -1,4 +1,4 @@
 interface:
   display_name: "Master Hermes on Atlas"
   short_description: "Practice Hermes safely on the live Atlas stack"
-  default_prompt: "Use $master-hermes-on-atlas to assess me and begin my next hands-on Hermes lab; first stop and tell me to select openai-codex/gpt-5.4 if this session is using the local model."
+  default_prompt: "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint. I will perform every setup and evidence step myself. Start with step 1 only, require live proof, and stop at every mutation or approval boundary. First tell me to select openai-codex/gpt-5.4 if this session is using the local model."

services/monitoring/dashboards/atlas-gpu.json

@@ -20,7 +20,7 @@
       },
       "targets": [
         {
-          "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
+          "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
           "refId": "A",
           "legendFormat": "{{namespace}}",
           "instant": true
@@ -72,7 +72,7 @@
           "targetBlank": false
         }
       ],
-      "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
+      "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero."
     },
     {
       "id": 2,

services/monitoring/dashboards/atlas-overview.json

@@ -3728,7 +3728,7 @@
       },
       "targets": [
         {
-          "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
+          "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
           "refId": "A",
           "legendFormat": "{{namespace}}",
           "instant": true
@@ -3780,7 +3780,7 @@
           "targetBlank": false
         }
       ],
-      "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
+      "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero."
     },
     {
       "id": 13,

services/monitoring/grafana-dashboard-gpu.yaml

@@ -29,7 +29,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
+              "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
               "refId": "A",
               "legendFormat": "{{namespace}}",
               "instant": true
@@ -81,7 +81,7 @@ data:
               "targetBlank": false
             }
           ],
-          "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
+          "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero."
         },
         {
           "id": 2,

services/monitoring/grafana-dashboard-overview.yaml

@@ -3737,7 +3737,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
+              "expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
               "refId": "A",
               "legendFormat": "{{namespace}}",
               "instant": true
@@ -3789,7 +3789,7 @@ data:
               "targetBlank": false
             }
           ],
-          "description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
+          "description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero."
         },
         {
           "id": 13,

services/monitoring/nvidia-process-exporter.yaml

@@ -49,6 +49,7 @@ spec:
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "9401"
+        monitoring.bstein.dev/restart-rev: "20260802-current-attribution"
     spec:
       serviceAccountName: nvidia-process-exporter
       imagePullSecrets:

services/monitoring/scripts/nvidia_process_exporter.py

@@ -134,7 +134,8 @@ def running_process_memory(handle):
 
 def process_utilization_samples(handle):
     try:
-        since = int(time.time() * 1000) - SAMPLE_WINDOW_MS
+        # NVML process sample timestamps are microseconds since the epoch.
+        since = int(time.time() * 1_000_000) - (SAMPLE_WINDOW_MS * 1000)
         samples = nvmlDeviceGetProcessUtilization(handle, since)
     except NVMLError_NotFound:
         return {}, 1
@@ -156,6 +157,32 @@ def process_utilization_samples(handle):
     return by_pid, 1
 
 
+def reconcile_namespace_utilization(namespace_sm, device_util):
+    """Make namespace attribution add up to the device's current utilization."""
+
+    current = max(float(device_util), 0.0)
+    reconciled = {
+        namespace: max(float(value), 0.0)
+        for namespace, value in namespace_sm.items()
+    }
+    attributed = sum(reconciled.values())
+
+    if current == 0:
+        return {namespace: 0.0 for namespace in reconciled}
+
+    # Per-process NVML samples and the device gauge are collected on slightly
+    # different intervals. Scale a stale/overlapping process sum down to the
+    # device's current total while preserving the namespace proportions.
+    if attributed > current and attributed > 0:
+        scale = current / attributed
+        return {namespace: value * scale for namespace, value in reconciled.items()}
+
+    residual = current - attributed
+    if residual > 0.1:
+        reconciled["host"] = reconciled.get("host", 0.0) + residual
+    return reconciled
+
+
 def collect_metrics():
     nvmlInit()
     pods = load_pods()
@@ -204,10 +231,7 @@ def collect_metrics():
             lines.append(metric_line("nvidia_process_gpu_sm_util_percent", labels, sm_util))
             lines.append(metric_line("nvidia_process_gpu_memory_used_bytes", labels, int(proc_info["memory"])))
 
-        attributed = sum(namespace_sm.values())
+        namespace_sm = reconcile_namespace_utilization(namespace_sm, device_util)
-        residual = max(device_util - attributed, 0.0)
-        if residual > 0.1:
-            namespace_sm["host"] = namespace_sm.get("host", 0.0) + residual
 
         for namespace, value in sorted(namespace_sm.items()):
             labels = {**base, "namespace": namespace, "pod": "__namespace_total__"}