bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ai(hermes): add operator guide and current GPU shares

Browse Source
This commit is contained in:
jenkins 2026-08-02 03:59:32 -03:00
parent 2fe327c7eb
commit 4991493d3b
17 changed files with 352 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
scripts/dashboards_render_atlas.py
View File

@ -363,21 +363,6 @@ def nvidia_process_gpu_usage_by_namespace(scope_var):
return f"(({usage}) > 0)"
def nvidia_process_gpu_usage_by_namespace_range(scope_var, window="$__range"):
namespace_samples = (
"sum_over_time(nvidia_namespace_gpu_sm_util_percent"
f"{{{namespace_gpu_selector(scope_var)}}}[{window}])"
)
device_samples = (
f"count_over_time(nvidia_gpu_device_utilization_percent[{window}])"
)
averaged = (
f"({namespace_samples}) / on(node,gpu,uuid,model) group_left() "
f"clamp_min({device_samples}, 1)"
)
return f"((sum by (namespace) ({averaged})) > 0)"
def nvidia_gpu_device_utilization():
return "max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))"
@ -411,41 +396,17 @@ def unattributed_gpu_usage():
)
def legacy_gpu_util_range_without_process_exporter(window="$__range"):
dcgm_range = dcgm_gpu_util_by_node().replace(
"DCGM_FI_DEV_GPU_UTIL", f"avg_over_time(DCGM_FI_DEV_GPU_UTIL[{window}])"
)
jetson_range = (
f'max by (node) (avg_over_time(jetson_gr3d_freq_percent{{node!=""}}[{window}]))'
)
process_exporter_nodes = (
f"max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[{window}]))"
)
return f"(({dcgm_range}) or ({jetson_range})) unless on(node) ({process_exporter_nodes})"
def unattributed_gpu_usage_range(window="$__range"):
total = f"(sum({legacy_gpu_util_range_without_process_exporter(window)}) or on() vector(0))"
return f'label_replace(({total} > 0), "namespace", "unattributed", "", "")'
def gpu_utilization_raw(scope_var):
return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})"
def gpu_utilization_range_raw(scope_var, window="$__range"):
attributed = nvidia_process_gpu_usage_by_namespace_range(scope_var, window)
unattributed = unattributed_gpu_usage_range(window)
return f"({attributed}) or ({unattributed})"
def gpu_pool_used_expr(scope_var):
raw_total = f"(sum({gpu_utilization_raw(scope_var)}) or on() vector(0))"
return f"100 * {raw_total} / clamp_min({gpu_capacity_percent()}, 1)"
def namespace_gpu_share_expr(scope_var):
activity = gpu_utilization_range_raw(scope_var)
activity = gpu_utilization_raw(scope_var)
total = f"(sum({activity}) or on() vector(0))"
share = f"100 * ({activity}) / clamp_min({total}, 1)"
idle = (
@ -1955,7 +1916,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
"Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.",
"Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.",
"Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.",
"Namespace GPU Utilization": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero.",
"Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.",
"Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.",
"Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.",
"Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.",

12
scripts/tests/test_dashboards_render_atlas.py
View File

@ -157,11 +157,10 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
assert "nvidia_gpu_device_utilization_percent" in gpu_expr
assert "sum_over_time" in gpu_expr
assert "count_over_time" in gpu_expr
assert "avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range])" in gpu_expr
assert "avg_over_time(jetson_gr3d_freq_percent" in gpu_expr
assert "$__range" in gpu_expr
assert "sum_over_time" not in gpu_expr
assert "count_over_time" not in gpu_expr
assert "avg_over_time" not in gpu_expr
assert "$__range" not in gpu_expr
assert "sum by (namespace)" in gpu_expr
assert 'namespace", "shared"' not in gpu_expr
assert "kube_node_labels" not in gpu_expr
@ -170,7 +169,8 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
assert 'namespace", "unattributed"' in gpu_expr
assert 'namespace", "idle"' in gpu_expr
assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
assert "Time-weighted share" in panels_by_title["Namespace GPU Utilization"]["description"]
assert "Current proportional share" in panels_by_title["Namespace GPU Utilization"]["description"]
assert "independent of the selected dashboard time range" in panels_by_title["Namespace GPU Utilization"]["description"]
def test_gpu_node_panel_prefers_stable_process_metrics_and_covers_all_gpu_families():

99
scripts/tests/test_nvidia_process_exporter.py Normal file
View File

@ -0,0 +1,99 @@
import importlib.util
from pathlib import Path
import sys
import types
import pytest
ROOT = Path(__file__).resolve().parents[2]
EXPORTER_PATH = ROOT / "services" / "monitoring" / "scripts" / "nvidia_process_exporter.py"
def load_exporter(monkeypatch):
"""Load the exporter without requiring an NVIDIA driver on the test host."""
pynvml = types.ModuleType("pynvml")
class NVMLError(Exception):
pass
class NVMLErrorNotFound(NVMLError):
pass
class NVMLErrorNotSupported(NVMLError):
pass
pynvml.NVMLError = NVMLError
pynvml.NVMLError_NotFound = NVMLErrorNotFound
pynvml.NVMLError_NotSupported = NVMLErrorNotSupported
for name in (
"nvmlDeviceGetComputeRunningProcesses_v3",
"nvmlDeviceGetCount",
"nvmlDeviceGetGraphicsRunningProcesses_v3",
"nvmlDeviceGetHandleByIndex",
"nvmlDeviceGetName",
"nvmlDeviceGetProcessUtilization",
"nvmlDeviceGetUUID",
"nvmlDeviceGetUtilizationRates",
"nvmlInit",
):
setattr(pynvml, name, lambda *args, **kwargs: None)
monkeypatch.setitem(sys.modules, "pynvml", pynvml)
spec = importlib.util.spec_from_file_location("nvidia_process_exporter_test", EXPORTER_PATH)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_process_sample_window_uses_nvml_microseconds(monkeypatch):
exporter = load_exporter(monkeypatch)
observed = []
monkeypatch.setattr(exporter.time, "time", lambda: 1_700_000_000.0)
monkeypatch.setattr(
exporter,
"nvmlDeviceGetProcessUtilization",
lambda handle, since: observed.append(since) or [],
)
samples, supported = exporter.process_utilization_samples(object())
assert samples == {}
assert supported == 1
assert observed == [1_700_000_000_000_000 - 30_000_000]
def test_namespace_attribution_scales_to_current_device_total(monkeypatch):
exporter = load_exporter(monkeypatch)
result = exporter.reconcile_namespace_utilization(
{"game-stream": 20, "hermes": 10},
device_util=3,
)
assert sum(result.values()) == pytest.approx(3)
assert result["game-stream"] == pytest.approx(2)
assert result["hermes"] == pytest.approx(1)
def test_namespace_attribution_assigns_unexplained_compute_to_host(monkeypatch):
exporter = load_exporter(monkeypatch)
result = exporter.reconcile_namespace_utilization(
{"hermes": 1},
device_util=3,
)
assert result == {"hermes": 1, "host": 2}
def test_zero_device_utilization_clears_stale_process_samples(monkeypatch):
exporter = load_exporter(monkeypatch)
result = exporter.reconcile_namespace_utilization(
{"hermes": 40, "game-stream": 5},
device_util=0,
)
assert result == {"hermes": 0, "game-stream": 0}

4
services/hermes/configmap.yaml
View File

@ -28,6 +28,10 @@ data:
external_dirs:
- /opt/data/workspace/skills
plugins:
enabled:
- atlas-operator-guide
terminal:
backend: local
cwd: /opt/data/workspace

17
services/hermes/deployment.yaml
View File

@ -23,7 +23,7 @@ spec:
ai.bstein.dev/model: gpt-oss:20b with openai-codex fallback
ai.bstein.dev/role: testing-triage
ai.bstein.dev/placement: arm64 gateway lane (rpi5 preferred)
ai.bstein.dev/config-rev: "20260802-mastery-lab"
ai.bstein.dev/config-rev: "20260802-operator-guide"
spec:
serviceAccountName: hermes-triage
automountServiceAccountToken: true
@ -197,6 +197,9 @@ spec:
- name: mastery-skill
mountPath: /opt/data/workspace/skills/master-hermes-on-atlas
readOnly: true
- name: operator-guide-plugin
mountPath: /opt/data/plugins/atlas-operator-guide
readOnly: true
readinessProbe:
httpGet:
path: /api/status
@ -251,3 +254,15 @@ spec:
path: references/incident-drills.md
- key: mastery-rubric.md
path: references/mastery-rubric.md
- key: two-hour-proof-sprint.md
path: references/two-hour-proof-sprint.md
- name: operator-guide-plugin
configMap:
name: hermes-operator-guide-plugin
items:
- key: manifest.json
path: dashboard/manifest.json
- key: index.js
path: dashboard/dist/index.js
- key: style.css
path: dashboard/dist/style.css

9
services/hermes/kustomization.yaml
View File

@ -38,5 +38,14 @@ configMapGenerator:
- curriculum.md=skills/master-hermes-on-atlas/references/curriculum.md
- incident-drills.md=skills/master-hermes-on-atlas/references/incident-drills.md
- mastery-rubric.md=skills/master-hermes-on-atlas/references/mastery-rubric.md
- two-hour-proof-sprint.md=skills/master-hermes-on-atlas/references/two-hour-proof-sprint.md
options:
disableNameSuffixHash: true
- name: hermes-operator-guide-plugin
namespace: hermes
files:
- manifest.json=plugins/atlas-operator-guide/dashboard/manifest.json
- index.js=plugins/atlas-operator-guide/dashboard/dist/index.js
- style.css=plugins/atlas-operator-guide/dashboard/dist/style.css
options:
disableNameSuffixHash: true

88
services/hermes/plugins/atlas-operator-guide/dashboard/dist/index.js vendored Normal file
View File

@ -0,0 +1,88 @@
(function () {
"use strict";
var pluginName = "atlas-operator-guide";
var storageKey = "atlas.operator-guide.hidden.v1";
var prompt = [
"Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint.",
"I will perform every setup and evidence step myself.",
"Start with step 1 only, require live proof, and stop at every mutation or approval boundary.",
"First tell me to select openai-codex/gpt-5.4 if this session is using the local model."
].join(" ");
function OperatorGuide() {
var sdk = window.__HERMES_PLUGIN_SDK__;
var React = sdk.React;
var state = sdk.hooks.useState(function () {
return window.localStorage.getItem(storageKey) === "1";
});
var hidden = state[0];
var setHidden = state[1];
var copiedState = sdk.hooks.useState(false);
var copied = copiedState[0];
var setCopied = copiedState[1];
if (hidden) {
return React.createElement(
"button",
{
className: "atlas-guide-restore",
onClick: function () {
window.localStorage.removeItem(storageKey);
setHidden(false);
},
type: "button"
},
"Show Atlas operator walkthrough"
);
}
function copyPrompt() {
navigator.clipboard.writeText(prompt).then(function () {
setCopied(true);
window.setTimeout(function () { setCopied(false); }, 2500);
});
}
return React.createElement(
"section",
{ className: "atlas-guide-card", role: "region", "aria-label": "Atlas operator walkthrough" },
React.createElement(
"div",
{ className: "atlas-guide-heading" },
React.createElement("div", null,
React.createElement("strong", null, "Start here: two-hour Hermes proof sprint"),
React.createElement("p", null, "You do the setup and evidence work. Hermes coaches, checks, and stops at approval boundaries.")
),
React.createElement(
"button",
{
className: "atlas-guide-hide",
onClick: function () {
window.localStorage.setItem(storageKey, "1");
setHidden(true);
},
type: "button"
},
"Hide"
)
),
React.createElement(
"ol",
{ className: "atlas-guide-steps" },
React.createElement("li", null, "Open Models and select openai-codex/gpt-5.4 for the coaching session."),
React.createElement("li", null, "Copy the sprint prompt, paste it into Chat, and press Enter."),
React.createElement("li", null, "Complete one live triage and build one writable skill yourself.")
),
React.createElement(
"div",
{ className: "atlas-guide-actions" },
React.createElement("a", { className: "atlas-guide-button", href: "/models" }, "Open Models"),
React.createElement("button", { className: "atlas-guide-button", onClick: copyPrompt, type: "button" }, copied ? "Prompt copied" : "Copy sprint prompt"),
React.createElement("a", { className: "atlas-guide-link", href: "/skills" }, "Inspect skills")
)
);
}
window.__HERMES_PLUGINS__.registerSlot(pluginName, "chat:top", OperatorGuide);
}());

64
services/hermes/plugins/atlas-operator-guide/dashboard/dist/style.css vendored Normal file
View File

@ -0,0 +1,64 @@
.atlas-guide-card {
border: 1px solid rgba(247, 201, 72, 0.7);
background: rgba(247, 201, 72, 0.08);
color: inherit;
padding: 0.75rem 1rem;
font-family: "Mondwest", monospace;
}
.atlas-guide-heading {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 1rem;
}
.atlas-guide-heading strong {
color: #f7c948;
font-family: "Rules Expanded", sans-serif;
letter-spacing: 0.04em;
text-transform: uppercase;
}
.atlas-guide-heading p {
margin: 0.25rem 0 0;
opacity: 0.8;
}
.atlas-guide-steps {
margin: 0.6rem 0;
padding-left: 1.25rem;
}
.atlas-guide-actions {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 0.5rem;
}
.atlas-guide-button,
.atlas-guide-hide,
.atlas-guide-restore {
border: 1px solid rgba(247, 201, 72, 0.7);
background: transparent;
color: #f7c948;
cursor: pointer;
padding: 0.35rem 0.6rem;
text-decoration: none;
}
.atlas-guide-button:hover,
.atlas-guide-hide:hover,
.atlas-guide-restore:hover {
background: rgba(247, 201, 72, 0.14);
}
.atlas-guide-hide,
.atlas-guide-restore {
font-size: 0.75rem;
}
.atlas-guide-link {
color: #f7c948;
}

16
services/hermes/plugins/atlas-operator-guide/dashboard/manifest.json Normal file
View File

@ -0,0 +1,16 @@
{
"name": "atlas-operator-guide",
"label": "Atlas Operator Guide",
"description": "Hands-on onboarding for supervised Atlas triage and Hermes skill building.",
"icon": "Sparkles",
"version": "1.0.0",
"tab": {
"path": "/atlas-operator-guide",
"hidden": true
},
"slots": [
"chat:top"
],
"entry": "dist/index.js",
"css": "dist/style.css"
}

14
services/hermes/skills/master-hermes-on-atlas/SKILL.md
View File

@ -22,15 +22,19 @@ single-question review.
1. Read `references/architecture.md` before teaching deployment-specific facts.
2. Read `references/curriculum.md` to select the next lab.
3. Ask whether to assess, resume, or choose a lab. If no progress record exists,
3. When Brad asks for the two-hour proof sprint, read
`references/two-hour-proof-sprint.md` and run it in order. Brad performs
every required UI action and evidence check; coach and verify without doing
the learning-critical steps for him.
4. Ask whether to assess, resume, or choose a lab. If no progress record exists,
default to a five-question assessment followed by the first weak area.
4. Give one bounded task at a time. State the goal, safety boundary, exact
5. Give one bounded task at a time. State the goal, safety boundary, exact
success evidence, and at most one initial hint.
5. Wait for Brad's answer or observed command output before revealing the
6. Wait for Brad's answer or observed command output before revealing the
explanation.
6. Grade with `references/mastery-rubric.md`. Separate demonstrated ability
7. Grade with `references/mastery-rubric.md`. Separate demonstrated ability
from verbal familiarity.
7. Offer a concise progress update. Write it to
8. Offer a concise progress update. Write it to
`/opt/data/workspace/hermes-training/progress.md` only after Brad explicitly
approves the file change.

2
services/hermes/skills/master-hermes-on-atlas/agents/openai.yaml
View File

@ -1,4 +1,4 @@
interface:
display_name: "Master Hermes on Atlas"
short_description: "Practice Hermes safely on the live Atlas stack"
default_prompt: "Use $master-hermes-on-atlas to assess me and begin my next hands-on Hermes lab; first stop and tell me to select openai-codex/gpt-5.4 if this session is using the local model."
default_prompt: "Use $master-hermes-on-atlas to run the Atlas two-hour proof sprint. I will perform every setup and evidence step myself. Start with step 1 only, require live proof, and stop at every mutation or approval boundary. First tell me to select openai-codex/gpt-5.4 if this session is using the local model."

4
services/monitoring/dashboards/atlas-gpu.json
View File

@ -20,7 +20,7 @@
},
"targets": [
{
"expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
"expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
"refId": "A",
"legendFormat": "{{namespace}}",
"instant": true
@ -72,7 +72,7 @@
"targetBlank": false
}
],
"description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
"description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero."
},
{
"id": 2,

4
services/monitoring/dashboards/atlas-overview.json
View File

@ -3728,7 +3728,7 @@
},
"targets": [
{
"expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
"expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
"refId": "A",
"legendFormat": "{{namespace}}",
"instant": true
@ -3780,7 +3780,7 @@
"targetBlank": false
}
],
"description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
"description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero."
},
{
"id": 13,

4
services/monitoring/grafana-dashboard-gpu.yaml
View File

@ -29,7 +29,7 @@ data:
},
"targets": [
{
"expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
"expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
"refId": "A",
"legendFormat": "{{namespace}}",
"instant": true
@ -81,7 +81,7 @@ data:
"targetBlank": false
}
],
"description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
"description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero."
},
{
"id": 2,

4
services/monitoring/grafana-dashboard-overview.yaml
View File

@ -3737,7 +3737,7 @@ data:
},
"targets": [
{
"expr": "(100 * ((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) ((sum_over_time(nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}[$__range])) / on(node,gpu,uuid,model) group_left() clamp_min(count_over_time(nvidia_gpu_device_utilization_percent[$__range]), 1))) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(avg_over_time(DCGM_FI_DEV_GPU_UTIL[$__range]), \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) or (max by (node) (avg_over_time(jetson_gr3d_freq_percent{node!=\"\"}[$__range])))) unless on(node) (max by (node) (count_over_time(nvidia_gpu_device_utilization_percent[$__range])))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
"expr": "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((max by (node) (nvidia_gpu_device_utilization_percent) or (avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"})) unless on(node) (max by (node) (nvidia_gpu_device_utilization_percent)) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))",
"refId": "A",
"legendFormat": "{{namespace}}",
"instant": true
@ -3789,7 +3789,7 @@ data:
"targetBlank": false
}
],
"description": "Time-weighted share of observed GPU compute activity across the selected range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. Idle appears only when observed activity is zero."
"description": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces; host is non-pod work, and Jetson titan-20/21 activity remains unattributed when process-level attribution is unavailable. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero."
},
{
"id": 13,

1
services/monitoring/nvidia-process-exporter.yaml
View File

@ -49,6 +49,7 @@ spec:
annotations:
prometheus.io/scrape: "true"
prometheus.io/port: "9401"
monitoring.bstein.dev/restart-rev: "20260802-current-attribution"
spec:
serviceAccountName: nvidia-process-exporter
imagePullSecrets:

34
services/monitoring/scripts/nvidia_process_exporter.py
View File

@ -134,7 +134,8 @@ def running_process_memory(handle):
def process_utilization_samples(handle):
try:
since = int(time.time() * 1000) - SAMPLE_WINDOW_MS
# NVML process sample timestamps are microseconds since the epoch.
since = int(time.time() * 1_000_000) - (SAMPLE_WINDOW_MS * 1000)
samples = nvmlDeviceGetProcessUtilization(handle, since)
except NVMLError_NotFound:
return {}, 1
@ -156,6 +157,32 @@ def process_utilization_samples(handle):
return by_pid, 1
def reconcile_namespace_utilization(namespace_sm, device_util):
"""Make namespace attribution add up to the device's current utilization."""
current = max(float(device_util), 0.0)
reconciled = {
namespace: max(float(value), 0.0)
for namespace, value in namespace_sm.items()
}
attributed = sum(reconciled.values())
if current == 0:
return {namespace: 0.0 for namespace in reconciled}
# Per-process NVML samples and the device gauge are collected on slightly
# different intervals. Scale a stale/overlapping process sum down to the
# device's current total while preserving the namespace proportions.
if attributed > current and attributed > 0:
scale = current / attributed
return {namespace: value * scale for namespace, value in reconciled.items()}
residual = current - attributed
if residual > 0.1:
reconciled["host"] = reconciled.get("host", 0.0) + residual
return reconciled
def collect_metrics():
nvmlInit()
pods = load_pods()
@ -204,10 +231,7 @@ def collect_metrics():
lines.append(metric_line("nvidia_process_gpu_sm_util_percent", labels, sm_util))
lines.append(metric_line("nvidia_process_gpu_memory_used_bytes", labels, int(proc_info["memory"])))
attributed = sum(namespace_sm.values())
residual = max(device_util - attributed, 0.0)
if residual > 0.1:
namespace_sm["host"] = namespace_sm.get("host", 0.0) + residual
namespace_sm = reconcile_namespace_utilization(namespace_sm, device_util)
for namespace, value in sorted(namespace_sm.items()):
labels = {**base, "namespace": namespace, "pod": "__namespace_total__"}