fix(monitoring): retain short Jetson GPU activity

scripts/render/dashboards_render_atlas.py

@@ -271,6 +271,13 @@ def jetson_gpu_util_by_node():
     return 'max by (node) (jetson_gr3d_freq_percent{node!=""})'
 
 
+def jetson_gpu_activity_by_node():
+    return (
+        '100 * sum by (node) (increase('
+        'jetson_gr3d_active_seconds_total{node!=""}[$__range])) / $__range_s'
+    )
+
+
 def dcgm_gpu_util_by_node():
     dcgm_pod = 'label_replace(DCGM_FI_DEV_GPU_UTIL, "pod", "$1", "Hostname", "(.*)")'
     dcgm_ns = 'label_replace(' + dcgm_pod + ', "namespace", "monitoring", "", "")'
@@ -330,7 +337,7 @@ def jetson_gpu_usage_by_namespace(scope_var):
     return (
         "sum by (namespace) ("
         f"({requests_by_ns}) / on(node) group_left() clamp_min({total_by_node}, 1) "
-        f"* on(node) group_left() {jetson_gpu_util_by_node()}"
+        f"* on(node) group_left() ({jetson_gpu_activity_by_node()})"
         ")"
     )
 
@@ -338,7 +345,7 @@ def jetson_gpu_usage_by_namespace(scope_var):
 def jetson_gpu_requested_nodes():
     all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
     requested = f"(sum by (node) ({all_requests}) > 0)"
-    return f"({requested}) and on(node) ({jetson_gpu_util_by_node()})"
+    return f"({requested}) and on(node) ({jetson_gpu_activity_by_node()})"
 
 
 def namespace_share_expr(resource_expr):
@@ -355,11 +362,12 @@ def namespace_ram_share_expr(scope_var):
 
 
 def nvidia_process_gpu_usage_by_namespace(scope_var):
-    usage = (
-        "sum by (namespace) ("
+    samples = (
+        "avg_over_time("
         f"nvidia_namespace_gpu_sm_util_percent{{{namespace_gpu_selector(scope_var)}}}"
-        ")"
+        "[$__range])"
     )
+    usage = f"sum by (namespace) ({samples})"
     return f"(({usage}) > 0)"
 
 
@@ -1941,7 +1949,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
     "Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.",
     "Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.",
     "Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.",
-    "Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.",
+    "Namespace GPU Utilization": "Proportional share of GPU compute activity observed over the selected time range. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 activity is continuously sampled and assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of observed compute; idle appears only when the selected range contains no activity.",
     "Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.",
     "Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.",
     "Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.",

scripts/tests/test_dashboards_render_atlas.py

@@ -173,10 +173,13 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     gpu_expr = panels_by_title["Namespace GPU Utilization"]["targets"][0]["expr"]
     assert "nvidia_namespace_gpu_sm_util_percent" in gpu_expr
     assert "nvidia_gpu_device_utilization_percent" in gpu_expr
+    assert "jetson_gr3d_active_seconds_total" in gpu_expr
+    assert "increase" in gpu_expr
     assert "sum_over_time" not in gpu_expr
     assert "count_over_time" not in gpu_expr
-    assert "avg_over_time" not in gpu_expr
-    assert "$__range" not in gpu_expr
+    assert "avg_over_time" in gpu_expr
+    assert "$__range" in gpu_expr
+    assert "$__range_s" in gpu_expr
     assert "sum by (namespace)" in gpu_expr
     assert 'namespace", "shared"' not in gpu_expr
     assert "kube_pod_container_resource_requests" in gpu_expr
@@ -188,8 +191,8 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
     assert 'namespace", "unattributed"' in gpu_expr
     assert 'namespace", "idle"' in gpu_expr
     assert panels_by_title["Namespace GPU Utilization"]["targets"][0]["instant"] is True
-    assert "Current proportional share" in panels_by_title["Namespace GPU Utilization"]["description"]
-    assert "independent of the selected dashboard time range" in panels_by_title["Namespace GPU Utilization"]["description"]
+    assert "selected time range" in panels_by_title["Namespace GPU Utilization"]["description"]
+    assert "continuously sampled" in panels_by_title["Namespace GPU Utilization"]["description"]
     assert "Kubernetes shared-GPU allocations" in panels_by_title["Namespace GPU Utilization"]["description"]
 
 

services/monitoring/jetson-tegrastats-exporter.yaml

@@ -17,7 +17,7 @@ spec:
       annotations:
         prometheus.io/scrape: "true"
         prometheus.io/port: "9100"
-        monitoring.bstein.dev/restart-rev: "7"
+        monitoring.bstein.dev/restart-rev: "8"
     spec:
       serviceAccountName: default
       hostPID: true

services/monitoring/scripts/jetson_tegrastats_exporter.py

@@ -3,7 +3,8 @@ import os
 import re
 import socketserver
 import subprocess
-from time import time
+import threading
+from time import sleep, time
 
 PORT = int(os.environ.get("JETSON_EXPORTER_PORT", "9100"))
 NODE_NAME = os.environ.get("NODE_NAME") or os.uname().nodename
@@ -16,7 +17,12 @@ BASE_METRICS = {
     "power_5v_in_mw": 0.0,
     "log_line_len": 0.0,
     "last_scrape_ts": 0.0,
+    "last_sample_timestamp_seconds": 0.0,
+    "gr3d_active_seconds_total": 0.0,
+    "tegrastats_samples_total": 0.0,
 }
+COUNTER_METRICS = {"gr3d_active_seconds_total", "tegrastats_samples_total"}
+
 
 def parse_line(line: str) -> dict:
     line = line.strip()
@@ -39,23 +45,73 @@ def parse_line(line: str) -> dict:
         updates["power_5v_in_mw"] = float(m.group(1))
     return updates
 
-def read_latest_line() -> str:
-    try:
-        proc = subprocess.Popen(
-            ["/host/usr/bin/tegrastats", "--interval", "1000"],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            text=True,
-        )
-        line = proc.stdout.readline()
-        proc.terminate()
+
+class MetricStore:
+    """Retain samples and integrate short GPU bursts between Prometheus scrapes."""
+
+    def __init__(self):
+        self.metrics = BASE_METRICS.copy()
+        self.last_sample_at = None
+        self.lock = threading.Lock()
+
+    def record(self, line: str, now: float | None = None) -> None:
+        """Record one tegrastats line and add utilization-weighted GPU seconds."""
+        sampled_at = time() if now is None else now
+        updates = parse_line(line)
+        if not updates:
+            return
+        with self.lock:
+            if self.last_sample_at is not None:
+                elapsed = min(max(sampled_at - self.last_sample_at, 0.0), 2.0)
+                previous = self.metrics["gr3d_freq_percent"]
+                self.metrics["gr3d_active_seconds_total"] += elapsed * previous / 100.0
+            self.metrics.update(updates)
+            self.metrics["log_line_len"] = float(len(line))
+            self.metrics["last_sample_timestamp_seconds"] = sampled_at
+            self.metrics["tegrastats_samples_total"] += 1
+            self.last_sample_at = sampled_at
+
+    def snapshot(self, now: float | None = None) -> dict:
+        """Return an atomic copy suitable for Prometheus exposition."""
+        scraped_at = time() if now is None else now
+        with self.lock:
+            metrics = self.metrics.copy()
+        metrics["last_scrape_ts"] = scraped_at
+        return metrics
+
+
+METRICS = MetricStore()
+
+
+def sample_forever() -> None:
+    """Keep tegrastats running so sub-scrape inference bursts are retained."""
+    while True:
         try:
+            proc = subprocess.Popen(
+                ["/host/usr/bin/tegrastats", "--interval", "250"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+            )
+            if proc.stdout is not None:
+                for line in proc.stdout:
+                    METRICS.record(line)
             proc.wait(timeout=1)
-        except subprocess.TimeoutExpired:
-            proc.kill()
-        return line
-    except OSError:
-        return ""
+        except (OSError, subprocess.SubprocessError):
+            pass
+        sleep(1)
+
+
+def render_metrics(metrics: dict) -> str:
+    """Render the current metric snapshot in Prometheus text format."""
+    out = []
+    label = f'{{node="{NODE_NAME}"}}'
+    for key, value in metrics.items():
+        metric_type = "counter" if key in COUNTER_METRICS else "gauge"
+        out.append(f"# TYPE jetson_{key} {metric_type}")
+        out.append(f"jetson_{key}{label} {value}")
+    return "\n".join(out) + "\n"
+
 
 class Handler(http.server.BaseHTTPRequestHandler):
     def do_GET(self):
@@ -63,18 +119,7 @@ class Handler(http.server.BaseHTTPRequestHandler):
             self.send_response(404)
             self.end_headers()
             return
-        metrics = BASE_METRICS.copy()
-        line = read_latest_line()
-        if line:
-            metrics.update(parse_line(line))
-            metrics["log_line_len"] = float(len(line))
-        metrics["last_scrape_ts"] = time()
-        out = []
-        label = f'{{node="{NODE_NAME}"}}'
-        for k, v in metrics.items():
-            out.append(f"# TYPE jetson_{k} gauge")
-            out.append(f"jetson_{k}{label} {v}")
-        body = "\n".join(out) + "\n"
+        body = render_metrics(METRICS.snapshot())
         self.send_response(200)
         self.send_header("Content-Type", "text/plain; version=0.0.4")
         self.send_header("Content-Length", str(len(body)))
@@ -84,6 +129,8 @@ class Handler(http.server.BaseHTTPRequestHandler):
     def log_message(self, fmt, *args):
         return
 
+
 if __name__ == "__main__":
+    threading.Thread(target=sample_forever, daemon=True).start()
     with socketserver.TCPServer(("", PORT), Handler) as httpd:
         httpd.serve_forever()

testing/tests/test_jetson_tegrastats_exporter.py

@@ -0,0 +1,54 @@
+"""Tests for continuous Jetson GPU activity accounting."""
+
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+
+SCRIPT = (
+    Path(__file__).parents[2]
+    / "services"
+    / "monitoring"
+    / "scripts"
+    / "jetson_tegrastats_exporter.py"
+)
+
+
+def load_module():
+    spec = importlib.util.spec_from_file_location("jetson_tegrastats_exporter", SCRIPT)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_metric_store_retains_gpu_work_between_scrapes():
+    module = load_module()
+    store = module.MetricStore()
+
+    store.record("GR3D_FREQ 0% GPU@40C RAM 1/2MB", now=10.0)
+    store.record("GR3D_FREQ 100% GPU@41C RAM 1/2MB", now=10.25)
+    store.record("GR3D_FREQ 0% GPU@40C RAM 1/2MB", now=11.0)
+    snapshot = store.snapshot(now=15.0)
+
+    assert snapshot["gr3d_freq_percent"] == 0
+    assert snapshot["gr3d_active_seconds_total"] == 0.75
+    assert snapshot["tegrastats_samples_total"] == 3
+    assert snapshot["last_sample_timestamp_seconds"] == 11.0
+    assert snapshot["last_scrape_ts"] == 15.0
+
+
+def test_activity_metrics_are_exported_as_counters():
+    module = load_module()
+    body = module.render_metrics(
+        {
+            "gr3d_freq_percent": 0,
+            "gr3d_active_seconds_total": 0.75,
+            "tegrastats_samples_total": 3,
+        }
+    )
+
+    assert "# TYPE jetson_gr3d_freq_percent gauge" in body
+    assert "# TYPE jetson_gr3d_active_seconds_total counter" in body
+    assert "# TYPE jetson_tegrastats_samples_total counter" in body