bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitoring(dashboards): tune namespace share metrics

Browse Source
This commit is contained in:
Brad Stein 2026-01-05 13:30:33 -03:00
parent 89d47cba79
commit 28a5d53c98
5 changed files with 23 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
scripts/dashboards_render_atlas.py
View File

@ -46,6 +46,8 @@ PERCENT_THRESHOLDS = {
],
}
NAMESPACE_CPU_WINDOW = "1m"
# ---------------------------------------------------------------------------
# Cluster metadata
# ---------------------------------------------------------------------------
@ -172,7 +174,7 @@ def node_io_expr(scope=""):
def namespace_selector(scope_var):
return f'namespace!="",pod!="",container!="",{scope_var}'
return f'namespace!="",pod!="",container!="",container!="POD",{scope_var}'
def namespace_gpu_selector(scope_var):
@ -180,7 +182,10 @@ def namespace_gpu_selector(scope_var):
def namespace_cpu_raw(scope_var):
return f"sum(rate(container_cpu_usage_seconds_total{{{namespace_selector(scope_var)}}}[5m])) by (namespace)"
return (
"sum(rate(container_cpu_usage_seconds_total"
f"{{{namespace_selector(scope_var)}}}[{NAMESPACE_CPU_WINDOW}])) by (namespace)"
)
def namespace_ram_raw(scope_var):
@ -942,7 +947,7 @@ def build_overview():
namespace_cpu_share_expr(cpu_scope),
{"h": 9, "w": 8, "x": 0, "y": 16},
links=namespace_scope_links("namespace_scope_cpu"),
description="Use panel links to switch namespace scope.",
description="Values are normalized within the selected scope; use panel links to switch scope.",
)
)
panels.append(
@ -952,7 +957,7 @@ def build_overview():
namespace_gpu_share_expr(gpu_scope),
{"h": 9, "w": 8, "x": 8, "y": 16},
links=namespace_scope_links("namespace_scope_gpu"),
description="Use panel links to switch namespace scope.",
description="Values are normalized within the selected scope; use panel links to switch scope.",
)
)
panels.append(
@ -962,7 +967,7 @@ def build_overview():
namespace_ram_share_expr(ram_scope),
{"h": 9, "w": 8, "x": 16, "y": 16},
links=namespace_scope_links("namespace_scope_ram"),
description="Use panel links to switch namespace scope.",
description="Values are normalized within the selected scope; use panel links to switch scope.",
)
)
@ -1783,7 +1788,7 @@ def build_gpu_dashboard():
namespace_gpu_share_expr(gpu_scope),
{"h": 8, "w": 12, "x": 0, "y": 0},
links=namespace_scope_links("namespace_scope_gpu"),
description="Use panel links to switch namespace scope.",
description="Values are normalized within the selected scope; use panel links to switch scope.",
)
)
panels.append(

2
services/monitoring/dashboards/atlas-gpu.json
View File

@ -71,7 +71,7 @@
"targetBlank": false
}
],
"description": "Use panel links to switch namespace scope."
"description": "Values are normalized within the selected scope; use panel links to switch scope."
},
{
"id": 2,

10
services/monitoring/dashboards/atlas-overview.json
View File

@ -1086,7 +1086,7 @@
},
"targets": [
{
"expr": "100 * ( sum(rate(container_cpu_usage_seconds_total{namespace!=\"\",pod!=\"\",container!=\"\",$namespace_scope_cpu}[5m])) by (namespace) ) / clamp_min(sum( sum(rate(container_cpu_usage_seconds_total{namespace!=\"\",pod!=\"\",container!=\"\",$namespace_scope_cpu}[5m])) by (namespace) ), 1)",
"expr": "100 * ( sum(rate(container_cpu_usage_seconds_total{namespace!=\"\",pod!=\"\",container!=\"\",container!=\"POD\",$namespace_scope_cpu}[1m])) by (namespace) ) / clamp_min(sum( sum(rate(container_cpu_usage_seconds_total{namespace!=\"\",pod!=\"\",container!=\"\",container!=\"POD\",$namespace_scope_cpu}[1m])) by (namespace) ), 1)",
"refId": "A",
"legendFormat": "{{namespace}}"
}
@ -1137,7 +1137,7 @@
"targetBlank": false
}
],
"description": "Use panel links to switch namespace scope."
"description": "Values are normalized within the selected scope; use panel links to switch scope."
},
{
"id": 12,
@ -1206,7 +1206,7 @@
"targetBlank": false
}
],
"description": "Use panel links to switch namespace scope."
"description": "Values are normalized within the selected scope; use panel links to switch scope."
},
{
"id": 13,
@ -1224,7 +1224,7 @@
},
"targets": [
{
"expr": "100 * ( sum(container_memory_working_set_bytes{namespace!=\"\",pod!=\"\",container!=\"\",$namespace_scope_ram}) by (namespace) ) / clamp_min(sum( sum(container_memory_working_set_bytes{namespace!=\"\",pod!=\"\",container!=\"\",$namespace_scope_ram}) by (namespace) ), 1)",
"expr": "100 * ( sum(container_memory_working_set_bytes{namespace!=\"\",pod!=\"\",container!=\"\",container!=\"POD\",$namespace_scope_ram}) by (namespace) ) / clamp_min(sum( sum(container_memory_working_set_bytes{namespace!=\"\",pod!=\"\",container!=\"\",container!=\"POD\",$namespace_scope_ram}) by (namespace) ), 1)",
"refId": "A",
"legendFormat": "{{namespace}}"
}
@ -1275,7 +1275,7 @@
"targetBlank": false
}
],
"description": "Use panel links to switch namespace scope."
"description": "Values are normalized within the selected scope; use panel links to switch scope."
},
{
"id": 14,

2
services/monitoring/grafana-dashboard-gpu.yaml
View File

@ -80,7 +80,7 @@ data:
"targetBlank": false
}
],
"description": "Use panel links to switch namespace scope."
"description": "Values are normalized within the selected scope; use panel links to switch scope."
},
{
"id": 2,

10
services/monitoring/grafana-dashboard-overview.yaml
View File

@ -1095,7 +1095,7 @@ data:
},
"targets": [
{
"expr": "100 * ( sum(rate(container_cpu_usage_seconds_total{namespace!=\"\",pod!=\"\",container!=\"\",$namespace_scope_cpu}[5m])) by (namespace) ) / clamp_min(sum( sum(rate(container_cpu_usage_seconds_total{namespace!=\"\",pod!=\"\",container!=\"\",$namespace_scope_cpu}[5m])) by (namespace) ), 1)",
"expr": "100 * ( sum(rate(container_cpu_usage_seconds_total{namespace!=\"\",pod!=\"\",container!=\"\",container!=\"POD\",$namespace_scope_cpu}[1m])) by (namespace) ) / clamp_min(sum( sum(rate(container_cpu_usage_seconds_total{namespace!=\"\",pod!=\"\",container!=\"\",container!=\"POD\",$namespace_scope_cpu}[1m])) by (namespace) ), 1)",
"refId": "A",
"legendFormat": "{{namespace}}"
}
@ -1146,7 +1146,7 @@ data:
"targetBlank": false
}
],
"description": "Use panel links to switch namespace scope."
"description": "Values are normalized within the selected scope; use panel links to switch scope."
},
{
"id": 12,
@ -1215,7 +1215,7 @@ data:
"targetBlank": false
}
],
"description": "Use panel links to switch namespace scope."
"description": "Values are normalized within the selected scope; use panel links to switch scope."
},
{
"id": 13,
@ -1233,7 +1233,7 @@ data:
},
"targets": [
{
"expr": "100 * ( sum(container_memory_working_set_bytes{namespace!=\"\",pod!=\"\",container!=\"\",$namespace_scope_ram}) by (namespace) ) / clamp_min(sum( sum(container_memory_working_set_bytes{namespace!=\"\",pod!=\"\",container!=\"\",$namespace_scope_ram}) by (namespace) ), 1)",
"expr": "100 * ( sum(container_memory_working_set_bytes{namespace!=\"\",pod!=\"\",container!=\"\",container!=\"POD\",$namespace_scope_ram}) by (namespace) ) / clamp_min(sum( sum(container_memory_working_set_bytes{namespace!=\"\",pod!=\"\",container!=\"\",container!=\"POD\",$namespace_scope_ram}) by (namespace) ), 1)",
"refId": "A",
"legendFormat": "{{namespace}}"
}
@ -1284,7 +1284,7 @@ data:
"targetBlank": false
}
],
"description": "Use panel links to switch namespace scope."
"description": "Values are normalized within the selected scope; use panel links to switch scope."
},
{
"id": 14,