bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitoring(overview): fix jenkins success/failure ranking with single-frame status labels

Browse Source
This commit is contained in:
Brad Stein 2026-04-13 23:13:45 -03:00
parent 50a9bda808
commit 6b75ae7dcc
3 changed files with 122 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
scripts/dashboards_render_atlas.py
View File

@ -505,12 +505,6 @@ JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS_BY_JOB = (
JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS_BY_JOB = (
f"min by (exported_job,job_url,weather_icon) ({JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS})"
)
JENKINS_BUILD_WEATHER_LAST_SUCCESS_TOP6_AGE_HOURS_BY_JOB = (
f"sort(bottomk(6, {JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS_BY_JOB}))"
)
JENKINS_BUILD_WEATHER_LAST_FAILURE_TOP6_AGE_HOURS_BY_JOB = (
f"sort(bottomk(6, {JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS_BY_JOB}))"
)
JENKINS_BUILD_WEATHER_LAST_DURATION_MINUTES = (
"ariadne_jenkins_build_weather_job_last_duration_seconds / 60"
)
@ -1407,6 +1401,25 @@ def _jenkins_weather_status_expr(base_expr, comparator):
)
def _jenkins_weather_topk_expr(base_expr, topk_n=6):
return f"sort(bottomk({topk_n}, {base_expr}))"
def _jenkins_weather_topk_with_status_label_expr(base_expr, topk_n=6):
topk_expr = _jenkins_weather_topk_expr(base_expr, topk_n=topk_n)
success_expr = (
f'label_replace(({topk_expr}) and on(exported_job,job_url,weather_icon) '
f'({JENKINS_BUILD_WEATHER_LAST_STATUS_BY_JOB} == 1), '
'"run_state", "ok", "exported_job", ".*")'
)
failure_expr = (
f'label_replace(({topk_expr}) and on(exported_job,job_url,weather_icon) '
f'({JENKINS_BUILD_WEATHER_LAST_STATUS_BY_JOB} != 1), '
'"run_state", "bad", "exported_job", ".*")'
)
return f"sort(({success_expr}) or ({failure_expr}))"
def jenkins_weather_bargauge_panel(
panel_id,
title,
@ -1526,7 +1539,6 @@ def jenkins_weather_statlist_panel(
limit=12,
title_size=12,
value_size=12,
merge_for_global_sort=False,
links=None,
description=None,
):
@ -1604,14 +1616,8 @@ def jenkins_weather_statlist_panel(
"textMode": "name_and_value",
"text": {"titleSize": title_size, "valueSize": value_size},
},
"transformations": [],
"transformations": [{"id": "sortBy", "options": {"fields": ["Value"], "order": sort_order}}],
}
if merge_for_global_sort:
# Merge per-status query frames so sortBy orders all rows globally.
panel["transformations"].append({"id": "merge", "options": {}})
panel["transformations"].append(
{"id": "sortBy", "options": {"fields": ["Value"], "order": sort_order}}
)
if limit:
panel["transformations"].append({"id": "limit", "options": {"limit": limit}})
if links:
@ -1621,6 +1627,78 @@ def jenkins_weather_statlist_panel(
return panel
def jenkins_weather_statlist_topk_panel(
panel_id,
title,
base_expr,
grid,
*,
topk_n=6,
unit="h",
decimals=1,
title_size=11,
value_size=11,
links=None,
description=None,
):
expr = _jenkins_weather_topk_with_status_label_expr(base_expr, topk_n=topk_n)
panel = {
"id": panel_id,
"type": "stat",
"title": title,
"datasource": PROM_DS,
"gridPos": grid,
"targets": [
{
"refId": "A",
"expr": expr,
"instant": True,
}
],
"fieldConfig": {
"defaults": {
"unit": unit,
"decimals": decimals,
"min": 0,
"displayName": "${__field.labels.weather_icon} ${__field.labels.exported_job}",
"links": [
{
"title": "Open Jenkins job",
"url": "https://ci.bstein.dev/job/${__field.labels.exported_job}/",
"targetBlank": True,
}
],
},
"overrides": [
{
"matcher": {"id": "byRegexp", "options": '.*run_state="ok".*'},
"properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}],
},
{
"matcher": {"id": "byRegexp", "options": '.*run_state="bad".*'},
"properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}],
},
],
},
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "left",
"orientation": "horizontal",
"wideLayout": True,
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"textMode": "name_and_value",
"text": {"titleSize": title_size, "valueSize": value_size},
},
"transformations": [{"id": "sortBy", "options": {"fields": ["Value"], "order": "asc"}}],
}
if links:
panel["links"] = links
if description:
panel["description"] = description
return panel
def text_panel(panel_id, title, content, grid):
return {
"id": panel_id,
@ -2202,18 +2280,16 @@ def build_overview():
)
panels.append(test_success)
panels.append(
jenkins_weather_statlist_panel(
jenkins_weather_statlist_topk_panel(
142,
"Jenkins Last Success (h, newest first)",
JENKINS_BUILD_WEATHER_LAST_SUCCESS_TOP6_AGE_HOURS_BY_JOB,
JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS_BY_JOB,
{"h": 5, "w": 4, "x": 8, "y": 32},
topk_n=6,
unit="h",
decimals=1,
sort_order="asc",
limit=None,
title_size=11,
value_size=11,
merge_for_global_sort=True,
links=link_to("atlas-jobs"),
description=(
"Top 6 most recent Jenkins successes by age (newest first). "
@ -2223,18 +2299,16 @@ def build_overview():
)
)
panels.append(
jenkins_weather_statlist_panel(
jenkins_weather_statlist_topk_panel(
243,
"Jenkins Last Failure (h, newest first)",
JENKINS_BUILD_WEATHER_LAST_FAILURE_TOP6_AGE_HOURS_BY_JOB,
JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS_BY_JOB,
{"h": 5, "w": 4, "x": 12, "y": 32},
topk_n=6,
unit="h",
decimals=1,
sort_order="asc",
limit=None,
title_size=11,
value_size=11,
merge_for_global_sort=True,
links=link_to("atlas-jobs"),
description=(
"Top 6 most recent Jenkins failures by age (newest first). "

128
services/monitoring/dashboards/atlas-overview.json
View File

@ -2298,26 +2298,7 @@
"targets": [
{
"refId": "A",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "B",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "C",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "D",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"expr": "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))",
"instant": true
}
],
@ -2326,6 +2307,7 @@
"unit": "h",
"decimals": 1,
"min": 0,
"displayName": "${__field.labels.weather_icon} ${__field.labels.exported_job}",
"links": [
{
"title": "Open Jenkins job",
@ -2337,8 +2319,8 @@
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "A"
"id": "byRegexp",
"options": ".*run_state=\"ok\".*"
},
"properties": [
{
@ -2352,38 +2334,8 @@
},
{
"matcher": {
"id": "byFrameRefID",
"options": "B"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "C"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "D"
"id": "byRegexp",
"options": ".*run_state=\"bad\".*"
},
"properties": [
{
@ -2417,10 +2369,6 @@
}
},
"transformations": [
{
"id": "merge",
"options": {}
},
{
"id": "sortBy",
"options": {
@ -2457,26 +2405,7 @@
"targets": [
{
"refId": "A",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "B",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "C",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "D",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"expr": "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))",
"instant": true
}
],
@ -2485,6 +2414,7 @@
"unit": "h",
"decimals": 1,
"min": 0,
"displayName": "${__field.labels.weather_icon} ${__field.labels.exported_job}",
"links": [
{
"title": "Open Jenkins job",
@ -2496,8 +2426,8 @@
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "A"
"id": "byRegexp",
"options": ".*run_state=\"ok\".*"
},
"properties": [
{
@ -2511,38 +2441,8 @@
},
{
"matcher": {
"id": "byFrameRefID",
"options": "B"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "C"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "D"
"id": "byRegexp",
"options": ".*run_state=\"bad\".*"
},
"properties": [
{
@ -2576,10 +2476,6 @@
}
},
"transformations": [
{
"id": "merge",
"options": {}
},
{
"id": "sortBy",
"options": {

128
services/monitoring/grafana-dashboard-overview.yaml
View File

@ -2307,26 +2307,7 @@ data:
"targets": [
{
"refId": "A",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "B",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "C",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "D",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"expr": "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))",
"instant": true
}
],
@ -2335,6 +2316,7 @@ data:
"unit": "h",
"decimals": 1,
"min": 0,
"displayName": "${__field.labels.weather_icon} ${__field.labels.exported_job}",
"links": [
{
"title": "Open Jenkins job",
@ -2346,8 +2328,8 @@ data:
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "A"
"id": "byRegexp",
"options": ".*run_state=\"ok\".*"
},
"properties": [
{
@ -2361,38 +2343,8 @@ data:
},
{
"matcher": {
"id": "byFrameRefID",
"options": "B"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "C"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "D"
"id": "byRegexp",
"options": ".*run_state=\"bad\".*"
},
"properties": [
{
@ -2426,10 +2378,6 @@ data:
}
},
"transformations": [
{
"id": "merge",
"options": {}
},
{
"id": "sortBy",
"options": {
@ -2466,26 +2414,7 @@ data:
"targets": [
{
"refId": "A",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "B",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "C",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true
},
{
"refId": "D",
"expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}",
"expr": "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))",
"instant": true
}
],
@ -2494,6 +2423,7 @@ data:
"unit": "h",
"decimals": 1,
"min": 0,
"displayName": "${__field.labels.weather_icon} ${__field.labels.exported_job}",
"links": [
{
"title": "Open Jenkins job",
@ -2505,8 +2435,8 @@ data:
"overrides": [
{
"matcher": {
"id": "byFrameRefID",
"options": "A"
"id": "byRegexp",
"options": ".*run_state=\"ok\".*"
},
"properties": [
{
@ -2520,38 +2450,8 @@ data:
},
{
"matcher": {
"id": "byFrameRefID",
"options": "B"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "C"
},
"properties": [
{
"id": "color",
"value": {
"mode": "fixed",
"fixedColor": "red"
}
}
]
},
{
"matcher": {
"id": "byFrameRefID",
"options": "D"
"id": "byRegexp",
"options": ".*run_state=\"bad\".*"
},
"properties": [
{
@ -2585,10 +2485,6 @@ data:
}
},
"transformations": [
{
"id": "merge",
"options": {}
},
{
"id": "sortBy",
"options": {