bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitoring(jenkins): dedupe weather metrics and cap newest list rows

Browse Source
This commit is contained in:
Brad Stein 2026-04-13 14:29:44 -03:00
parent cf988e361b
commit 3bbd0a6f90
5 changed files with 36 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
scripts/dashboards_render_atlas.py
View File

@ -496,11 +496,20 @@ JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS = (
JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS = ( JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS = (
"(time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600" "(time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600"
) )
JENKINS_BUILD_WEATHER_LAST_STATUS_BY_JOB = (
f"max by (exported_job,job_url,weather_icon) ({JENKINS_BUILD_WEATHER_LAST_STATUS})"
)
JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS_BY_JOB = (
f"min by (exported_job,job_url,weather_icon) ({JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS})"
)
JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS_BY_JOB = (
f"min by (exported_job,job_url,weather_icon) ({JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS})"
)
JENKINS_BUILD_WEATHER_LAST_SUCCESS_NEWEST_6_HOURS = ( JENKINS_BUILD_WEATHER_LAST_SUCCESS_NEWEST_6_HOURS = (
f"sort(bottomk(6, {JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS}))" f"sort(bottomk(6, {JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS_BY_JOB}))"
) )
JENKINS_BUILD_WEATHER_LAST_FAILURE_NEWEST_6_HOURS = ( JENKINS_BUILD_WEATHER_LAST_FAILURE_NEWEST_6_HOURS = (
f"sort(bottomk(6, {JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS}))" f"sort(bottomk(6, {JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS_BY_JOB}))"
) )
JENKINS_BUILD_WEATHER_LAST_DURATION_MINUTES = ( JENKINS_BUILD_WEATHER_LAST_DURATION_MINUTES = (
"ariadne_jenkins_build_weather_job_last_duration_seconds / 60" "ariadne_jenkins_build_weather_job_last_duration_seconds / 60"
@ -1394,7 +1403,7 @@ def bargauge_panel(
def _jenkins_weather_status_expr(base_expr, comparator): def _jenkins_weather_status_expr(base_expr, comparator):
return ( return (
f"({base_expr}) and on(exported_job,job_url,weather_icon) " f"({base_expr}) and on(exported_job,job_url,weather_icon) "
f"({JENKINS_BUILD_WEATHER_LAST_STATUS} {comparator})" f"({JENKINS_BUILD_WEATHER_LAST_STATUS_BY_JOB} {comparator})"
) )

8
services/monitoring/dashboards/atlas-jobs.json
View File

@ -1551,25 +1551,25 @@
"targets": [ "targets": [
{ {
"refId": "A", "refId": "A",
"expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "B", "refId": "B",
"expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "C", "refId": "C",
"expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "D", "refId": "D",
"expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
} }

16
services/monitoring/dashboards/atlas-overview.json
View File

@ -2298,25 +2298,25 @@
"targets": [ "targets": [
{ {
"refId": "A", "refId": "A",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "B", "refId": "B",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "C", "refId": "C",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "D", "refId": "D",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
} }
@ -2453,25 +2453,25 @@
"targets": [ "targets": [
{ {
"refId": "A", "refId": "A",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "B", "refId": "B",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "C", "refId": "C",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "D", "refId": "D",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
} }

8
services/monitoring/grafana-dashboard-jobs.yaml
View File

@ -1560,25 +1560,25 @@ data:
"targets": [ "targets": [
{ {
"refId": "A", "refId": "A",
"expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "B", "refId": "B",
"expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "C", "refId": "C",
"expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "D", "refId": "D",
"expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
} }

16
services/monitoring/grafana-dashboard-overview.yaml
View File

@ -2307,25 +2307,25 @@ data:
"targets": [ "targets": [
{ {
"refId": "A", "refId": "A",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "B", "refId": "B",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "C", "refId": "C",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "D", "refId": "D",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
} }
@ -2462,25 +2462,25 @@ data:
"targets": [ "targets": [
{ {
"refId": "A", "refId": "A",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "B", "refId": "B",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "C", "refId": "C",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
}, },
{ {
"refId": "D", "refId": "D",
"expr": "(sort(bottomk(6, (time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600))) and on(exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)",
"legendFormat": "{{weather_icon}} {{exported_job}}", "legendFormat": "{{weather_icon}} {{exported_job}}",
"instant": true "instant": true
} }