diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index b7687d13..9d8ec7b3 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -505,12 +505,6 @@ JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS_BY_JOB = ( JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS_BY_JOB = ( f"min by (exported_job,job_url,weather_icon) ({JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS})" ) -JENKINS_BUILD_WEATHER_LAST_SUCCESS_TOP6_AGE_HOURS_BY_JOB = ( - f"sort(bottomk(6, {JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS_BY_JOB}))" -) -JENKINS_BUILD_WEATHER_LAST_FAILURE_TOP6_AGE_HOURS_BY_JOB = ( - f"sort(bottomk(6, {JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS_BY_JOB}))" -) JENKINS_BUILD_WEATHER_LAST_DURATION_MINUTES = ( "ariadne_jenkins_build_weather_job_last_duration_seconds / 60" ) @@ -1407,6 +1401,25 @@ def _jenkins_weather_status_expr(base_expr, comparator): ) +def _jenkins_weather_topk_expr(base_expr, topk_n=6): + return f"sort(bottomk({topk_n}, {base_expr}))" + + +def _jenkins_weather_topk_with_status_label_expr(base_expr, topk_n=6): + topk_expr = _jenkins_weather_topk_expr(base_expr, topk_n=topk_n) + success_expr = ( + f'label_replace(({topk_expr}) and on(exported_job,job_url,weather_icon) ' + f'({JENKINS_BUILD_WEATHER_LAST_STATUS_BY_JOB} == 1), ' + '"run_state", "ok", "exported_job", ".*")' + ) + failure_expr = ( + f'label_replace(({topk_expr}) and on(exported_job,job_url,weather_icon) ' + f'({JENKINS_BUILD_WEATHER_LAST_STATUS_BY_JOB} != 1), ' + '"run_state", "bad", "exported_job", ".*")' + ) + return f"sort(({success_expr}) or ({failure_expr}))" + + def jenkins_weather_bargauge_panel( panel_id, title, @@ -1526,7 +1539,6 @@ def jenkins_weather_statlist_panel( limit=12, title_size=12, value_size=12, - merge_for_global_sort=False, links=None, description=None, ): @@ -1604,14 +1616,8 @@ def jenkins_weather_statlist_panel( "textMode": "name_and_value", "text": {"titleSize": title_size, "valueSize": value_size}, }, - "transformations": [], + "transformations": [{"id": "sortBy", "options": {"fields": ["Value"], "order": sort_order}}], } - if merge_for_global_sort: - # Merge per-status query frames so sortBy orders all rows globally. - panel["transformations"].append({"id": "merge", "options": {}}) - panel["transformations"].append( - {"id": "sortBy", "options": {"fields": ["Value"], "order": sort_order}} - ) if limit: panel["transformations"].append({"id": "limit", "options": {"limit": limit}}) if links: @@ -1621,6 +1627,78 @@ def jenkins_weather_statlist_panel( return panel +def jenkins_weather_statlist_topk_panel( + panel_id, + title, + base_expr, + grid, + *, + topk_n=6, + unit="h", + decimals=1, + title_size=11, + value_size=11, + links=None, + description=None, +): + expr = _jenkins_weather_topk_with_status_label_expr(base_expr, topk_n=topk_n) + panel = { + "id": panel_id, + "type": "stat", + "title": title, + "datasource": PROM_DS, + "gridPos": grid, + "targets": [ + { + "refId": "A", + "expr": expr, + "instant": True, + } + ], + "fieldConfig": { + "defaults": { + "unit": unit, + "decimals": decimals, + "min": 0, + "displayName": "${__field.labels.weather_icon} ${__field.labels.exported_job}", + "links": [ + { + "title": "Open Jenkins job", + "url": "https://ci.bstein.dev/job/${__field.labels.exported_job}/", + "targetBlank": True, + } + ], + }, + "overrides": [ + { + "matcher": {"id": "byRegexp", "options": '.*run_state="ok".*'}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}], + }, + { + "matcher": {"id": "byRegexp", "options": '.*run_state="bad".*'}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}], + }, + ], + }, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "left", + "orientation": "horizontal", + "wideLayout": True, + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + "textMode": "name_and_value", + "text": {"titleSize": title_size, "valueSize": value_size}, + }, + "transformations": [{"id": "sortBy", "options": {"fields": ["Value"], "order": "asc"}}], + } + if links: + panel["links"] = links + if description: + panel["description"] = description + return panel + + def text_panel(panel_id, title, content, grid): return { "id": panel_id, @@ -2202,18 +2280,16 @@ def build_overview(): ) panels.append(test_success) panels.append( - jenkins_weather_statlist_panel( + jenkins_weather_statlist_topk_panel( 142, "Jenkins Last Success (h, newest first)", - JENKINS_BUILD_WEATHER_LAST_SUCCESS_TOP6_AGE_HOURS_BY_JOB, + JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS_BY_JOB, {"h": 5, "w": 4, "x": 8, "y": 32}, + topk_n=6, unit="h", decimals=1, - sort_order="asc", - limit=None, title_size=11, value_size=11, - merge_for_global_sort=True, links=link_to("atlas-jobs"), description=( "Top 6 most recent Jenkins successes by age (newest first). " @@ -2223,18 +2299,16 @@ def build_overview(): ) ) panels.append( - jenkins_weather_statlist_panel( + jenkins_weather_statlist_topk_panel( 243, "Jenkins Last Failure (h, newest first)", - JENKINS_BUILD_WEATHER_LAST_FAILURE_TOP6_AGE_HOURS_BY_JOB, + JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS_BY_JOB, {"h": 5, "w": 4, "x": 12, "y": 32}, + topk_n=6, unit="h", decimals=1, - sort_order="asc", - limit=None, title_size=11, value_size=11, - merge_for_global_sort=True, links=link_to("atlas-jobs"), description=( "Top 6 most recent Jenkins failures by age (newest first). " diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index 62586dde..ae1c747e 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -2298,26 +2298,7 @@ "targets": [ { "refId": "A", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "B", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "C", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "D", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)", - "legendFormat": "{{weather_icon}} {{exported_job}}", + "expr": "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))", "instant": true } ], @@ -2326,6 +2307,7 @@ "unit": "h", "decimals": 1, "min": 0, + "displayName": "${__field.labels.weather_icon} ${__field.labels.exported_job}", "links": [ { "title": "Open Jenkins job", @@ -2337,8 +2319,8 @@ "overrides": [ { "matcher": { - "id": "byFrameRefID", - "options": "A" + "id": "byRegexp", + "options": ".*run_state=\"ok\".*" }, "properties": [ { @@ -2352,38 +2334,8 @@ }, { "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "red" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "red" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "D" + "id": "byRegexp", + "options": ".*run_state=\"bad\".*" }, "properties": [ { @@ -2417,10 +2369,6 @@ } }, "transformations": [ - { - "id": "merge", - "options": {} - }, { "id": "sortBy", "options": { @@ -2457,26 +2405,7 @@ "targets": [ { "refId": "A", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "B", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "C", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "D", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)", - "legendFormat": "{{weather_icon}} {{exported_job}}", + "expr": "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))", "instant": true } ], @@ -2485,6 +2414,7 @@ "unit": "h", "decimals": 1, "min": 0, + "displayName": "${__field.labels.weather_icon} ${__field.labels.exported_job}", "links": [ { "title": "Open Jenkins job", @@ -2496,8 +2426,8 @@ "overrides": [ { "matcher": { - "id": "byFrameRefID", - "options": "A" + "id": "byRegexp", + "options": ".*run_state=\"ok\".*" }, "properties": [ { @@ -2511,38 +2441,8 @@ }, { "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "red" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "red" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "D" + "id": "byRegexp", + "options": ".*run_state=\"bad\".*" }, "properties": [ { @@ -2576,10 +2476,6 @@ } }, "transformations": [ - { - "id": "merge", - "options": {} - }, { "id": "sortBy", "options": { diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index 9fd8263f..45e2f4be 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -2307,26 +2307,7 @@ data: "targets": [ { "refId": "A", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "B", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "C", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "D", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)", - "legendFormat": "{{weather_icon}} {{exported_job}}", + "expr": "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))", "instant": true } ], @@ -2335,6 +2316,7 @@ data: "unit": "h", "decimals": 1, "min": 0, + "displayName": "${__field.labels.weather_icon} ${__field.labels.exported_job}", "links": [ { "title": "Open Jenkins job", @@ -2346,8 +2328,8 @@ data: "overrides": [ { "matcher": { - "id": "byFrameRefID", - "options": "A" + "id": "byRegexp", + "options": ".*run_state=\"ok\".*" }, "properties": [ { @@ -2361,38 +2343,8 @@ data: }, { "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "red" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "red" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "D" + "id": "byRegexp", + "options": ".*run_state=\"bad\".*" }, "properties": [ { @@ -2426,10 +2378,6 @@ data: } }, "transformations": [ - { - "id": "merge", - "options": {} - }, { "id": "sortBy", "options": { @@ -2466,26 +2414,7 @@ data: "targets": [ { "refId": "A", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "B", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 0)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "C", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 2)", - "legendFormat": "{{weather_icon}} {{exported_job}}", - "instant": true - }, - { - "refId": "D", - "expr": "(sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) < 0)", - "legendFormat": "{{weather_icon}} {{exported_job}}", + "expr": "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))", "instant": true } ], @@ -2494,6 +2423,7 @@ data: "unit": "h", "decimals": 1, "min": 0, + "displayName": "${__field.labels.weather_icon} ${__field.labels.exported_job}", "links": [ { "title": "Open Jenkins job", @@ -2505,8 +2435,8 @@ data: "overrides": [ { "matcher": { - "id": "byFrameRefID", - "options": "A" + "id": "byRegexp", + "options": ".*run_state=\"ok\".*" }, "properties": [ { @@ -2520,38 +2450,8 @@ data: }, { "matcher": { - "id": "byFrameRefID", - "options": "B" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "red" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "C" - }, - "properties": [ - { - "id": "color", - "value": { - "mode": "fixed", - "fixedColor": "red" - } - } - ] - }, - { - "matcher": { - "id": "byFrameRefID", - "options": "D" + "id": "byRegexp", + "options": ".*run_state=\"bad\".*" }, "properties": [ { @@ -2585,10 +2485,6 @@ data: } }, "transformations": [ - { - "id": "merge", - "options": {} - }, { "id": "sortBy", "options": {