From c409c7ca806c172219a63c292bf1a29e96bf3ff5 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Mon, 13 Apr 2026 00:25:33 -0300 Subject: [PATCH] monitoring(jobs): add jenkins build weather job list panels --- scripts/dashboards_render_atlas.py | 272 ++++++--- services/maintenance/ariadne-deployment.yaml | 6 + .../monitoring/dashboards/atlas-jobs.json | 515 +++++++++++++++++- .../monitoring/dashboards/atlas-overview.json | 186 +++++-- .../monitoring/grafana-dashboard-jobs.yaml | 515 +++++++++++++++++- .../grafana-dashboard-overview.yaml | 186 +++++-- 6 files changed, 1489 insertions(+), 191 deletions(-) diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index e6a7447b..a1065078 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -486,6 +486,19 @@ JENKINS_CLEANUP_LAST_SUCCESS_AGE_HOURS = ( ) JENKINS_CLEANUP_LAST_DELETED = "ariadne_jenkins_workspace_cleanup_last_deleted_total or on() vector(0)" JENKINS_CLEANUP_LAST_PLANNED = "ariadne_jenkins_workspace_cleanup_last_planned_total or on() vector(0)" +JENKINS_BUILD_WEATHER_LAST_STATUS = "ariadne_jenkins_build_weather_job_last_status" +JENKINS_BUILD_WEATHER_LAST_RUN_AGE_HOURS = ( + "(time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600" +) +JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS = ( + "(time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600" +) +JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS = ( + "(time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600" +) +JENKINS_BUILD_WEATHER_LAST_DURATION_MINUTES = ( + "ariadne_jenkins_build_weather_job_last_duration_seconds / 60" +) JENKINS_WORKSPACE_PV_STALE_COUNT = ( 'sum((kube_persistentvolume_status_phase{phase=~"Released|Failed"} > bool 0) ' '* on(persistentvolume) group_left(claim_namespace,name) ' @@ -1335,6 +1348,121 @@ def bargauge_panel( return panel +def _jenkins_weather_status_expr(base_expr, comparator): + return ( + f"({base_expr}) and on(job,job_url,weather_icon) " + f"({JENKINS_BUILD_WEATHER_LAST_STATUS} {comparator})" + ) + + +def jenkins_weather_bargauge_panel( + panel_id, + title, + expr, + grid, + *, + unit="h", + decimals=2, + sort_order="asc", + limit=12, + thresholds=None, + links=None, + description=None, +): + panel = { + "id": panel_id, + "type": "bargauge", + "title": title, + "datasource": PROM_DS, + "gridPos": grid, + "targets": [ + { + "refId": "A", + "expr": _jenkins_weather_status_expr(expr, "== 1"), + "legendFormat": "{{weather_icon}} {{job}}", + "instant": True, + }, + { + "refId": "B", + "expr": _jenkins_weather_status_expr(expr, "== 0"), + "legendFormat": "{{weather_icon}} {{job}}", + "instant": True, + }, + { + "refId": "C", + "expr": _jenkins_weather_status_expr(expr, "== 2"), + "legendFormat": "{{weather_icon}} {{job}}", + "instant": True, + }, + { + "refId": "D", + "expr": _jenkins_weather_status_expr(expr, "< 0"), + "legendFormat": "{{weather_icon}} {{job}}", + "instant": True, + }, + ], + "fieldConfig": { + "defaults": { + "unit": unit, + "min": 0, + "thresholds": thresholds + or { + "mode": "absolute", + "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 6}, + {"color": "orange", "value": 24}, + {"color": "red", "value": 72}, + ], + }, + "links": [ + { + "title": "Open Jenkins job", + "url": "${__field.labels.job_url}", + "targetBlank": True, + } + ], + }, + "overrides": [ + { + "matcher": {"id": "byFrameRefID", "options": "A"}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}], + }, + { + "matcher": {"id": "byFrameRefID", "options": "B"}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}], + }, + { + "matcher": {"id": "byFrameRefID", "options": "C"}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "yellow"}}], + }, + { + "matcher": {"id": "byFrameRefID", "options": "D"}, + "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "gray"}}], + }, + ], + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, + }, + "transformations": [ + {"id": "merge", "options": {}}, + {"id": "sortBy", "options": {"fields": ["Value"], "order": sort_order}}, + ], + } + if decimals is not None: + panel["fieldConfig"]["defaults"]["decimals"] = decimals + if limit: + panel["transformations"].append({"id": "limit", "options": {"limit": limit}}) + if links: + panel["links"] = links + if description: + panel["description"] = description + return panel + + def text_panel(panel_id, title, content, grid): return { "id": panel_id, @@ -1578,33 +1706,6 @@ def build_overview(): {"color": "green", "value": 98}, ], } - jenkins_weather_mapping = [ - { - "type": "range", - "options": { - "from": 95, - "to": 101, - "result": {"text": "☀️ Sunny"}, - }, - }, - { - "type": "range", - "options": { - "from": 80, - "to": 95, - "result": {"text": "⛅ Cloudy"}, - }, - }, - { - "type": "range", - "options": { - "from": 0, - "to": 80, - "result": {"text": "⛈️ Storm"}, - }, - }, - ] - panels.append( canvas_metric_grid_panel( 40, @@ -1910,50 +2011,22 @@ def build_overview(): "Oldest successful backup age in hours by PVC. PVCs with missing or unhealthy backup state are forced to 999h so the red bars stay visible in the overview." ) panels.append( - { - "id": 142, - "type": "stat", - "title": "Jenkins Build Weather (24h)", - "datasource": PROM_DS, - "gridPos": {"h": 5, "w": 8, "x": 8, "y": 32}, - "targets": [ - { - "expr": PLATFORM_TEST_SUCCESS_RATE_24H_BY_SUITE, - "refId": "A", - "legendFormat": "{{suite}}", - "instant": True, - } - ], - "fieldConfig": { - "defaults": { - "color": {"mode": "thresholds"}, - "mappings": jenkins_weather_mapping, - "thresholds": { - "mode": "absolute", - "steps": [ - {"color": "red", "value": None}, - {"color": "orange", "value": 80}, - {"color": "green", "value": 95}, - ], - }, - "unit": "percent", - "decimals": 1, - "custom": {"displayMode": "auto"}, - }, - "overrides": [], - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False}, - "textMode": "name_and_value", - "orientation": "horizontal", - "wideLayout": True, - }, - "links": link_to("atlas-jobs"), - "description": "24h build health by suite: sunny (>=95%), cloudy (80-95%), storm (<80%).", - } + jenkins_weather_bargauge_panel( + 142, + "Jenkins Build Weather (last run h, newest first)", + JENKINS_BUILD_WEATHER_LAST_RUN_AGE_HOURS, + {"h": 5, "w": 8, "x": 8, "y": 32}, + unit="h", + decimals=2, + sort_order="asc", + limit=8, + thresholds=age_thresholds, + links=link_to("atlas-jobs"), + description=( + "Live Jenkins job weather from Ariadne: icon + name, green/red status color, " + "and last-run age in hours. Click a job bar to open Jenkins." + ), + ) ) panels.append( @@ -3565,11 +3638,64 @@ def build_jobs_dashboard(): legend_placement="right", ) ) + panels.append( + jenkins_weather_bargauge_panel( + 24, + "Jenkins Build Weather (last run h, newest first)", + JENKINS_BUILD_WEATHER_LAST_RUN_AGE_HOURS, + {"h": 8, "w": 8, "x": 0, "y": 44}, + unit="h", + decimals=2, + sort_order="asc", + limit=20, + thresholds=age_thresholds, + description=( + "Jenkins homepage-style weather list: bars are color-coded by last build " + "status, icons mirror job weather, and values show age since the last run." + ), + ) + ) + panels.append( + jenkins_weather_bargauge_panel( + 25, + "Jenkins Last Success (hours ago)", + JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS, + {"h": 8, "w": 8, "x": 8, "y": 44}, + unit="h", + decimals=2, + sort_order="asc", + limit=20, + thresholds=old_age_thresholds, + description="Per-job age since the most recent successful run.", + ) + ) + panels.append( + jenkins_weather_bargauge_panel( + 26, + "Jenkins Last Duration (minutes)", + JENKINS_BUILD_WEATHER_LAST_DURATION_MINUTES, + {"h": 8, "w": 8, "x": 16, "y": 44}, + unit="m", + decimals=2, + sort_order="desc", + limit=20, + thresholds={ + "mode": "absolute", + "steps": [ + {"color": "green", "value": None}, + {"color": "yellow", "value": 5}, + {"color": "orange", "value": 15}, + {"color": "red", "value": 30}, + ], + }, + description="Most recent completed build duration per Jenkins job.", + ) + ) stale_volume_panel = bargauge_panel( - 24, + 27, "Jenkins Workspace PV Age (h, detached only)", JENKINS_WORKSPACE_PV_STALE_AGE_HOURS, - {"h": 10, "w": 24, "x": 0, "y": 44}, + {"h": 10, "w": 24, "x": 0, "y": 52}, unit="h", instant=True, legend="{{name}} -> {{persistentvolume}}", diff --git a/services/maintenance/ariadne-deployment.yaml b/services/maintenance/ariadne-deployment.yaml index 99a9ec2d..9c830761 100644 --- a/services/maintenance/ariadne-deployment.yaml +++ b/services/maintenance/ariadne-deployment.yaml @@ -345,6 +345,12 @@ spec: value: "15" - name: ARIADNE_SCHEDULE_METIS_SENTINEL_WATCH value: "*/30 * * * *" + - name: ARIADNE_SCHEDULE_JENKINS_BUILD_WEATHER + value: "*/10 * * * *" + - name: JENKINS_BASE_URL + value: https://ci.bstein.dev + - name: JENKINS_API_TIMEOUT_SEC + value: "10" - name: ARIADNE_SCHEDULE_JENKINS_WORKSPACE_CLEANUP value: "45 */6 * * *" - name: JENKINS_WORKSPACE_NAMESPACE diff --git a/services/monitoring/dashboards/atlas-jobs.json b/services/monitoring/dashboards/atlas-jobs.json index ded03d1f..ede00f38 100644 --- a/services/monitoring/dashboards/atlas-jobs.json +++ b/services/monitoring/dashboards/atlas-jobs.json @@ -1537,6 +1537,519 @@ { "id": 24, "type": "bargauge", + "title": "Jenkins Build Weather (last run h, newest first)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 44 + }, + "targets": [ + { + "refId": "A", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "B", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "C", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "D", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "h", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 6 + }, + { + "color": "orange", + "value": 24 + }, + { + "color": "red", + "value": 48 + } + ] + }, + "links": [ + { + "title": "Open Jenkins job", + "url": "${__field.labels.job_url}", + "targetBlank": true + } + ], + "decimals": 2 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "gray" + } + } + ] + } + ] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "asc" + } + }, + { + "id": "limit", + "options": { + "limit": 20 + } + } + ], + "description": "Jenkins homepage-style weather list: bars are color-coded by last build status, icons mirror job weather, and values show age since the last run." + }, + { + "id": 25, + "type": "bargauge", + "title": "Jenkins Last Success (hours ago)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 44 + }, + "targets": [ + { + "refId": "A", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "B", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "C", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "D", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "h", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 24 + }, + { + "color": "orange", + "value": 72 + }, + { + "color": "red", + "value": 168 + } + ] + }, + "links": [ + { + "title": "Open Jenkins job", + "url": "${__field.labels.job_url}", + "targetBlank": true + } + ], + "decimals": 2 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "gray" + } + } + ] + } + ] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "asc" + } + }, + { + "id": "limit", + "options": { + "limit": 20 + } + } + ], + "description": "Per-job age since the most recent successful run." + }, + { + "id": 26, + "type": "bargauge", + "title": "Jenkins Last Duration (minutes)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 44 + }, + "targets": [ + { + "refId": "A", + "expr": "(ariadne_jenkins_build_weather_job_last_duration_seconds / 60) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "B", + "expr": "(ariadne_jenkins_build_weather_job_last_duration_seconds / 60) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "C", + "expr": "(ariadne_jenkins_build_weather_job_last_duration_seconds / 60) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "D", + "expr": "(ariadne_jenkins_build_weather_job_last_duration_seconds / 60) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "m", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "orange", + "value": 15 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "links": [ + { + "title": "Open Jenkins job", + "url": "${__field.labels.job_url}", + "targetBlank": true + } + ], + "decimals": 2 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "gray" + } + } + ] + } + ] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "desc" + } + }, + { + "id": "limit", + "options": { + "limit": 20 + } + } + ], + "description": "Most recent completed build duration per Jenkins job." + }, + { + "id": 27, + "type": "bargauge", "title": "Jenkins Workspace PV Age (h, detached only)", "datasource": { "type": "prometheus", @@ -1546,7 +2059,7 @@ "h": 10, "w": 24, "x": 0, - "y": 44 + "y": 52 }, "targets": [ { diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index af363aef..1de28e57 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -2830,8 +2830,8 @@ }, { "id": 142, - "type": "stat", - "title": "Jenkins Build Weather (24h)", + "type": "bargauge", + "title": "Jenkins Build Weather (last run h, newest first)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -2844,89 +2844,159 @@ }, "targets": [ { - "expr": "sort_desc((100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\",status=~\"ok|passed|success\"}[24h]))) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\"}[24h]))), 1)) and on(suite) ((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\"}[24h]))) > 0))", "refId": "A", - "legendFormat": "{{suite}}", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "B", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "C", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "D", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", + "legendFormat": "{{weather_icon}} {{job}}", "instant": true } ], "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "range", - "options": { - "from": 95, - "to": 101, - "result": { - "text": "\u2600\ufe0f Sunny" - } - } - }, - { - "type": "range", - "options": { - "from": 80, - "to": 95, - "result": { - "text": "\u26c5 Cloudy" - } - } - }, - { - "type": "range", - "options": { - "from": 0, - "to": 80, - "result": { - "text": "\u26c8\ufe0f Storm" - } - } - } - ], + "unit": "h", + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "red", + "color": "green", "value": null }, { - "color": "orange", - "value": 80 + "color": "yellow", + "value": 6 }, { - "color": "green", - "value": 95 + "color": "orange", + "value": 24 + }, + { + "color": "red", + "value": 48 } ] }, - "unit": "percent", - "decimals": 1, - "custom": { - "displayMode": "auto" - } + "links": [ + { + "title": "Open Jenkins job", + "url": "${__field.labels.job_url}", + "targetBlank": true + } + ], + "decimals": 2 }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "gray" + } + } + ] + } + ] }, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", + "displayMode": "basic", + "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false - }, - "textMode": "name_and_value", - "orientation": "horizontal", - "wideLayout": true + } }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "asc" + } + }, + { + "id": "limit", + "options": { + "limit": 8 + } + } + ], "links": [ { "title": "Open atlas-jobs dashboard", @@ -2934,7 +3004,7 @@ "targetBlank": true } ], - "description": "24h build health by suite: sunny (>=95%), cloudy (80-95%), storm (<80%)." + "description": "Live Jenkins job weather from Ariadne: icon + name, green/red status color, and last-run age in hours. Click a job bar to open Jenkins." }, { "id": 30, diff --git a/services/monitoring/grafana-dashboard-jobs.yaml b/services/monitoring/grafana-dashboard-jobs.yaml index 51794165..bf4ec8c1 100644 --- a/services/monitoring/grafana-dashboard-jobs.yaml +++ b/services/monitoring/grafana-dashboard-jobs.yaml @@ -1546,6 +1546,519 @@ data: { "id": 24, "type": "bargauge", + "title": "Jenkins Build Weather (last run h, newest first)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 44 + }, + "targets": [ + { + "refId": "A", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "B", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "C", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "D", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "h", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 6 + }, + { + "color": "orange", + "value": 24 + }, + { + "color": "red", + "value": 48 + } + ] + }, + "links": [ + { + "title": "Open Jenkins job", + "url": "${__field.labels.job_url}", + "targetBlank": true + } + ], + "decimals": 2 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "gray" + } + } + ] + } + ] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "asc" + } + }, + { + "id": "limit", + "options": { + "limit": 20 + } + } + ], + "description": "Jenkins homepage-style weather list: bars are color-coded by last build status, icons mirror job weather, and values show age since the last run." + }, + { + "id": 25, + "type": "bargauge", + "title": "Jenkins Last Success (hours ago)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 44 + }, + "targets": [ + { + "refId": "A", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "B", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "C", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "D", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "h", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 24 + }, + { + "color": "orange", + "value": 72 + }, + { + "color": "red", + "value": 168 + } + ] + }, + "links": [ + { + "title": "Open Jenkins job", + "url": "${__field.labels.job_url}", + "targetBlank": true + } + ], + "decimals": 2 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "gray" + } + } + ] + } + ] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "asc" + } + }, + { + "id": "limit", + "options": { + "limit": 20 + } + } + ], + "description": "Per-job age since the most recent successful run." + }, + { + "id": 26, + "type": "bargauge", + "title": "Jenkins Last Duration (minutes)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 44 + }, + "targets": [ + { + "refId": "A", + "expr": "(ariadne_jenkins_build_weather_job_last_duration_seconds / 60) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "B", + "expr": "(ariadne_jenkins_build_weather_job_last_duration_seconds / 60) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "C", + "expr": "(ariadne_jenkins_build_weather_job_last_duration_seconds / 60) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "D", + "expr": "(ariadne_jenkins_build_weather_job_last_duration_seconds / 60) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "m", + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "orange", + "value": 15 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "links": [ + { + "title": "Open Jenkins job", + "url": "${__field.labels.job_url}", + "targetBlank": true + } + ], + "decimals": 2 + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "gray" + } + } + ] + } + ] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "desc" + } + }, + { + "id": "limit", + "options": { + "limit": 20 + } + } + ], + "description": "Most recent completed build duration per Jenkins job." + }, + { + "id": 27, + "type": "bargauge", "title": "Jenkins Workspace PV Age (h, detached only)", "datasource": { "type": "prometheus", @@ -1555,7 +2068,7 @@ data: "h": 10, "w": 24, "x": 0, - "y": 44 + "y": 52 }, "targets": [ { diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index c8d14c7f..319e8206 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -2839,8 +2839,8 @@ data: }, { "id": 142, - "type": "stat", - "title": "Jenkins Build Weather (24h)", + "type": "bargauge", + "title": "Jenkins Build Weather (last run h, newest first)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -2853,89 +2853,159 @@ data: }, "targets": [ { - "expr": "sort_desc((100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\",status=~\"ok|passed|success\"}[24h]))) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\"}[24h]))), 1)) and on(suite) ((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\"}[24h]))) > 0))", "refId": "A", - "legendFormat": "{{suite}}", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 1)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "B", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 0)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "C", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status == 2)", + "legendFormat": "{{weather_icon}} {{job}}", + "instant": true + }, + { + "refId": "D", + "expr": "((time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600) and on(job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status < 0)", + "legendFormat": "{{weather_icon}} {{job}}", "instant": true } ], "fieldConfig": { "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "type": "range", - "options": { - "from": 95, - "to": 101, - "result": { - "text": "\u2600\ufe0f Sunny" - } - } - }, - { - "type": "range", - "options": { - "from": 80, - "to": 95, - "result": { - "text": "\u26c5 Cloudy" - } - } - }, - { - "type": "range", - "options": { - "from": 0, - "to": 80, - "result": { - "text": "\u26c8\ufe0f Storm" - } - } - } - ], + "unit": "h", + "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "red", + "color": "green", "value": null }, { - "color": "orange", - "value": 80 + "color": "yellow", + "value": 6 }, { - "color": "green", - "value": 95 + "color": "orange", + "value": 24 + }, + { + "color": "red", + "value": 48 } ] }, - "unit": "percent", - "decimals": 1, - "custom": { - "displayMode": "auto" - } + "links": [ + { + "title": "Open Jenkins job", + "url": "${__field.labels.job_url}", + "targetBlank": true + } + ], + "decimals": 2 }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "gray" + } + } + ] + } + ] }, "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", + "displayMode": "basic", + "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false - }, - "textMode": "name_and_value", - "orientation": "horizontal", - "wideLayout": true + } }, + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "asc" + } + }, + { + "id": "limit", + "options": { + "limit": 8 + } + } + ], "links": [ { "title": "Open atlas-jobs dashboard", @@ -2943,7 +3013,7 @@ data: "targetBlank": true } ], - "description": "24h build health by suite: sunny (>=95%), cloudy (80-95%), storm (<80%)." + "description": "Live Jenkins job weather from Ariadne: icon + name, green/red status color, and last-run age in hours. Click a job bar to open Jenkins." }, { "id": 30,