diff --git a/scripts/render/dashboards_render_atlas.py b/scripts/render/dashboards_render_atlas.py index 4be4f6b0c..631f3b578 100644 --- a/scripts/render/dashboards_render_atlas.py +++ b/scripts/render/dashboards_render_atlas.py @@ -16,7 +16,7 @@ from pathlib import Path # Paths, folders, and shared metadata # --------------------------------------------------------------------------- -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] DASHBOARD_DIR = ROOT / "services" / "monitoring" / "dashboards" CONFIG_TEMPLATE = textwrap.dedent( """# {relative_path} @@ -5048,11 +5048,100 @@ def build_jobs_dashboard(): } +HERMES_TRIAGE_OPEN_EXPR = ( + 'sum(ariadne_hermes_triage_incident{status="human_required"})' +) +HERMES_TRIAGE_RESOLVED_EXPR = ( + 'sum(increase(ariadne_hermes_triage_action_total{result="success"}[24h]))' +) +HERMES_TRIAGE_DIAGNOSIS_EXPR = ( + 'ariadne_hermes_triage_duration_seconds{phase="diagnosis"}' +) +HERMES_TRIAGE_ACTIONS_EXPR = ( + "sum by (action, result) " + "(increase(ariadne_hermes_triage_action_total[1h]))" +) +HERMES_TRIAGE_INCIDENTS_EXPR = ( + "sum by (jenkins_job, status) (ariadne_hermes_triage_incident) > 0" +) + + +def _hermes_triage_panels(): + """Return the Hermes automated-triage panels for the testing dashboard. + + Inputs: none. Outputs: a list of Grafana panel definitions covering open + escalations, recent automated actions, diagnosis latency and per-job + incident state, sourced from Ariadne's bounded triage metrics. + """ + + return [ + stat_panel( + 600, + "Triage Escalations Awaiting a Human", + HERMES_TRIAGE_OPEN_EXPR, + {"h": 4, "w": 6, "x": 0, "y": 100}, + instant=True, + description=( + "Incidents Hermes diagnosed where Ariadne refused to act " + "automatically. Each one has a Gitea issue when its job is " + "mapped, and fires HermesTriageHumanRequired." + ), + ), + stat_panel( + 601, + "Automated Actions Succeeded (24h)", + HERMES_TRIAGE_RESOLVED_EXPR, + {"h": 4, "w": 6, "x": 6, "y": 100}, + instant=True, + description=( + "Allowlisted actions Ariadne executed and completed: fixture " + "repair, transient-infra retry, or a pushed patch proposal." + ), + ), + stat_panel( + 602, + "Hermes Diagnosis Time (s)", + HERMES_TRIAGE_DIAGNOSIS_EXPR, + {"h": 4, "w": 6, "x": 12, "y": 100}, + unit="s", + decimals=1, + instant=True, + description=( + "Wall-clock time of the most recent Hermes Agent run. This is " + "the pause between a red build and a diagnosis." + ), + ), + timeseries_panel( + 603, + "Triage Actions by Result (1h rate)", + HERMES_TRIAGE_ACTIONS_EXPR, + {"h": 8, "w": 12, "x": 0, "y": 104}, + legend="{{action}} · {{result}}", + description=( + "requested/accepted/rejected/success/failed per action id. A " + "rejected action means an authorization gate refused it." + ), + ), + timeseries_panel( + 604, + "Incident State by Job", + HERMES_TRIAGE_INCIDENTS_EXPR, + {"h": 8, "w": 12, "x": 12, "y": 104}, + legend="{{jenkins_job}} · {{status}}", + description=( + "Lifecycle of each incident: detected, diagnosed, repairing, " + "awaiting_rebuild, resolved, human_required or failed." + ), + ), + ] + + def build_testing_dashboard(): dashboard = build_jobs_dashboard() dashboard["uid"] = "atlas-testing" dashboard["folderUid"] = PUBLIC_DASHBOARD_FOLDER dashboard["editable"] = False + dashboard["panels"] = list(dashboard["panels"]) + _hermes_triage_panels() return dashboard diff --git a/services/monitoring/dashboards/atlas-testing.json b/services/monitoring/dashboards/atlas-testing.json index d4d6638a2..94eec73ae 100644 --- a/services/monitoring/dashboards/atlas-testing.json +++ b/services/monitoring/dashboards/atlas-testing.json @@ -1801,7 +1801,7 @@ }, "gridPos": { "h": 7, - "w": 6, + "w": 8, "x": 0, "y": 36 }, @@ -1881,8 +1881,8 @@ }, "gridPos": { "h": 7, - "w": 6, - "x": 6, + "w": 8, + "x": 8, "y": 36 }, "targets": [ @@ -1953,86 +1953,6 @@ { "id": 136, "type": "state-timeline", - "title": "Semgrep Failure Rate", - "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 36 - }, - "targets": [ - { - "expr": "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))", - "refId": "A", - "legendFormat": "{{suite}}", - "format": "time_series", - "instant": false, - "range": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "unit": "percent", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-blue", - "value": null - }, - { - "color": "dark-green", - "value": 0.01 - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-orange", - "value": 3 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": true - }, - "min": 0, - "max": 100 - }, - "overrides": [] - }, - "options": { - "mergeValues": true, - "showValue": "never", - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "timeFrom": "7d" - }, - { - "id": 137, - "type": "state-timeline", "title": "Supply Chain Failure Rate", "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", "datasource": { @@ -2041,8 +1961,8 @@ }, "gridPos": { "h": 7, - "w": 6, - "x": 18, + "w": 8, + "x": 16, "y": 36 }, "targets": [ @@ -2126,7 +2046,7 @@ "collapsed": true, "panels": [ { - "id": 160, + "id": 138, "type": "state-timeline", "title": "Tests Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2206,7 +2126,7 @@ "timeFrom": "7d" }, { - "id": 161, + "id": 139, "type": "state-timeline", "title": "Coverage Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2286,7 +2206,7 @@ "timeFrom": "7d" }, { - "id": 162, + "id": 140, "type": "state-timeline", "title": "LOC Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2366,7 +2286,7 @@ "timeFrom": "7d" }, { - "id": 163, + "id": 141, "type": "state-timeline", "title": "Style Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2446,7 +2366,7 @@ "timeFrom": "7d" }, { - "id": 164, + "id": 142, "type": "state-timeline", "title": "Gate Glue Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2456,7 +2376,7 @@ }, "gridPos": { "h": 7, - "w": 6, + "w": 8, "x": 0, "y": 50 }, @@ -2526,7 +2446,7 @@ "timeFrom": "7d" }, { - "id": 165, + "id": 143, "type": "state-timeline", "title": "SonarQube Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2536,8 +2456,8 @@ }, "gridPos": { "h": 7, - "w": 6, - "x": 6, + "w": 8, + "x": 8, "y": 50 }, "targets": [ @@ -2606,87 +2526,7 @@ "timeFrom": "7d" }, { - "id": 166, - "type": "state-timeline", - "title": "Semgrep Healthy Rate", - "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 50 - }, - "targets": [ - { - "expr": "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))", - "refId": "A", - "legendFormat": "{{suite}}", - "format": "time_series", - "instant": false, - "range": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "unit": "percent", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-red", - "value": null - }, - { - "color": "dark-orange", - "value": 90 - }, - { - "color": "dark-yellow", - "value": 93 - }, - { - "color": "dark-green", - "value": 95 - }, - { - "color": "dark-blue", - "value": 100 - } - ] - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": true - }, - "min": 0, - "max": 100 - }, - "overrides": [] - }, - "options": { - "mergeValues": true, - "showValue": "never", - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "timeFrom": "7d" - }, - { - "id": 167, + "id": 144, "type": "state-timeline", "title": "Supply Chain Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2696,8 +2536,8 @@ }, "gridPos": { "h": 7, - "w": 6, - "x": 18, + "w": 8, + "x": 16, "y": 50 }, "targets": [ @@ -4805,450 +4645,267 @@ "description": "SonarQube availability, projects, fetch errors, and gate status." }, { - "id": 506, - "type": "row", - "title": "Public ZAP Baseline", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 + "id": 600, + "type": "stat", + "title": "Triage Escalations Awaiting a Human", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" }, - "collapsed": true, - "panels": [ + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 100 + }, + "targets": [ { - "id": 168, - "type": "stat", - "title": "ZAP Targets Scanned", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 117 - }, - "targets": [ - { - "expr": "(count(max by (host) (platform_zap_baseline_last_run_timestamp_seconds{exported_job=\"platform-security-zap\"})) or on() vector(0))", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-green", - "value": null - }, - { - "color": "dark-red", - "value": 1 - } - ] - }, - "unit": "none", - "custom": { - "displayMode": "auto" - } - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Number of public hostnames with a latest OWASP ZAP baseline scan sample." - }, - { - "id": 169, - "type": "stat", - "title": "ZAP High/Medium Alerts", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 117 - }, - "targets": [ - { - "expr": "(sum(platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk=~\"high|medium\"}) or on() vector(0))", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-blue", - "value": null - }, - { - "color": "dark-green", - "value": 0.01 - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-orange", - "value": 3 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "unit": "none", - "custom": { - "displayMode": "auto" - } - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Current high and medium ZAP alert instance count across public targets." - }, - { - "id": 170, - "type": "stat", - "title": "ZAP Scan Errors", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 117 - }, - "targets": [ - { - "expr": "(sum(platform_zap_baseline_scan_status{exported_job=\"platform-security-zap\",status=~\"fail|error\"}) or on() vector(0))", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-blue", - "value": null - }, - { - "color": "dark-green", - "value": 0.01 - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-orange", - "value": 3 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "unit": "none", - "custom": { - "displayMode": "auto" - } - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Targets whose latest ZAP baseline failed or did not produce a usable report." - }, - { - "id": 171, - "type": "piechart", - "title": "ZAP Alerts by Risk", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 117 - }, - "targets": [ - { - "expr": "sum by (risk) (platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk!=\"\"}) or on() vector(0)", - "refId": "A", - "legendFormat": "{{risk}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percent", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "right" - }, - "pieType": "pie", - "displayLabels": [], - "tooltip": { - "mode": "single" - }, - "colorScheme": "interpolateSpectral", - "colorBy": "value", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "description": "Current ZAP alert distribution by risk level." - }, - { - "id": 172, - "type": "state-timeline", - "title": "ZAP Target Health", - "description": "Latest OWASP ZAP passive baseline health per public hostname. High alerts and scan errors drop a target to zero; medium/low alerts mark warning health.", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 117 - }, - "targets": [ - { - "expr": "(max by (host) (platform_zap_baseline_target_health_percent{exported_job=\"platform-security-zap\"}) or on() vector(0))", - "refId": "A", - "legendFormat": "{{host}}", - "format": "time_series", - "instant": false, - "range": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "unit": "percent", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-red", - "value": null - }, - { - "color": "dark-orange", - "value": 90 - }, - { - "color": "dark-yellow", - "value": 93 - }, - { - "color": "dark-green", - "value": 95 - }, - { - "color": "dark-blue", - "value": 100 - } - ] - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": true - }, - "min": 0, - "max": 100 - }, - "overrides": [] - }, - "options": { - "mergeValues": true, - "showValue": "never", - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "timeFrom": "7d" - }, - { - "id": 173, - "type": "bargauge", - "title": "ZAP Alerts by Host", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 123 - }, - "targets": [ - { - "expr": "sort_desc(sum by (host, risk) (platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk=~\"high|medium|low\"}) or on() vector(0))", - "refId": "A", - "legendFormat": "{{host}} \u00b7 {{risk}}", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "unit": "none", - "min": 0, - "max": null, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-blue", - "value": null - }, - { - "color": "dark-green", - "value": 0.01 - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-orange", - "value": 3 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "displayMode": "basic", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "transformations": [ - { - "id": "sortBy", - "options": { - "fields": [ - "Value" - ], - "order": "desc" - } - }, - { - "id": "limit", - "options": { - "limit": 20 - } - } - ], - "description": "Current high, medium, and low ZAP alert counts by public hostname." + "expr": "sum(ariadne_hermes_triage_incident{status=\"human_required\"})", + "refId": "A", + "instant": true } ], - "description": "OWASP ZAP passive baseline results for externally addressed Atlas domains." + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Incidents Hermes diagnosed where Ariadne refused to act automatically. Each one has a Gitea issue when its job is mapped, and fires HermesTriageHumanRequired." + }, + { + "id": 601, + "type": "stat", + "title": "Automated Actions Succeeded (24h)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 100 + }, + "targets": [ + { + "expr": "sum(increase(ariadne_hermes_triage_action_total{result=\"success\"}[24h]))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Allowlisted actions Ariadne executed and completed: fixture repair, transient-infra retry, or a pushed patch proposal." + }, + { + "id": 602, + "type": "stat", + "title": "Hermes Diagnosis Time (s)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 100 + }, + "targets": [ + { + "expr": "ariadne_hermes_triage_duration_seconds{phase=\"diagnosis\"}", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Wall-clock time of the most recent Hermes Agent run. This is the pause between a red build and a diagnosis." + }, + { + "id": 603, + "type": "timeseries", + "title": "Triage Actions by Result (1h rate)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 104 + }, + "targets": [ + { + "expr": "sum by (action, result) (increase(ariadne_hermes_triage_action_total[1h]))", + "refId": "A", + "legendFormat": "{{action}} \u00b7 {{result}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "requested/accepted/rejected/success/failed per action id. A rejected action means an authorization gate refused it." + }, + { + "id": 604, + "type": "timeseries", + "title": "Incident State by Job", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 104 + }, + "targets": [ + { + "expr": "sum by (jenkins_job, status) (ariadne_hermes_triage_incident) > 0", + "refId": "A", + "legendFormat": "{{jenkins_job}} \u00b7 {{status}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "Lifecycle of each incident: detected, diagnosed, repairing, awaiting_rebuild, resolved, human_required or failed." } ], "time": { diff --git a/services/monitoring/grafana-dashboard-testing.yaml b/services/monitoring/grafana-dashboard-testing.yaml index 5eef4e1d8..c5d1369f0 100644 --- a/services/monitoring/grafana-dashboard-testing.yaml +++ b/services/monitoring/grafana-dashboard-testing.yaml @@ -1810,7 +1810,7 @@ data: }, "gridPos": { "h": 7, - "w": 6, + "w": 8, "x": 0, "y": 36 }, @@ -1890,8 +1890,8 @@ data: }, "gridPos": { "h": 7, - "w": 6, - "x": 6, + "w": 8, + "x": 8, "y": 36 }, "targets": [ @@ -1962,86 +1962,6 @@ data: { "id": 136, "type": "state-timeline", - "title": "Semgrep Failure Rate", - "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 36 - }, - "targets": [ - { - "expr": "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))", - "refId": "A", - "legendFormat": "{{suite}}", - "format": "time_series", - "instant": false, - "range": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "unit": "percent", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-blue", - "value": null - }, - { - "color": "dark-green", - "value": 0.01 - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-orange", - "value": 3 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": true - }, - "min": 0, - "max": 100 - }, - "overrides": [] - }, - "options": { - "mergeValues": true, - "showValue": "never", - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "timeFrom": "7d" - }, - { - "id": 137, - "type": "state-timeline", "title": "Supply Chain Failure Rate", "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", "datasource": { @@ -2050,8 +1970,8 @@ data: }, "gridPos": { "h": 7, - "w": 6, - "x": 18, + "w": 8, + "x": 16, "y": 36 }, "targets": [ @@ -2135,7 +2055,7 @@ data: "collapsed": true, "panels": [ { - "id": 160, + "id": 138, "type": "state-timeline", "title": "Tests Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2215,7 +2135,7 @@ data: "timeFrom": "7d" }, { - "id": 161, + "id": 139, "type": "state-timeline", "title": "Coverage Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2295,7 +2215,7 @@ data: "timeFrom": "7d" }, { - "id": 162, + "id": 140, "type": "state-timeline", "title": "LOC Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2375,7 +2295,7 @@ data: "timeFrom": "7d" }, { - "id": 163, + "id": 141, "type": "state-timeline", "title": "Style Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2455,7 +2375,7 @@ data: "timeFrom": "7d" }, { - "id": 164, + "id": 142, "type": "state-timeline", "title": "Gate Glue Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2465,7 +2385,7 @@ data: }, "gridPos": { "h": 7, - "w": 6, + "w": 8, "x": 0, "y": 50 }, @@ -2535,7 +2455,7 @@ data: "timeFrom": "7d" }, { - "id": 165, + "id": 143, "type": "state-timeline", "title": "SonarQube Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2545,8 +2465,8 @@ data: }, "gridPos": { "h": 7, - "w": 6, - "x": 6, + "w": 8, + "x": 8, "y": 50 }, "targets": [ @@ -2615,87 +2535,7 @@ data: "timeFrom": "7d" }, { - "id": 166, - "type": "state-timeline", - "title": "Semgrep Healthy Rate", - "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 50 - }, - "targets": [ - { - "expr": "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))", - "refId": "A", - "legendFormat": "{{suite}}", - "format": "time_series", - "instant": false, - "range": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "unit": "percent", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-red", - "value": null - }, - { - "color": "dark-orange", - "value": 90 - }, - { - "color": "dark-yellow", - "value": 93 - }, - { - "color": "dark-green", - "value": 95 - }, - { - "color": "dark-blue", - "value": 100 - } - ] - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": true - }, - "min": 0, - "max": 100 - }, - "overrides": [] - }, - "options": { - "mergeValues": true, - "showValue": "never", - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "timeFrom": "7d" - }, - { - "id": 167, + "id": 144, "type": "state-timeline", "title": "Supply Chain Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2705,8 +2545,8 @@ data: }, "gridPos": { "h": 7, - "w": 6, - "x": 18, + "w": 8, + "x": 16, "y": 50 }, "targets": [ @@ -4814,450 +4654,267 @@ data: "description": "SonarQube availability, projects, fetch errors, and gate status." }, { - "id": 506, - "type": "row", - "title": "Public ZAP Baseline", - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 + "id": 600, + "type": "stat", + "title": "Triage Escalations Awaiting a Human", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" }, - "collapsed": true, - "panels": [ + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 100 + }, + "targets": [ { - "id": 168, - "type": "stat", - "title": "ZAP Targets Scanned", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 117 - }, - "targets": [ - { - "expr": "(count(max by (host) (platform_zap_baseline_last_run_timestamp_seconds{exported_job=\"platform-security-zap\"})) or on() vector(0))", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-green", - "value": null - }, - { - "color": "dark-red", - "value": 1 - } - ] - }, - "unit": "none", - "custom": { - "displayMode": "auto" - } - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Number of public hostnames with a latest OWASP ZAP baseline scan sample." - }, - { - "id": 169, - "type": "stat", - "title": "ZAP High/Medium Alerts", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 117 - }, - "targets": [ - { - "expr": "(sum(platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk=~\"high|medium\"}) or on() vector(0))", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-blue", - "value": null - }, - { - "color": "dark-green", - "value": 0.01 - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-orange", - "value": 3 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "unit": "none", - "custom": { - "displayMode": "auto" - } - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Current high and medium ZAP alert instance count across public targets." - }, - { - "id": 170, - "type": "stat", - "title": "ZAP Scan Errors", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 117 - }, - "targets": [ - { - "expr": "(sum(platform_zap_baseline_scan_status{exported_job=\"platform-security-zap\",status=~\"fail|error\"}) or on() vector(0))", - "refId": "A", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-blue", - "value": null - }, - { - "color": "dark-green", - "value": 0.01 - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-orange", - "value": 3 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "unit": "none", - "custom": { - "displayMode": "auto" - } - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "value" - }, - "description": "Targets whose latest ZAP baseline failed or did not produce a usable report." - }, - { - "id": 171, - "type": "piechart", - "title": "ZAP Alerts by Risk", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 117 - }, - "targets": [ - { - "expr": "sum by (risk) (platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk!=\"\"}) or on() vector(0)", - "refId": "A", - "legendFormat": "{{risk}}" - } - ], - "fieldConfig": { - "defaults": { - "unit": "percent", - "color": { - "mode": "palette-classic" - } - }, - "overrides": [] - }, - "options": { - "legend": { - "displayMode": "list", - "placement": "right" - }, - "pieType": "pie", - "displayLabels": [], - "tooltip": { - "mode": "single" - }, - "colorScheme": "interpolateSpectral", - "colorBy": "value", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "description": "Current ZAP alert distribution by risk level." - }, - { - "id": 172, - "type": "state-timeline", - "title": "ZAP Target Health", - "description": "Latest OWASP ZAP passive baseline health per public hostname. High alerts and scan errors drop a target to zero; medium/low alerts mark warning health.", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 117 - }, - "targets": [ - { - "expr": "(max by (host) (platform_zap_baseline_target_health_percent{exported_job=\"platform-security-zap\"}) or on() vector(0))", - "refId": "A", - "legendFormat": "{{host}}", - "format": "time_series", - "instant": false, - "range": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "unit": "percent", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-red", - "value": null - }, - { - "color": "dark-orange", - "value": 90 - }, - { - "color": "dark-yellow", - "value": 93 - }, - { - "color": "dark-green", - "value": 95 - }, - { - "color": "dark-blue", - "value": 100 - } - ] - }, - "custom": { - "fillOpacity": 70, - "lineWidth": 0, - "spanNulls": true - }, - "min": 0, - "max": 100 - }, - "overrides": [] - }, - "options": { - "mergeValues": true, - "showValue": "never", - "legend": { - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "timeFrom": "7d" - }, - { - "id": 173, - "type": "bargauge", - "title": "ZAP Alerts by Host", - "datasource": { - "type": "prometheus", - "uid": "atlas-vm" - }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 123 - }, - "targets": [ - { - "expr": "sort_desc(sum by (host, risk) (platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk=~\"high|medium|low\"}) or on() vector(0))", - "refId": "A", - "legendFormat": "{{host}} \u00b7 {{risk}}", - "instant": true - } - ], - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "unit": "none", - "min": 0, - "max": null, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-blue", - "value": null - }, - { - "color": "dark-green", - "value": 0.01 - }, - { - "color": "dark-yellow", - "value": 1 - }, - { - "color": "dark-orange", - "value": 3 - }, - { - "color": "dark-red", - "value": 5 - } - ] - }, - "decimals": 0 - }, - "overrides": [] - }, - "options": { - "displayMode": "basic", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - } - }, - "transformations": [ - { - "id": "sortBy", - "options": { - "fields": [ - "Value" - ], - "order": "desc" - } - }, - { - "id": "limit", - "options": { - "limit": 20 - } - } - ], - "description": "Current high, medium, and low ZAP alert counts by public hostname." + "expr": "sum(ariadne_hermes_triage_incident{status=\"human_required\"})", + "refId": "A", + "instant": true } ], - "description": "OWASP ZAP passive baseline results for externally addressed Atlas domains." + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Incidents Hermes diagnosed where Ariadne refused to act automatically. Each one has a Gitea issue when its job is mapped, and fires HermesTriageHumanRequired." + }, + { + "id": 601, + "type": "stat", + "title": "Automated Actions Succeeded (24h)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 100 + }, + "targets": [ + { + "expr": "sum(increase(ariadne_hermes_triage_action_total{result=\"success\"}[24h]))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Allowlisted actions Ariadne executed and completed: fixture repair, transient-infra retry, or a pushed patch proposal." + }, + { + "id": 602, + "type": "stat", + "title": "Hermes Diagnosis Time (s)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 100 + }, + "targets": [ + { + "expr": "ariadne_hermes_triage_duration_seconds{phase=\"diagnosis\"}", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(115, 115, 115, 1)", + "value": null + }, + { + "color": "dark-green", + "value": 1 + } + ] + }, + "unit": "s", + "custom": { + "displayMode": "auto" + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Wall-clock time of the most recent Hermes Agent run. This is the pause between a red build and a diagnosis." + }, + { + "id": 603, + "type": "timeseries", + "title": "Triage Actions by Result (1h rate)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 104 + }, + "targets": [ + { + "expr": "sum by (action, result) (increase(ariadne_hermes_triage_action_total[1h]))", + "refId": "A", + "legendFormat": "{{action}} \u00b7 {{result}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "requested/accepted/rejected/success/failed per action id. A rejected action means an authorization gate refused it." + }, + { + "id": 604, + "type": "timeseries", + "title": "Incident State by Job", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 104 + }, + "targets": [ + { + "expr": "sum by (jenkins_job, status) (ariadne_hermes_triage_incident) > 0", + "refId": "A", + "legendFormat": "{{jenkins_job}} \u00b7 {{status}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "description": "Lifecycle of each incident: detected, diagnosed, repairing, awaiting_rebuild, resolved, human_required or failed." } ], "time": {