From f02db9801c6f039214639bdb99259097a85a01bf Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 20 Apr 2026 08:35:05 -0300 Subject: [PATCH] monitoring(testing): add per-test metrics and flaky-test panels --- ci/scripts/publish_test_metrics.py | 39 +++ scripts/dashboards_render_atlas.py | 115 +++++++-- .../monitoring/dashboards/atlas-jobs.json | 224 ++++++++++++++++-- .../monitoring/grafana-dashboard-jobs.yaml | 224 ++++++++++++++++-- 4 files changed, 554 insertions(+), 48 deletions(-) diff --git a/ci/scripts/publish_test_metrics.py b/ci/scripts/publish_test_metrics.py index 6822d4cc..83ab707d 100644 --- a/ci/scripts/publish_test_metrics.py +++ b/ci/scripts/publish_test_metrics.py @@ -93,6 +93,36 @@ def _collect_junit_totals(pattern: str) -> dict[str, int]: return totals +def _collect_junit_cases(pattern: str) -> list[tuple[str, str]]: + """Collect individual JUnit test-case statuses for flaky-test trend panels.""" + cases: list[tuple[str, str]] = [] + for path in sorted(glob(pattern)): + if not os.path.exists(path): + continue + root = ET.parse(path).getroot() + suites: list[ET.Element] + if root.tag == "testsuite": + suites = [root] + elif root.tag == "testsuites": + suites = [elem for elem in root if elem.tag == "testsuite"] + else: + suites = [] + for suite in suites: + for test_case in suite.findall("testcase"): + case_name = test_case.attrib.get("name", "").strip() + class_name = test_case.attrib.get("classname", "").strip() + if not case_name: + continue + full_name = f"{class_name}.{case_name}" if class_name else case_name + status = "passed" + if test_case.find("failure") is not None or test_case.find("error") is not None: + status = "failed" + elif test_case.find("skipped") is not None: + status = "skipped" + cases.append((full_name, status)) + return cases + + def _read_exit_code(path: str) -> int: """Read the quality-gate exit code, defaulting to failure if missing.""" try: @@ -293,6 +323,7 @@ def _build_payload( suite: str, status: str, tests: dict[str, int], + test_cases: list[tuple[str, str]], ok_count: int, failed_count: int, branch: str, @@ -337,6 +368,12 @@ def _build_payload( lines.append( f'titan_iac_quality_gate_checks_total{{suite="{suite}",check="{_escape_label(check_name)}",result="{_escape_label(check_status)}"}} 1' ) + if test_cases: + lines.append("# TYPE platform_quality_gate_test_case_result gauge") + for test_name, test_status in test_cases: + lines.append( + f'platform_quality_gate_test_case_result{{suite="{suite}",test="{_escape_label(test_name)}",status="{_escape_label(test_status)}"}} 1' + ) return "\n".join(lines) + "\n" @@ -352,6 +389,7 @@ def main() -> int: build_number = os.getenv("BUILD_NUMBER", "") tests = _collect_junit_totals(junit_glob) + test_cases = _collect_junit_cases(junit_glob) exit_code = _read_exit_code(exit_code_path) status = "ok" if exit_code == 0 else "failed" summary = _load_summary(summary_path) @@ -393,6 +431,7 @@ def main() -> int: suite=suite, status=status, tests=tests, + test_cases=test_cases, ok_count=ok_count, failed_count=failed_count, branch=branch, diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py index b9f903e6..bf3e5701 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/dashboards_render_atlas.py @@ -1108,6 +1108,24 @@ def testing_suite_variable(): } +def testing_case_variable(): + return { + "name": "test", + "label": "Test Case", + "type": "query", + "query": 'label_values(platform_quality_gate_test_case_result{suite=~"${suite:regex}",job="platform-quality-ci"}, test)', + "current": {"text": "All", "value": "$__all", "selected": True}, + "options": [], + "hide": 0, + "multi": False, + "includeAll": True, + "allValue": ".*", + "refresh": 2, + "sort": 1, + "skipUrlSync": False, + } + + def bargauge_panel( panel_id, title, @@ -2951,6 +2969,7 @@ def build_mail_dashboard(): def build_jobs_dashboard(): panels = [] suite_var = "${suite:regex}" + test_var = "${test:regex}" success = PLATFORM_TEST_SUCCESS_STATUS exported = PLATFORM_TEST_EXPORT_FILTER runs_selector = f'suite=~"{suite_var}",{exported}' @@ -2961,6 +2980,7 @@ def build_jobs_dashboard(): coverage_metric_selector = f'__name__=~".*_quality_gate_coverage_percent",suite=~"{suite_var}",{exported}' workspace_coverage_selector = f'suite=~"{suite_var}",{exported}' smell_selector = f'suite=~"{suite_var}",{exported}' + test_case_selector = f'suite=~"{suite_var}",test=~"{test_var}",{exported}' suite_universe = " or ".join( f'label_replace(vector(1), "suite", "{suite}", "__name__", ".*")' @@ -2987,25 +3007,23 @@ def build_jobs_dashboard(): failures_by_suite_24h = ( f'sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_failure_selector}}}[24h]))' ) - success_history_by_suite_core = ( + success_history_by_suite = ( f'100 * (sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_success_selector}}}[$__interval])) ' f'/ clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[$__interval]))), 1))' ) - success_history_by_suite = ( - f'({success_history_by_suite_core}) ' - f'or on(suite) (0 * sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[$__range])))' - ) coverage_by_suite = ( f'(max by (suite) ({{{coverage_metric_selector}}})) ' f'or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{{{workspace_coverage_selector}}}))' ) coverage_with_missing = ( f"({coverage_by_suite}) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[30d]))) - 1)" + ")" ) coverage_gap = f"clamp_min(95 - ({coverage_by_suite}), 0)" smell_by_suite = f'max by (suite) (platform_quality_gate_source_lines_over_500_total{{{smell_selector}}})' smell_with_missing = ( f"({smell_by_suite}) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[30d]))) - 1)" + ")" ) average_coverage = f"(avg(({coverage_by_suite})) or on() vector(0))" suites_loc_violating = f'(sum((({smell_by_suite}) > bool 0)) or on() vector(0))' @@ -3047,6 +3065,30 @@ def build_jobs_dashboard(): ) return f'({core}) or on(suite) (0 * ({suite_universe}))' + problematic_tests_history = ( + f'topk(12, sum by (suite, test) (increase(platform_quality_gate_test_case_result{{suite=~"{suite_var}",status="failed",{exported}}}[$__interval])))' + ) + worst_test_per_suite = ( + f'topk by (suite) (1, sum by (suite, test) (increase(platform_quality_gate_test_case_result{{suite=~"{suite_var}",status="failed",{exported}}}[30d])))' + ) + selected_test_pass_fail = [ + { + "refId": "A", + "expr": f'sum by (suite) (increase(platform_quality_gate_test_case_result{{{test_case_selector},status="passed"}}[$__interval])) or on() vector(0)', + "legendFormat": "passed · {{suite}}", + }, + { + "refId": "B", + "expr": f'sum by (suite) (increase(platform_quality_gate_test_case_result{{{test_case_selector},status="failed"}}[$__interval])) or on() vector(0)', + "legendFormat": "failed · {{suite}}", + }, + { + "refId": "C", + "expr": f'sum by (suite) (increase(platform_quality_gate_test_case_result{{{test_case_selector},status="skipped"}}[$__interval])) or on() vector(0)', + "legendFormat": "skipped · {{suite}}", + }, + ] + missing_tests_by_suite = ( f'(({suite_universe}) unless on(suite) count by (suite) ({{__name__=~".*_quality_gate_tests_total",{exported}}}))' ) @@ -3589,12 +3631,52 @@ def build_jobs_dashboard(): limit=24, ) ) + panels.append( + timeseries_panel( + 145, + "Problematic Tests Over Time (Top failures)", + problematic_tests_history, + {"h": 8, "w": 12, "x": 0, "y": 45}, + unit="none", + legend="{{suite}} · {{test}}", + legend_display="list", + legend_placement="bottom", + legend_calcs=["lastNotNull", "max", "sum"], + ) + ) + panels.append( + timeseries_panel( + 146, + "Selected Test Pass/Fail History", + None, + {"h": 8, "w": 8, "x": 12, "y": 45}, + unit="none", + targets=selected_test_pass_fail, + legend_display="list", + legend_placement="bottom", + legend_calcs=["lastNotNull", "sum"], + ) + ) + panels.append( + bargauge_panel( + 147, + "Most Problematic Test by Suite (30d)", + worst_test_per_suite, + {"h": 8, "w": 4, "x": 20, "y": 45}, + unit="none", + instant=True, + legend="{{suite}} · {{test}}", + sort_order="desc", + thresholds=failures_thresholds, + limit=9, + ) + ) coverage_panel = bargauge_panel( 17, "Coverage by Suite (Latest, gate 95)", coverage_with_missing, - {"h": 8, "w": 12, "x": 0, "y": 45}, + {"h": 8, "w": 12, "x": 0, "y": 53}, unit="percent", instant=True, legend="{{suite}}", @@ -3611,7 +3693,7 @@ def build_jobs_dashboard(): 18, "Files >500 LOC by Suite (Latest)", smell_with_missing, - {"h": 8, "w": 12, "x": 12, "y": 45}, + {"h": 8, "w": 12, "x": 12, "y": 53}, unit="none", instant=True, legend="{{suite}}", @@ -3628,7 +3710,7 @@ def build_jobs_dashboard(): 27, "Missing Tests Metrics by Suite", missing_tests_by_suite, - {"h": 7, "w": 6, "x": 0, "y": 53}, + {"h": 7, "w": 6, "x": 0, "y": 61}, unit="none", instant=True, legend="{{suite}}", @@ -3642,7 +3724,7 @@ def build_jobs_dashboard(): 28, "Missing Checks Metrics by Suite", missing_checks_by_suite, - {"h": 7, "w": 6, "x": 6, "y": 53}, + {"h": 7, "w": 6, "x": 6, "y": 61}, unit="none", instant=True, legend="{{suite}}", @@ -3656,7 +3738,7 @@ def build_jobs_dashboard(): 29, "Missing Coverage Metrics by Suite", missing_coverage_by_suite, - {"h": 7, "w": 6, "x": 12, "y": 53}, + {"h": 7, "w": 6, "x": 12, "y": 61}, unit="none", instant=True, legend="{{suite}}", @@ -3670,7 +3752,7 @@ def build_jobs_dashboard(): 30, "Missing LOC Metrics by Suite", missing_loc_by_suite, - {"h": 7, "w": 6, "x": 18, "y": 53}, + {"h": 7, "w": 6, "x": 18, "y": 61}, unit="none", instant=True, legend="{{suite}}", @@ -3684,7 +3766,7 @@ def build_jobs_dashboard(): 31, "SonarQube API Up", "(max(sonarqube_up) or on() vector(0))", - {"h": 6, "w": 4, "x": 0, "y": 60}, + {"h": 6, "w": 4, "x": 0, "y": 68}, unit="none", instant=True, thresholds={ @@ -3701,7 +3783,7 @@ def build_jobs_dashboard(): 32, "Sonar Projects (Selected)", f'(count(sonarqube_project_quality_gate_pass{{project_key=~"{suite_var}"}}) or on() vector(0))', - {"h": 6, "w": 4, "x": 4, "y": 60}, + {"h": 6, "w": 4, "x": 4, "y": 68}, unit="none", instant=True, thresholds=failures_thresholds, @@ -3712,7 +3794,7 @@ def build_jobs_dashboard(): 33, "Sonar Gate Fetch Errors", "(max(sonarqube_quality_gate_fetch_errors_total) or on() vector(0))", - {"h": 6, "w": 4, "x": 8, "y": 60}, + {"h": 6, "w": 4, "x": 8, "y": 68}, unit="none", instant=True, thresholds=failures_thresholds, @@ -3722,7 +3804,7 @@ def build_jobs_dashboard(): 34, "Sonar Gate Status Mix (Selected)", f'count by (status) (sonarqube_project_quality_gate_pass{{project_key=~"{suite_var}"}})', - {"h": 6, "w": 6, "x": 12, "y": 60}, + {"h": 6, "w": 6, "x": 12, "y": 68}, ) sonar_status_mix_panel["targets"][0]["legendFormat"] = "{{status}}" panels.append(sonar_status_mix_panel) @@ -3731,7 +3813,7 @@ def build_jobs_dashboard(): 35, "Projects Failing Sonar Gate", f'sort_desc(count by (project_key) (sonarqube_project_quality_gate_pass{{project_key=~"{suite_var}",status!~"OK|ok"}}))', - {"h": 6, "w": 6, "x": 18, "y": 60}, + {"h": 6, "w": 6, "x": 18, "y": 68}, unit="none", instant=True, legend="{{project_key}}", @@ -3754,6 +3836,7 @@ def build_jobs_dashboard(): "templating": { "list": [ testing_suite_variable(), + testing_case_variable(), ] }, } diff --git a/services/monitoring/dashboards/atlas-jobs.json b/services/monitoring/dashboards/atlas-jobs.json index 851dbd1b..5511144c 100644 --- a/services/monitoring/dashboards/atlas-jobs.json +++ b/services/monitoring/dashboards/atlas-jobs.json @@ -1210,7 +1210,7 @@ }, "targets": [ { - "expr": "(100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))) or on(suite) (0 * sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__range])))", + "expr": "100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))", "refId": "A", "legendFormat": "{{suite}}" } @@ -2045,9 +2045,9 @@ ] }, { - "id": 17, - "type": "bargauge", - "title": "Coverage by Suite (Latest, gate 95)", + "id": 145, + "type": "timeseries", + "title": "Problematic Tests Over Time (Top failures)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -2060,7 +2060,180 @@ }, "targets": [ { - "expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))", + "expr": "topk(12, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[$__interval])))", + "refId": "A", + "legendFormat": "{{suite}} \u00b7 {{test}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max", + "sum" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 146, + "type": "timeseries", + "title": "Selected Test Pass/Fail History", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 45 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"passed\"}[$__interval])) or on() vector(0)", + "legendFormat": "passed \u00b7 {{suite}}" + }, + { + "refId": "B", + "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"failed\"}[$__interval])) or on() vector(0)", + "legendFormat": "failed \u00b7 {{suite}}" + }, + { + "refId": "C", + "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"skipped\"}[$__interval])) or on() vector(0)", + "legendFormat": "skipped \u00b7 {{suite}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "sum" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 147, + "type": "bargauge", + "title": "Most Problematic Test by Suite (30d)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 45 + }, + "targets": [ + { + "expr": "sort_desc(topk by (suite) (1, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[30d]))))", + "refId": "A", + "legendFormat": "{{suite}} \u00b7 {{test}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "min": 0, + "max": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "orange", + "value": 3 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "desc" + } + }, + { + "id": "limit", + "options": { + "limit": 9 + } + } + ] + }, + { + "id": 17, + "type": "bargauge", + "title": "Coverage by Suite (Latest, gate 95)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "targets": [ + { + "expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))", "refId": "A", "legendFormat": "{{suite}}", "instant": true @@ -2141,11 +2314,11 @@ "h": 8, "w": 12, "x": 12, - "y": 45 + "y": 53 }, "targets": [ { - "expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))", + "expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))", "refId": "A", "legendFormat": "{{suite}}", "instant": true @@ -2229,7 +2402,7 @@ "h": 7, "w": 6, "x": 0, - "y": 53 + "y": 61 }, "targets": [ { @@ -2296,7 +2469,7 @@ "h": 7, "w": 6, "x": 6, - "y": 53 + "y": 61 }, "targets": [ { @@ -2363,7 +2536,7 @@ "h": 7, "w": 6, "x": 12, - "y": 53 + "y": 61 }, "targets": [ { @@ -2430,7 +2603,7 @@ "h": 7, "w": 6, "x": 18, - "y": 53 + "y": 61 }, "targets": [ { @@ -2497,7 +2670,7 @@ "h": 6, "w": 4, "x": 0, - "y": 60 + "y": 68 }, "targets": [ { @@ -2558,7 +2731,7 @@ "h": 6, "w": 4, "x": 4, - "y": 60 + "y": 68 }, "targets": [ { @@ -2627,7 +2800,7 @@ "h": 6, "w": 4, "x": 8, - "y": 60 + "y": 68 }, "targets": [ { @@ -2696,7 +2869,7 @@ "h": 6, "w": 6, "x": 12, - "y": 60 + "y": 68 }, "targets": [ { @@ -2747,7 +2920,7 @@ "h": 6, "w": 6, "x": 18, - "y": 60 + "y": 68 }, "targets": [ { @@ -2891,6 +3064,25 @@ "refresh": 1, "sort": 1, "skipUrlSync": false + }, + { + "name": "test", + "label": "Test Case", + "type": "query", + "query": "label_values(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",job=\"platform-quality-ci\"}, test)", + "current": { + "text": "All", + "value": "$__all", + "selected": true + }, + "options": [], + "hide": 0, + "multi": false, + "includeAll": true, + "allValue": ".*", + "refresh": 2, + "sort": 1, + "skipUrlSync": false } ] } diff --git a/services/monitoring/grafana-dashboard-jobs.yaml b/services/monitoring/grafana-dashboard-jobs.yaml index d2925c10..b1d73b9d 100644 --- a/services/monitoring/grafana-dashboard-jobs.yaml +++ b/services/monitoring/grafana-dashboard-jobs.yaml @@ -1219,7 +1219,7 @@ data: }, "targets": [ { - "expr": "(100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))) or on(suite) (0 * sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__range])))", + "expr": "100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))", "refId": "A", "legendFormat": "{{suite}}" } @@ -2054,9 +2054,9 @@ data: ] }, { - "id": 17, - "type": "bargauge", - "title": "Coverage by Suite (Latest, gate 95)", + "id": 145, + "type": "timeseries", + "title": "Problematic Tests Over Time (Top failures)", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -2069,7 +2069,180 @@ data: }, "targets": [ { - "expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))", + "expr": "topk(12, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[$__interval])))", + "refId": "A", + "legendFormat": "{{suite}} \u00b7 {{test}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max", + "sum" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 146, + "type": "timeseries", + "title": "Selected Test Pass/Fail History", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 12, + "y": 45 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"passed\"}[$__interval])) or on() vector(0)", + "legendFormat": "passed \u00b7 {{suite}}" + }, + { + "refId": "B", + "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"failed\"}[$__interval])) or on() vector(0)", + "legendFormat": "failed \u00b7 {{suite}}" + }, + { + "refId": "C", + "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"skipped\"}[$__interval])) or on() vector(0)", + "legendFormat": "skipped \u00b7 {{suite}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "sum" + ] + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "id": 147, + "type": "bargauge", + "title": "Most Problematic Test by Suite (30d)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 45 + }, + "targets": [ + { + "expr": "sort_desc(topk by (suite) (1, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[30d]))))", + "refId": "A", + "legendFormat": "{{suite}} \u00b7 {{test}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "min": 0, + "max": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "orange", + "value": 3 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "desc" + } + }, + { + "id": "limit", + "options": { + "limit": 9 + } + } + ] + }, + { + "id": 17, + "type": "bargauge", + "title": "Coverage by Suite (Latest, gate 95)", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "targets": [ + { + "expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))", "refId": "A", "legendFormat": "{{suite}}", "instant": true @@ -2150,11 +2323,11 @@ data: "h": 8, "w": 12, "x": 12, - "y": 45 + "y": 53 }, "targets": [ { - "expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))", + "expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))", "refId": "A", "legendFormat": "{{suite}}", "instant": true @@ -2238,7 +2411,7 @@ data: "h": 7, "w": 6, "x": 0, - "y": 53 + "y": 61 }, "targets": [ { @@ -2305,7 +2478,7 @@ data: "h": 7, "w": 6, "x": 6, - "y": 53 + "y": 61 }, "targets": [ { @@ -2372,7 +2545,7 @@ data: "h": 7, "w": 6, "x": 12, - "y": 53 + "y": 61 }, "targets": [ { @@ -2439,7 +2612,7 @@ data: "h": 7, "w": 6, "x": 18, - "y": 53 + "y": 61 }, "targets": [ { @@ -2506,7 +2679,7 @@ data: "h": 6, "w": 4, "x": 0, - "y": 60 + "y": 68 }, "targets": [ { @@ -2567,7 +2740,7 @@ data: "h": 6, "w": 4, "x": 4, - "y": 60 + "y": 68 }, "targets": [ { @@ -2636,7 +2809,7 @@ data: "h": 6, "w": 4, "x": 8, - "y": 60 + "y": 68 }, "targets": [ { @@ -2705,7 +2878,7 @@ data: "h": 6, "w": 6, "x": 12, - "y": 60 + "y": 68 }, "targets": [ { @@ -2756,7 +2929,7 @@ data: "h": 6, "w": 6, "x": 18, - "y": 60 + "y": 68 }, "targets": [ { @@ -2900,6 +3073,25 @@ data: "refresh": 1, "sort": 1, "skipUrlSync": false + }, + { + "name": "test", + "label": "Test Case", + "type": "query", + "query": "label_values(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",job=\"platform-quality-ci\"}, test)", + "current": { + "text": "All", + "value": "$__all", + "selected": true + }, + "options": [], + "hide": 0, + "multi": false, + "includeAll": true, + "allValue": ".*", + "refresh": 2, + "sort": 1, + "skipUrlSync": false } ] }