monitoring(testing): add per-test metrics and flaky-test panels

ci/scripts/publish_test_metrics.py

@@ -93,6 +93,36 @@ def _collect_junit_totals(pattern: str) -> dict[str, int]:
     return totals
 
 
+def _collect_junit_cases(pattern: str) -> list[tuple[str, str]]:
+    """Collect individual JUnit test-case statuses for flaky-test trend panels."""
+    cases: list[tuple[str, str]] = []
+    for path in sorted(glob(pattern)):
+        if not os.path.exists(path):
+            continue
+        root = ET.parse(path).getroot()
+        suites: list[ET.Element]
+        if root.tag == "testsuite":
+            suites = [root]
+        elif root.tag == "testsuites":
+            suites = [elem for elem in root if elem.tag == "testsuite"]
+        else:
+            suites = []
+        for suite in suites:
+            for test_case in suite.findall("testcase"):
+                case_name = test_case.attrib.get("name", "").strip()
+                class_name = test_case.attrib.get("classname", "").strip()
+                if not case_name:
+                    continue
+                full_name = f"{class_name}.{case_name}" if class_name else case_name
+                status = "passed"
+                if test_case.find("failure") is not None or test_case.find("error") is not None:
+                    status = "failed"
+                elif test_case.find("skipped") is not None:
+                    status = "skipped"
+                cases.append((full_name, status))
+    return cases
+
+
 def _read_exit_code(path: str) -> int:
     """Read the quality-gate exit code, defaulting to failure if missing."""
     try:
@@ -293,6 +323,7 @@ def _build_payload(
     suite: str,
     status: str,
     tests: dict[str, int],
+    test_cases: list[tuple[str, str]],
     ok_count: int,
     failed_count: int,
     branch: str,
@@ -337,6 +368,12 @@ def _build_payload(
             lines.append(
                 f'titan_iac_quality_gate_checks_total{{suite="{suite}",check="{_escape_label(check_name)}",result="{_escape_label(check_status)}"}} 1'
             )
+    if test_cases:
+        lines.append("# TYPE platform_quality_gate_test_case_result gauge")
+        for test_name, test_status in test_cases:
+            lines.append(
+                f'platform_quality_gate_test_case_result{{suite="{suite}",test="{_escape_label(test_name)}",status="{_escape_label(test_status)}"}} 1'
+            )
     return "\n".join(lines) + "\n"
 
 
@@ -352,6 +389,7 @@ def main() -> int:
     build_number = os.getenv("BUILD_NUMBER", "")
 
     tests = _collect_junit_totals(junit_glob)
+    test_cases = _collect_junit_cases(junit_glob)
     exit_code = _read_exit_code(exit_code_path)
     status = "ok" if exit_code == 0 else "failed"
     summary = _load_summary(summary_path)
@@ -393,6 +431,7 @@ def main() -> int:
         suite=suite,
         status=status,
         tests=tests,
+        test_cases=test_cases,
         ok_count=ok_count,
         failed_count=failed_count,
         branch=branch,

scripts/dashboards_render_atlas.py

@@ -1108,6 +1108,24 @@ def testing_suite_variable():
     }
 
 
+def testing_case_variable():
+    return {
+        "name": "test",
+        "label": "Test Case",
+        "type": "query",
+        "query": 'label_values(platform_quality_gate_test_case_result{suite=~"${suite:regex}",job="platform-quality-ci"}, test)',
+        "current": {"text": "All", "value": "$__all", "selected": True},
+        "options": [],
+        "hide": 0,
+        "multi": False,
+        "includeAll": True,
+        "allValue": ".*",
+        "refresh": 2,
+        "sort": 1,
+        "skipUrlSync": False,
+    }
+
+
 def bargauge_panel(
     panel_id,
     title,
@@ -2951,6 +2969,7 @@ def build_mail_dashboard():
 def build_jobs_dashboard():
     panels = []
     suite_var = "${suite:regex}"
+    test_var = "${test:regex}"
     success = PLATFORM_TEST_SUCCESS_STATUS
     exported = PLATFORM_TEST_EXPORT_FILTER
     runs_selector = f'suite=~"{suite_var}",{exported}'
@@ -2961,6 +2980,7 @@ def build_jobs_dashboard():
     coverage_metric_selector = f'__name__=~".*_quality_gate_coverage_percent",suite=~"{suite_var}",{exported}'
     workspace_coverage_selector = f'suite=~"{suite_var}",{exported}'
     smell_selector = f'suite=~"{suite_var}",{exported}'
+    test_case_selector = f'suite=~"{suite_var}",test=~"{test_var}",{exported}'
 
     suite_universe = " or ".join(
         f'label_replace(vector(1), "suite", "{suite}", "__name__", ".*")'
@@ -2987,25 +3007,23 @@ def build_jobs_dashboard():
     failures_by_suite_24h = (
         f'sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_failure_selector}}}[24h]))'
     )
-    success_history_by_suite_core = (
+    success_history_by_suite = (
         f'100 * (sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_success_selector}}}[$__interval])) '
         f'/ clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[$__interval]))), 1))'
     )
-    success_history_by_suite = (
-        f'({success_history_by_suite_core}) '
-        f'or on(suite) (0 * sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[$__range])))'
-    )
     coverage_by_suite = (
         f'(max by (suite) ({{{coverage_metric_selector}}})) '
         f'or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{{{workspace_coverage_selector}}}))'
     )
     coverage_with_missing = (
         f"({coverage_by_suite}) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[30d]))) - 1)"
+        ")"
     )
     coverage_gap = f"clamp_min(95 - ({coverage_by_suite}), 0)"
     smell_by_suite = f'max by (suite) (platform_quality_gate_source_lines_over_500_total{{{smell_selector}}})'
     smell_with_missing = (
         f"({smell_by_suite}) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[30d]))) - 1)"
+        ")"
     )
     average_coverage = f"(avg(({coverage_by_suite})) or on() vector(0))"
     suites_loc_violating = f'(sum((({smell_by_suite}) > bool 0)) or on() vector(0))'
@@ -3047,6 +3065,30 @@ def build_jobs_dashboard():
         )
         return f'({core}) or on(suite) (0 * ({suite_universe}))'
 
+    problematic_tests_history = (
+        f'topk(12, sum by (suite, test) (increase(platform_quality_gate_test_case_result{{suite=~"{suite_var}",status="failed",{exported}}}[$__interval])))'
+    )
+    worst_test_per_suite = (
+        f'topk by (suite) (1, sum by (suite, test) (increase(platform_quality_gate_test_case_result{{suite=~"{suite_var}",status="failed",{exported}}}[30d])))'
+    )
+    selected_test_pass_fail = [
+        {
+            "refId": "A",
+            "expr": f'sum by (suite) (increase(platform_quality_gate_test_case_result{{{test_case_selector},status="passed"}}[$__interval])) or on() vector(0)',
+            "legendFormat": "passed · {{suite}}",
+        },
+        {
+            "refId": "B",
+            "expr": f'sum by (suite) (increase(platform_quality_gate_test_case_result{{{test_case_selector},status="failed"}}[$__interval])) or on() vector(0)',
+            "legendFormat": "failed · {{suite}}",
+        },
+        {
+            "refId": "C",
+            "expr": f'sum by (suite) (increase(platform_quality_gate_test_case_result{{{test_case_selector},status="skipped"}}[$__interval])) or on() vector(0)',
+            "legendFormat": "skipped · {{suite}}",
+        },
+    ]
+
     missing_tests_by_suite = (
         f'(({suite_universe}) unless on(suite) count by (suite) ({{__name__=~".*_quality_gate_tests_total",{exported}}}))'
     )
@@ -3589,12 +3631,52 @@ def build_jobs_dashboard():
             limit=24,
         )
     )
+    panels.append(
+        timeseries_panel(
+            145,
+            "Problematic Tests Over Time (Top failures)",
+            problematic_tests_history,
+            {"h": 8, "w": 12, "x": 0, "y": 45},
+            unit="none",
+            legend="{{suite}} · {{test}}",
+            legend_display="list",
+            legend_placement="bottom",
+            legend_calcs=["lastNotNull", "max", "sum"],
+        )
+    )
+    panels.append(
+        timeseries_panel(
+            146,
+            "Selected Test Pass/Fail History",
+            None,
+            {"h": 8, "w": 8, "x": 12, "y": 45},
+            unit="none",
+            targets=selected_test_pass_fail,
+            legend_display="list",
+            legend_placement="bottom",
+            legend_calcs=["lastNotNull", "sum"],
+        )
+    )
+    panels.append(
+        bargauge_panel(
+            147,
+            "Most Problematic Test by Suite (30d)",
+            worst_test_per_suite,
+            {"h": 8, "w": 4, "x": 20, "y": 45},
+            unit="none",
+            instant=True,
+            legend="{{suite}} · {{test}}",
+            sort_order="desc",
+            thresholds=failures_thresholds,
+            limit=9,
+        )
+    )
 
     coverage_panel = bargauge_panel(
         17,
         "Coverage by Suite (Latest, gate 95)",
         coverage_with_missing,
-        {"h": 8, "w": 12, "x": 0, "y": 45},
+        {"h": 8, "w": 12, "x": 0, "y": 53},
         unit="percent",
         instant=True,
         legend="{{suite}}",
@@ -3611,7 +3693,7 @@ def build_jobs_dashboard():
         18,
         "Files >500 LOC by Suite (Latest)",
         smell_with_missing,
-        {"h": 8, "w": 12, "x": 12, "y": 45},
+        {"h": 8, "w": 12, "x": 12, "y": 53},
         unit="none",
         instant=True,
         legend="{{suite}}",
@@ -3628,7 +3710,7 @@ def build_jobs_dashboard():
             27,
             "Missing Tests Metrics by Suite",
             missing_tests_by_suite,
-            {"h": 7, "w": 6, "x": 0, "y": 53},
+            {"h": 7, "w": 6, "x": 0, "y": 61},
             unit="none",
             instant=True,
             legend="{{suite}}",
@@ -3642,7 +3724,7 @@ def build_jobs_dashboard():
             28,
             "Missing Checks Metrics by Suite",
             missing_checks_by_suite,
-            {"h": 7, "w": 6, "x": 6, "y": 53},
+            {"h": 7, "w": 6, "x": 6, "y": 61},
             unit="none",
             instant=True,
             legend="{{suite}}",
@@ -3656,7 +3738,7 @@ def build_jobs_dashboard():
             29,
             "Missing Coverage Metrics by Suite",
             missing_coverage_by_suite,
-            {"h": 7, "w": 6, "x": 12, "y": 53},
+            {"h": 7, "w": 6, "x": 12, "y": 61},
             unit="none",
             instant=True,
             legend="{{suite}}",
@@ -3670,7 +3752,7 @@ def build_jobs_dashboard():
             30,
             "Missing LOC Metrics by Suite",
             missing_loc_by_suite,
-            {"h": 7, "w": 6, "x": 18, "y": 53},
+            {"h": 7, "w": 6, "x": 18, "y": 61},
             unit="none",
             instant=True,
             legend="{{suite}}",
@@ -3684,7 +3766,7 @@ def build_jobs_dashboard():
             31,
             "SonarQube API Up",
             "(max(sonarqube_up) or on() vector(0))",
-            {"h": 6, "w": 4, "x": 0, "y": 60},
+            {"h": 6, "w": 4, "x": 0, "y": 68},
             unit="none",
             instant=True,
             thresholds={
@@ -3701,7 +3783,7 @@ def build_jobs_dashboard():
             32,
             "Sonar Projects (Selected)",
             f'(count(sonarqube_project_quality_gate_pass{{project_key=~"{suite_var}"}}) or on() vector(0))',
-            {"h": 6, "w": 4, "x": 4, "y": 60},
+            {"h": 6, "w": 4, "x": 4, "y": 68},
             unit="none",
             instant=True,
             thresholds=failures_thresholds,
@@ -3712,7 +3794,7 @@ def build_jobs_dashboard():
             33,
             "Sonar Gate Fetch Errors",
             "(max(sonarqube_quality_gate_fetch_errors_total) or on() vector(0))",
-            {"h": 6, "w": 4, "x": 8, "y": 60},
+            {"h": 6, "w": 4, "x": 8, "y": 68},
             unit="none",
             instant=True,
             thresholds=failures_thresholds,
@@ -3722,7 +3804,7 @@ def build_jobs_dashboard():
         34,
         "Sonar Gate Status Mix (Selected)",
         f'count by (status) (sonarqube_project_quality_gate_pass{{project_key=~"{suite_var}"}})',
-        {"h": 6, "w": 6, "x": 12, "y": 60},
+        {"h": 6, "w": 6, "x": 12, "y": 68},
     )
     sonar_status_mix_panel["targets"][0]["legendFormat"] = "{{status}}"
     panels.append(sonar_status_mix_panel)
@@ -3731,7 +3813,7 @@ def build_jobs_dashboard():
             35,
             "Projects Failing Sonar Gate",
             f'sort_desc(count by (project_key) (sonarqube_project_quality_gate_pass{{project_key=~"{suite_var}",status!~"OK|ok"}}))',
-            {"h": 6, "w": 6, "x": 18, "y": 60},
+            {"h": 6, "w": 6, "x": 18, "y": 68},
             unit="none",
             instant=True,
             legend="{{project_key}}",
@@ -3754,6 +3836,7 @@ def build_jobs_dashboard():
         "templating": {
             "list": [
                 testing_suite_variable(),
+                testing_case_variable(),
             ]
         },
     }

services/monitoring/dashboards/atlas-jobs.json

@@ -1210,7 +1210,7 @@
       },
       "targets": [
         {
-          "expr": "(100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))) or on(suite) (0 * sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__range])))",
+          "expr": "100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))",
           "refId": "A",
           "legendFormat": "{{suite}}"
         }
@@ -2045,9 +2045,9 @@
       ]
     },
     {
-      "id": 17,
+      "id": 145,
-      "type": "bargauge",
+      "type": "timeseries",
-      "title": "Coverage by Suite (Latest, gate 95)",
+      "title": "Problematic Tests Over Time (Top failures)",
       "datasource": {
         "type": "prometheus",
         "uid": "atlas-vm"
@@ -2060,7 +2060,180 @@
       },
       "targets": [
         {
-          "expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))",
+          "expr": "topk(12, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[$__interval])))",
+          "refId": "A",
+          "legendFormat": "{{suite}} \u00b7 {{test}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull",
+            "max",
+            "sum"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 146,
+      "type": "timeseries",
+      "title": "Selected Test Pass/Fail History",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "atlas-vm"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 12,
+        "y": 45
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"passed\"}[$__interval])) or on() vector(0)",
+          "legendFormat": "passed \u00b7 {{suite}}"
+        },
+        {
+          "refId": "B",
+          "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"failed\"}[$__interval])) or on() vector(0)",
+          "legendFormat": "failed \u00b7 {{suite}}"
+        },
+        {
+          "refId": "C",
+          "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"skipped\"}[$__interval])) or on() vector(0)",
+          "legendFormat": "skipped \u00b7 {{suite}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "calcs": [
+            "lastNotNull",
+            "sum"
+          ]
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      }
+    },
+    {
+      "id": 147,
+      "type": "bargauge",
+      "title": "Most Problematic Test by Suite (30d)",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "atlas-vm"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 4,
+        "x": 20,
+        "y": 45
+      },
+      "targets": [
+        {
+          "expr": "sort_desc(topk by (suite) (1, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[30d]))))",
+          "refId": "A",
+          "legendFormat": "{{suite}} \u00b7 {{test}}",
+          "instant": true
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0,
+          "max": null,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 1
+              },
+              {
+                "color": "orange",
+                "value": 3
+              },
+              {
+                "color": "red",
+                "value": 5
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "displayMode": "gradient",
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        }
+      },
+      "transformations": [
+        {
+          "id": "sortBy",
+          "options": {
+            "fields": [
+              "Value"
+            ],
+            "order": "desc"
+          }
+        },
+        {
+          "id": "limit",
+          "options": {
+            "limit": 9
+          }
+        }
+      ]
+    },
+    {
+      "id": 17,
+      "type": "bargauge",
+      "title": "Coverage by Suite (Latest, gate 95)",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "atlas-vm"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 53
+      },
+      "targets": [
+        {
+          "expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))",
           "refId": "A",
           "legendFormat": "{{suite}}",
           "instant": true
@@ -2141,11 +2314,11 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 45
+        "y": 53
       },
       "targets": [
         {
-          "expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))",
+          "expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))",
           "refId": "A",
           "legendFormat": "{{suite}}",
           "instant": true
@@ -2229,7 +2402,7 @@
         "h": 7,
         "w": 6,
         "x": 0,
-        "y": 53
+        "y": 61
       },
       "targets": [
         {
@@ -2296,7 +2469,7 @@
         "h": 7,
         "w": 6,
         "x": 6,
-        "y": 53
+        "y": 61
       },
       "targets": [
         {
@@ -2363,7 +2536,7 @@
         "h": 7,
         "w": 6,
         "x": 12,
-        "y": 53
+        "y": 61
       },
       "targets": [
         {
@@ -2430,7 +2603,7 @@
         "h": 7,
         "w": 6,
         "x": 18,
-        "y": 53
+        "y": 61
       },
       "targets": [
         {
@@ -2497,7 +2670,7 @@
         "h": 6,
         "w": 4,
         "x": 0,
-        "y": 60
+        "y": 68
       },
       "targets": [
         {
@@ -2558,7 +2731,7 @@
         "h": 6,
         "w": 4,
         "x": 4,
-        "y": 60
+        "y": 68
       },
       "targets": [
         {
@@ -2627,7 +2800,7 @@
         "h": 6,
         "w": 4,
         "x": 8,
-        "y": 60
+        "y": 68
       },
       "targets": [
         {
@@ -2696,7 +2869,7 @@
         "h": 6,
         "w": 6,
         "x": 12,
-        "y": 60
+        "y": 68
       },
       "targets": [
         {
@@ -2747,7 +2920,7 @@
         "h": 6,
         "w": 6,
         "x": 18,
-        "y": 60
+        "y": 68
       },
       "targets": [
         {
@@ -2891,6 +3064,25 @@
         "refresh": 1,
         "sort": 1,
         "skipUrlSync": false
+      },
+      {
+        "name": "test",
+        "label": "Test Case",
+        "type": "query",
+        "query": "label_values(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",job=\"platform-quality-ci\"}, test)",
+        "current": {
+          "text": "All",
+          "value": "$__all",
+          "selected": true
+        },
+        "options": [],
+        "hide": 0,
+        "multi": false,
+        "includeAll": true,
+        "allValue": ".*",
+        "refresh": 2,
+        "sort": 1,
+        "skipUrlSync": false
       }
     ]
   }

services/monitoring/grafana-dashboard-jobs.yaml

@@ -1219,7 +1219,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))) or on(suite) (0 * sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__range])))",
+              "expr": "100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))",
               "refId": "A",
               "legendFormat": "{{suite}}"
             }
@@ -2054,9 +2054,9 @@ data:
           ]
         },
         {
-          "id": 17,
+          "id": 145,
-          "type": "bargauge",
+          "type": "timeseries",
-          "title": "Coverage by Suite (Latest, gate 95)",
+          "title": "Problematic Tests Over Time (Top failures)",
           "datasource": {
             "type": "prometheus",
             "uid": "atlas-vm"
@@ -2069,7 +2069,180 @@ data:
           },
           "targets": [
             {
-              "expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))",
+              "expr": "topk(12, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[$__interval])))",
+              "refId": "A",
+              "legendFormat": "{{suite}} \u00b7 {{test}}"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "options": {
+            "legend": {
+              "displayMode": "list",
+              "placement": "bottom",
+              "calcs": [
+                "lastNotNull",
+                "max",
+                "sum"
+              ]
+            },
+            "tooltip": {
+              "mode": "multi"
+            }
+          }
+        },
+        {
+          "id": 146,
+          "type": "timeseries",
+          "title": "Selected Test Pass/Fail History",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "atlas-vm"
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 8,
+            "x": 12,
+            "y": 45
+          },
+          "targets": [
+            {
+              "refId": "A",
+              "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"passed\"}[$__interval])) or on() vector(0)",
+              "legendFormat": "passed \u00b7 {{suite}}"
+            },
+            {
+              "refId": "B",
+              "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"failed\"}[$__interval])) or on() vector(0)",
+              "legendFormat": "failed \u00b7 {{suite}}"
+            },
+            {
+              "refId": "C",
+              "expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"skipped\"}[$__interval])) or on() vector(0)",
+              "legendFormat": "skipped \u00b7 {{suite}}"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "none"
+            },
+            "overrides": []
+          },
+          "options": {
+            "legend": {
+              "displayMode": "list",
+              "placement": "bottom",
+              "calcs": [
+                "lastNotNull",
+                "sum"
+              ]
+            },
+            "tooltip": {
+              "mode": "multi"
+            }
+          }
+        },
+        {
+          "id": 147,
+          "type": "bargauge",
+          "title": "Most Problematic Test by Suite (30d)",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "atlas-vm"
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 4,
+            "x": 20,
+            "y": 45
+          },
+          "targets": [
+            {
+              "expr": "sort_desc(topk by (suite) (1, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[30d]))))",
+              "refId": "A",
+              "legendFormat": "{{suite}} \u00b7 {{test}}",
+              "instant": true
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "none",
+              "min": 0,
+              "max": null,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  {
+                    "color": "green",
+                    "value": null
+                  },
+                  {
+                    "color": "yellow",
+                    "value": 1
+                  },
+                  {
+                    "color": "orange",
+                    "value": 3
+                  },
+                  {
+                    "color": "red",
+                    "value": 5
+                  }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "options": {
+            "displayMode": "gradient",
+            "orientation": "horizontal",
+            "reduceOptions": {
+              "calcs": [
+                "lastNotNull"
+              ],
+              "fields": "",
+              "values": false
+            }
+          },
+          "transformations": [
+            {
+              "id": "sortBy",
+              "options": {
+                "fields": [
+                  "Value"
+                ],
+                "order": "desc"
+              }
+            },
+            {
+              "id": "limit",
+              "options": {
+                "limit": 9
+              }
+            }
+          ]
+        },
+        {
+          "id": 17,
+          "type": "bargauge",
+          "title": "Coverage by Suite (Latest, gate 95)",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "atlas-vm"
+          },
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 53
+          },
+          "targets": [
+            {
+              "expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))",
               "refId": "A",
               "legendFormat": "{{suite}}",
               "instant": true
@@ -2150,11 +2323,11 @@ data:
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 45
+            "y": 53
           },
           "targets": [
             {
-              "expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))",
+              "expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))",
               "refId": "A",
               "legendFormat": "{{suite}}",
               "instant": true
@@ -2238,7 +2411,7 @@ data:
             "h": 7,
             "w": 6,
             "x": 0,
-            "y": 53
+            "y": 61
           },
           "targets": [
             {
@@ -2305,7 +2478,7 @@ data:
             "h": 7,
             "w": 6,
             "x": 6,
-            "y": 53
+            "y": 61
           },
           "targets": [
             {
@@ -2372,7 +2545,7 @@ data:
             "h": 7,
             "w": 6,
             "x": 12,
-            "y": 53
+            "y": 61
           },
           "targets": [
             {
@@ -2439,7 +2612,7 @@ data:
             "h": 7,
             "w": 6,
             "x": 18,
-            "y": 53
+            "y": 61
           },
           "targets": [
             {
@@ -2506,7 +2679,7 @@ data:
             "h": 6,
             "w": 4,
             "x": 0,
-            "y": 60
+            "y": 68
           },
           "targets": [
             {
@@ -2567,7 +2740,7 @@ data:
             "h": 6,
             "w": 4,
             "x": 4,
-            "y": 60
+            "y": 68
           },
           "targets": [
             {
@@ -2636,7 +2809,7 @@ data:
             "h": 6,
             "w": 4,
             "x": 8,
-            "y": 60
+            "y": 68
           },
           "targets": [
             {
@@ -2705,7 +2878,7 @@ data:
             "h": 6,
             "w": 6,
             "x": 12,
-            "y": 60
+            "y": 68
           },
           "targets": [
             {
@@ -2756,7 +2929,7 @@ data:
             "h": 6,
             "w": 6,
             "x": 18,
-            "y": 60
+            "y": 68
           },
           "targets": [
             {
@@ -2900,6 +3073,25 @@ data:
             "refresh": 1,
             "sort": 1,
             "skipUrlSync": false
+          },
+          {
+            "name": "test",
+            "label": "Test Case",
+            "type": "query",
+            "query": "label_values(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",job=\"platform-quality-ci\"}, test)",
+            "current": {
+              "text": "All",
+              "value": "$__all",
+              "selected": true
+            },
+            "options": [],
+            "hide": 0,
+            "multi": false,
+            "includeAll": true,
+            "allValue": ".*",
+            "refresh": 2,
+            "sort": 1,
+            "skipUrlSync": false
           }
         ]
       }