ci(bstein-home): publish per-test case metrics for flaky tracking

testing/ci/publish_metrics.py

@@ -7,7 +7,7 @@ import json
 import os
 from pathlib import Path
 
-from .summary import load_junit_summary, publish_quality_metrics
+from .summary import load_junit_cases, load_junit_summary, publish_quality_metrics
 
 
 def _build_parser() -> argparse.ArgumentParser:
@@ -92,7 +92,9 @@ def main(argv: list[str] | None = None) -> int:
 
     parser = _build_parser()
     args = parser.parse_args(argv)
-    summary = load_junit_summary(Path(path) for path in args.junit)
+    junit_paths = [Path(path) for path in args.junit]
+    summary = load_junit_summary(junit_paths)
+    test_cases = load_junit_cases(junit_paths)
     coverage_percent, source_lines_over_500, checks = _load_quality_report(Path(args.quality_report))
     publish_quality_metrics(
         gateway=args.gateway,
@@ -103,6 +105,7 @@ def main(argv: list[str] | None = None) -> int:
         workspace_line_coverage_percent=coverage_percent,
         source_lines_over_500=source_lines_over_500,
         checks=checks,
+        test_cases=test_cases,
     )
     return 0
 

testing/ci/summary.py

@@ -10,6 +10,11 @@ from pathlib import Path
 from typing import Iterable
 
 
+def _escape_label(value: str) -> str:
+    """Escape Prometheus label values."""
+    return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+
 @dataclass(frozen=True)
 class RunSummary:
     """Aggregate counts from a collection of JUnit XML files."""
@@ -48,6 +53,30 @@ def load_junit_summary(paths: Iterable[Path]) -> RunSummary:
     return RunSummary(**totals)
 
 
+def load_junit_cases(paths: Iterable[Path]) -> list[tuple[str, str]]:
+    """Load per-test outcomes from one or more JUnit XML files."""
+    cases: list[tuple[str, str]] = []
+    for path in paths:
+        if not path.exists():
+            continue
+        root = ET.parse(path).getroot()
+        suites = [root] if root.tag == "testsuite" else list(root.findall("testsuite")) if root.tag == "testsuites" else []
+        for suite in suites:
+            for test_case in suite.findall("testcase"):
+                case_name = (test_case.attrib.get("name") or "").strip()
+                class_name = (test_case.attrib.get("classname") or "").strip()
+                if not case_name:
+                    continue
+                full_name = f"{class_name}.{case_name}" if class_name else case_name
+                status = "passed"
+                if test_case.find("failure") is not None or test_case.find("error") is not None:
+                    status = "failed"
+                elif test_case.find("skipped") is not None:
+                    status = "skipped"
+                cases.append((full_name, status))
+    return cases
+
+
 def read_pushgateway_counters(text: str, *, suite: str, job: str) -> dict[str, float]:
     """Read the current quality-gate counters for a suite from Pushgateway text."""
 
@@ -72,6 +101,7 @@ def render_payload(
     workspace_line_coverage_percent: float = 0.0,
     source_lines_over_500: int = 0,
     checks: dict[str, str] | None = None,
+    test_cases: list[tuple[str, str]] | None = None,
 ) -> str:
     """Render the Pushgateway payload for the quality-gate counters."""
     payload = (
@@ -94,6 +124,12 @@ def render_payload(
             f'bstein_home_quality_gate_checks_total{{suite="{suite}",check="{check_name}",result="{check_status}"}} 1\n'
             for check_name, check_status in checks.items()
         )
+    if test_cases:
+        payload += "# TYPE platform_quality_gate_test_case_result gauge\n"
+        payload += "".join(
+            f'platform_quality_gate_test_case_result{{suite="{suite}",test="{_escape_label(test_name)}",status="{_escape_label(test_status)}"}} 1\n'
+            for test_name, test_status in test_cases
+        )
     return payload
 
 
@@ -107,6 +143,7 @@ def publish_quality_metrics(
     workspace_line_coverage_percent: float = 0.0,
     source_lines_over_500: int = 0,
     checks: dict[str, str] | None = None,
+    test_cases: list[tuple[str, str]] | None = None,
 ) -> None:
     """Publish run and test totals to Pushgateway."""
 
@@ -126,6 +163,7 @@ def publish_quality_metrics(
         workspace_line_coverage_percent=workspace_line_coverage_percent,
         source_lines_over_500=source_lines_over_500,
         checks=checks,
+        test_cases=test_cases,
     )
     req = urllib.request.Request(
         f"{gateway}/metrics/job/{job}/suite/{suite}",