monitoring(testing): add per-test history panels and metric emitter

ci/scripts/publish_test_metrics.py

@@ -29,11 +29,11 @@ def _read_text(url: str) -> str:
 
 
 def _post_text(url: str, payload: str) -> None:
-    """POST a plain-text payload and fail on any 4xx/5xx response."""
+    """PUT a plain-text payload and fail on any 4xx/5xx response."""
     request = urllib.request.Request(
         url,
         data=payload.encode("utf-8"),
-        method="POST",
+        method="PUT",
         headers={"Content-Type": "text/plain"},
     )
     with urllib.request.urlopen(request, timeout=10) as response:
@@ -78,6 +78,48 @@ def _collect_junit_totals(pattern: str) -> dict[str, int]:
     return totals
 
 
+def _load_junit_cases(path: str) -> list[tuple[str, str]]:
+    """Parse individual JUnit test case outcomes for flakiness panels."""
+    if not os.path.exists(path):
+        return []
+
+    tree = ET.parse(path)
+    root = tree.getroot()
+    suites: list[ET.Element]
+    if root.tag == "testsuite":
+        suites = [root]
+    elif root.tag == "testsuites":
+        suites = [elem for elem in root if elem.tag == "testsuite"]
+    else:
+        suites = []
+
+    cases: list[tuple[str, str]] = []
+    for suite in suites:
+        for case in suite.findall("testcase"):
+            name = (case.attrib.get("name") or "").strip()
+            classname = (case.attrib.get("classname") or "").strip()
+            if not name:
+                continue
+            test_id = f"{classname}::{name}" if classname else name
+            status = "passed"
+            if case.find("failure") is not None:
+                status = "failed"
+            elif case.find("error") is not None:
+                status = "error"
+            elif case.find("skipped") is not None:
+                status = "skipped"
+            cases.append((test_id, status))
+    return cases
+
+
+def _collect_junit_cases(pattern: str) -> list[tuple[str, str]]:
+    """Collect test-case statuses across all matching JUnit XML files."""
+    cases: list[tuple[str, str]] = []
+    for path in sorted(glob(pattern)):
+        cases.extend(_load_junit_cases(path))
+    return cases
+
+
 def _read_exit_code(path: str) -> int:
     """Read the quality-gate exit code, defaulting to failure if missing."""
     try:
@@ -136,6 +178,7 @@ def _build_payload(
     suite: str,
     status: str,
     tests: dict[str, int],
+    test_cases: list[tuple[str, str]],
     ok_count: int,
     failed_count: int,
     branch: str,
@@ -171,7 +214,12 @@ def _build_payload(
         f'platform_quality_gate_workspace_line_coverage_percent{{suite="{suite}"}} {workspace_line_coverage_percent:.3f}',
         "# TYPE platform_quality_gate_source_lines_over_500_total gauge",
         f'platform_quality_gate_source_lines_over_500_total{{suite="{suite}"}} {source_lines_over_500}',
+        "# TYPE platform_quality_gate_test_case_result gauge",
     ]
+    lines.extend(
+        f'platform_quality_gate_test_case_result{{suite="{suite}",test="{_escape_label(test_name)}",status="{_escape_label(test_status)}"}} 1'
+        for test_name, test_status in test_cases
+    )
     results = summary.get("results", []) if isinstance(summary, dict) else []
     if results:
         lines.append("# TYPE titan_iac_quality_gate_checks_total gauge")
@@ -188,7 +236,7 @@ def _build_payload(
 
 def main() -> int:
     """Publish the quality-gate metrics and print a compact run summary."""
-    suite = os.getenv("SUITE_NAME", "titan-iac")
+    suite = os.getenv("SUITE_NAME", "titan_iac")
     pushgateway_url = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091")
     job_name = os.getenv("QUALITY_GATE_JOB_NAME", "platform-quality-ci")
     junit_glob = os.getenv("JUNIT_GLOB", os.getenv("JUNIT_PATH", "build/junit-*.xml"))
@@ -198,6 +246,7 @@ def main() -> int:
     build_number = os.getenv("BUILD_NUMBER", "")
 
     tests = _collect_junit_totals(junit_glob)
+    test_cases = _collect_junit_cases(junit_glob)
     exit_code = _read_exit_code(exit_code_path)
     status = "ok" if exit_code == 0 else "failed"
     summary = _load_summary(summary_path)
@@ -227,6 +276,7 @@ def main() -> int:
         suite=suite,
         status=status,
         tests=tests,
+        test_cases=test_cases,
         ok_count=ok_count,
         failed_count=failed_count,
         branch=branch,

services/logging/Jenkinsfile.data-prepper

@@ -31,11 +31,21 @@ spec:
 """
     }
   }
+  environment {
+    SUITE_NAME = 'data_prepper'
+    PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
+    QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
+    QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
+  }
   parameters {
     string(name: 'HARBOR_REPO', defaultValue: 'registry.bstein.dev/streaming/data-prepper', description: 'Docker repository for Data Prepper')
     string(name: 'IMAGE_TAG', defaultValue: '2.8.0', description: 'Image tag to publish')
     booleanParam(name: 'PUSH_LATEST', defaultValue: true, description: 'Also push the latest tag')
   }
+  options {
+    disableConcurrentBuilds()
+    buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
+  }
   stages {
     stage('Checkout') {
       steps {
@@ -44,6 +54,79 @@ spec:
         }
       }
     }
+    stage('Collect quality evidence') {
+      steps {
+        container('git') {
+          sh '''
+            set -euo pipefail
+            apk add --no-cache curl jq >/dev/null 2>&1 || true
+            mkdir -p build
+
+            sonar_report="${QUALITY_GATE_SONARQUBE_REPORT:-build/sonarqube-quality-gate.json}"
+            if [ ! -f "${sonar_report}" ]; then
+              if [ -n "${SONARQUBE_HOST_URL:-}" ] && [ -n "${SONARQUBE_PROJECT_KEY:-}" ]; then
+                host="${SONARQUBE_HOST_URL%/}"
+                query="$(printf '%s' "${SONARQUBE_PROJECT_KEY}" | sed 's/ /%20/g')"
+                sonar_ok=0
+                if [ -n "${SONARQUBE_TOKEN:-}" ]; then
+                  auth="$(printf '%s:' "${SONARQUBE_TOKEN}" | base64 | tr -d '\\n')"
+                  if curl -fsS -H "Authorization: Basic ${auth}" "${host}/api/qualitygates/project_status?projectKey=${query}" > "${sonar_report}"; then
+                    sonar_ok=1
+                  fi
+                else
+                  if curl -fsS "${host}/api/qualitygates/project_status?projectKey=${query}" > "${sonar_report}"; then
+                    sonar_ok=1
+                  fi
+                fi
+                if [ "${sonar_ok}" -ne 1 ]; then
+                  cat > "${sonar_report}" <<EOF
+{
+  "status": "ERROR",
+  "error": "sonarqube query failed"
+}
+EOF
+                fi
+              else
+                cat > "${sonar_report}" <<EOF
+{
+  "status": "ERROR",
+  "note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY"
+}
+EOF
+              fi
+            fi
+
+            ironbank_report="${QUALITY_GATE_IRONBANK_REPORT:-build/ironbank-compliance.json}"
+            if [ ! -f "${ironbank_report}" ]; then
+              status="${IRONBANK_COMPLIANCE_STATUS:-unknown}"
+              compliant="${IRONBANK_COMPLIANT:-}"
+              if [ -n "${compliant}" ]; then
+                compliant_lc="$(printf '%s' "${compliant}" | tr '[:upper:]' '[:lower:]')"
+                compliant_json="null"
+                case "${compliant_lc}" in
+                  1|true|yes|on) compliant_json="true" ;;
+                  0|false|no|off) compliant_json="false" ;;
+                esac
+                cat > "${ironbank_report}" <<EOF
+{
+  "status": "${status}",
+  "compliant": ${compliant_json},
+  "note": "Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT or write build/ironbank-compliance.json in image-building repos."
+}
+EOF
+              else
+                cat > "${ironbank_report}" <<EOF
+{
+  "status": "${status}",
+  "note": "Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT or write build/ironbank-compliance.json in image-building repos."
+}
+EOF
+              fi
+            fi
+          '''
+        }
+      }
+    }
     stage('Build & Push') {
       steps {
         container('kaniko') {
@@ -80,4 +163,95 @@ EOF
       }
     }
   }
+  post {
+    always {
+      script {
+        env.QUALITY_OUTCOME = currentBuild.currentResult == 'SUCCESS' ? 'ok' : 'failed'
+      }
+      container('git') {
+        sh '''
+          set -euo pipefail
+          apk add --no-cache curl jq >/dev/null 2>&1 || true
+          suite="${SUITE_NAME}"
+          gateway="${PUSHGATEWAY_URL}"
+          status="${QUALITY_OUTCOME:-failed}"
+          fetch_counter() {
+            status_name="$1"
+            line="$(curl -fsS "${gateway}/metrics" 2>/dev/null | awk -v suite="${suite}" -v status="${status_name}" '
+              /platform_quality_gate_runs_total/ {
+                if (index($0, "job=\\"platform-quality-ci\\"") && index($0, "suite=\\"" suite "\\"") && index($0, "status=\\"" status "\\"")) {
+                  print $2
+                  exit
+                }
+              }
+            ' || true)"
+            [ -n "${line}" ] && printf '%s\n' "${line}" || printf '0\n'
+          }
+          ok_count="$(fetch_counter ok)"
+          failed_count="$(fetch_counter failed)"
+          if [ "${status}" = "ok" ]; then
+            ok_count=$((ok_count + 1))
+          else
+            failed_count=$((failed_count + 1))
+          fi
+          sonarqube_check="not_applicable"
+          if [ -f build/sonarqube-quality-gate.json ]; then
+            sonar_status="$(jq -r '.status // .projectStatus.status // .qualityGate.status // empty' build/sonarqube-quality-gate.json 2>/dev/null | tr '[:upper:]' '[:lower:]')"
+            if [ -n "${sonar_status}" ]; then
+              case "${sonar_status}" in
+                ok|pass|passed|success) sonarqube_check="ok" ;;
+                *) sonarqube_check="failed" ;;
+              esac
+            else
+              sonarqube_check="failed"
+            fi
+          fi
+          supply_chain_check="not_applicable"
+          if [ -f build/ironbank-compliance.json ]; then
+            compliant="$(jq -r '.compliant // empty' build/ironbank-compliance.json 2>/dev/null)"
+            if [ "${compliant}" = "true" ]; then
+              supply_chain_check="ok"
+            elif [ "${compliant}" = "false" ]; then
+              supply_chain_check="failed"
+            else
+              ironbank_status="$(jq -r '.status // .result // .compliance // empty' build/ironbank-compliance.json 2>/dev/null | tr '[:upper:]' '[:lower:]')"
+              case "${ironbank_status}" in
+                ok|pass|passed|success|compliant) supply_chain_check="ok" ;;
+                "") supply_chain_check="failed" ;;
+                *) supply_chain_check="failed" ;;
+              esac
+            fi
+          fi
+          gate_glue_check="ok"
+          if [ "${status}" != "ok" ]; then
+            gate_glue_check="failed"
+          fi
+          cat <<METRICS | curl -fsS -X PUT --data-binary @- "${gateway}/metrics/job/platform-quality-ci/suite/${suite}" >/dev/null || \
+            echo "warning: metrics push failed for suite=${suite}" >&2
+# TYPE platform_quality_gate_runs_total counter
+platform_quality_gate_runs_total{suite="${suite}",status="ok"} ${ok_count}
+platform_quality_gate_runs_total{suite="${suite}",status="failed"} ${failed_count}
+# TYPE data_prepper_quality_gate_tests_total gauge
+data_prepper_quality_gate_tests_total{suite="${suite}",result="passed"} 0
+data_prepper_quality_gate_tests_total{suite="${suite}",result="failed"} 0
+data_prepper_quality_gate_tests_total{suite="${suite}",result="error"} 0
+data_prepper_quality_gate_tests_total{suite="${suite}",result="skipped"} 0
+# TYPE platform_quality_gate_workspace_line_coverage_percent gauge
+platform_quality_gate_workspace_line_coverage_percent{suite="${suite}"} 0
+# TYPE platform_quality_gate_source_lines_over_500_total gauge
+platform_quality_gate_source_lines_over_500_total{suite="${suite}"} 0
+# TYPE data_prepper_quality_gate_checks_total gauge
+data_prepper_quality_gate_checks_total{suite="${suite}",check="tests",result="not_applicable"} 1
+data_prepper_quality_gate_checks_total{suite="${suite}",check="coverage",result="not_applicable"} 1
+data_prepper_quality_gate_checks_total{suite="${suite}",check="loc",result="not_applicable"} 1
+data_prepper_quality_gate_checks_total{suite="${suite}",check="docs_naming",result="not_applicable"} 1
+data_prepper_quality_gate_checks_total{suite="${suite}",check="gate_glue",result="${gate_glue_check}"} 1
+data_prepper_quality_gate_checks_total{suite="${suite}",check="sonarqube",result="${sonarqube_check}"} 1
+data_prepper_quality_gate_checks_total{suite="${suite}",check="supply_chain",result="${supply_chain_check}"} 1
+METRICS
+        '''
+      }
+      archiveArtifacts artifacts: 'build/**/*.json,build/**/*.xml,build/**/*.txt,build/**/*.rc', allowEmptyArchive: true, fingerprint: true
+    }
+  }
 }

services/monitoring/dashboards/atlas-overview.json

@@ -1318,11 +1318,6 @@
           "refId": "B",
           "expr": "((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100)",
           "legendFormat": "Statera"
-        },
-        {
-          "refId": "C",
-          "expr": "sum((ananke_ups_load_percent{job=\"ananke-power\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\"}) / 100)",
-          "legendFormat": "combined"
         }
       ],
       "fieldConfig": {

services/monitoring/dashboards/atlas-power.json

@@ -245,11 +245,6 @@
           "refId": "B",
           "expr": "((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100)",
           "legendFormat": "Statera"
-        },
-        {
-          "refId": "C",
-          "expr": "sum((ananke_ups_load_percent{job=\"ananke-power\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\"}) / 100)",
-          "legendFormat": "combined"
         }
       ],
       "fieldConfig": {
@@ -267,7 +262,7 @@
           "mode": "multi"
         }
       },
-      "description": "Historical UPS power consumption in watts for titan-db, tethys, and combined load."
+      "description": "Historical UPS power consumption in watts for titan-db and tethys."
     },
     {
       "id": 3,

services/monitoring/grafana-dashboard-overview.yaml

@@ -1327,11 +1327,6 @@ data:
               "refId": "B",
               "expr": "((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100)",
               "legendFormat": "Statera"
-            },
-            {
-              "refId": "C",
-              "expr": "sum((ananke_ups_load_percent{job=\"ananke-power\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\"}) / 100)",
-              "legendFormat": "combined"
             }
           ],
           "fieldConfig": {

services/monitoring/grafana-dashboard-power.yaml

@@ -254,11 +254,6 @@ data:
               "refId": "B",
               "expr": "((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100)",
               "legendFormat": "Statera"
-            },
-            {
-              "refId": "C",
-              "expr": "sum((ananke_ups_load_percent{job=\"ananke-power\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\"}) / 100)",
-              "legendFormat": "combined"
             }
           ],
           "fieldConfig": {
@@ -276,7 +271,7 @@ data:
               "mode": "multi"
             }
           },
-          "description": "Historical UPS power consumption in watts for titan-db, tethys, and combined load."
+          "description": "Historical UPS power consumption in watts for titan-db and tethys."
         },
         {
           "id": 3,

testing/tests/test_publish_test_metrics.py

@@ -40,6 +40,27 @@ def test_collect_junit_totals_sums_multiple_files(tmp_path: Path):
     assert totals == {"tests": 5, "failures": 1, "errors": 1, "skipped": 1}
 
 
+def test_collect_junit_cases_tracks_individual_statuses(tmp_path: Path):
+    junit = tmp_path / "junit.xml"
+    junit.write_text(
+        (
+            "<testsuite>"
+            '<testcase classname="pkg.mod" name="test_ok" />'
+            '<testcase classname="pkg.mod" name="test_fail"><failure /></testcase>'
+            '<testcase classname="pkg.mod" name="test_error"><error /></testcase>'
+            '<testcase classname="pkg.mod" name="test_skip"><skipped /></testcase>'
+            "</testsuite>"
+        ),
+        encoding="utf-8",
+    )
+
+    cases = publish_test_metrics._collect_junit_cases(str(tmp_path / "junit*.xml"))
+    assert ("pkg.mod::test_ok", "passed") in cases
+    assert ("pkg.mod::test_fail", "failed") in cases
+    assert ("pkg.mod::test_error", "error") in cases
+    assert ("pkg.mod::test_skip", "skipped") in cases
+
+
 def test_parse_junit_handles_testsuites_and_invalid_counts(tmp_path: Path):
     junit_path = tmp_path / "suite.xml"
     junit_path.write_text(
@@ -171,6 +192,7 @@ def test_build_payload_includes_summary_metrics():
         suite="titan-iac",
         status="ok",
         tests={"tests": 4, "failures": 1, "errors": 0, "skipped": 1},
+        test_cases=[("pkg.mod::test_ok", "passed"), ("pkg.mod::test_fail", "failed")],
         ok_count=7,
         failed_count=2,
         branch="main",
@@ -190,6 +212,7 @@ def test_build_payload_includes_summary_metrics():
     assert 'titan_iac_quality_gate_checks_total{suite="titan-iac",check="unit",result="failed"} 1' in payload
     assert 'platform_quality_gate_workspace_line_coverage_percent{suite="titan-iac"} 97.125' in payload
     assert 'platform_quality_gate_source_lines_over_500_total{suite="titan-iac"} 3' in payload
+    assert 'platform_quality_gate_test_case_result{suite="titan-iac",test="pkg.mod::test_fail",status="failed"} 1' in payload
 
 
 def test_build_payload_skips_incomplete_results():
@@ -197,6 +220,7 @@ def test_build_payload_skips_incomplete_results():
         suite="titan-iac",
         status="failed",
         tests={"tests": 0, "failures": 0, "errors": 0, "skipped": 0},
+        test_cases=[],
         ok_count=1,
         failed_count=2,
         branch="",