bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitoring(testing): add per-test metrics and flaky-test panels

Browse Source
This commit is contained in:
jenkins 2026-04-20 08:35:05 -03:00
parent 7d113291c9
commit f02db9801c
4 changed files with 554 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
ci/scripts/publish_test_metrics.py
View File

@ -93,6 +93,36 @@ def _collect_junit_totals(pattern: str) -> dict[str, int]:
return totals
def _collect_junit_cases(pattern: str) -> list[tuple[str, str]]:
"""Collect individual JUnit test-case statuses for flaky-test trend panels."""
cases: list[tuple[str, str]] = []
for path in sorted(glob(pattern)):
if not os.path.exists(path):
continue
root = ET.parse(path).getroot()
suites: list[ET.Element]
if root.tag == "testsuite":
suites = [root]
elif root.tag == "testsuites":
suites = [elem for elem in root if elem.tag == "testsuite"]
else:
suites = []
for suite in suites:
for test_case in suite.findall("testcase"):
case_name = test_case.attrib.get("name", "").strip()
class_name = test_case.attrib.get("classname", "").strip()
if not case_name:
continue
full_name = f"{class_name}.{case_name}" if class_name else case_name
status = "passed"
if test_case.find("failure") is not None or test_case.find("error") is not None:
status = "failed"
elif test_case.find("skipped") is not None:
status = "skipped"
cases.append((full_name, status))
return cases
def _read_exit_code(path: str) -> int:
"""Read the quality-gate exit code, defaulting to failure if missing."""
try:
@ -293,6 +323,7 @@ def _build_payload(
suite: str,
status: str,
tests: dict[str, int],
test_cases: list[tuple[str, str]],
ok_count: int,
failed_count: int,
branch: str,
@ -337,6 +368,12 @@ def _build_payload(
lines.append(
f'titan_iac_quality_gate_checks_total{{suite="{suite}",check="{_escape_label(check_name)}",result="{_escape_label(check_status)}"}} 1'
)
if test_cases:
lines.append("# TYPE platform_quality_gate_test_case_result gauge")
for test_name, test_status in test_cases:
lines.append(
f'platform_quality_gate_test_case_result{{suite="{suite}",test="{_escape_label(test_name)}",status="{_escape_label(test_status)}"}} 1'
)
return "\n".join(lines) + "\n"
@ -352,6 +389,7 @@ def main() -> int:
build_number = os.getenv("BUILD_NUMBER", "")
tests = _collect_junit_totals(junit_glob)
test_cases = _collect_junit_cases(junit_glob)
exit_code = _read_exit_code(exit_code_path)
status = "ok" if exit_code == 0 else "failed"
summary = _load_summary(summary_path)
@ -393,6 +431,7 @@ def main() -> int:
suite=suite,
status=status,
tests=tests,
test_cases=test_cases,
ok_count=ok_count,
failed_count=failed_count,
branch=branch,

115
scripts/dashboards_render_atlas.py
View File

@ -1108,6 +1108,24 @@ def testing_suite_variable():
}
def testing_case_variable():
return {
"name": "test",
"label": "Test Case",
"type": "query",
"query": 'label_values(platform_quality_gate_test_case_result{suite=~"${suite:regex}",job="platform-quality-ci"}, test)',
"current": {"text": "All", "value": "$__all", "selected": True},
"options": [],
"hide": 0,
"multi": False,
"includeAll": True,
"allValue": ".*",
"refresh": 2,
"sort": 1,
"skipUrlSync": False,
}
def bargauge_panel(
panel_id,
title,
@ -2951,6 +2969,7 @@ def build_mail_dashboard():
def build_jobs_dashboard():
panels = []
suite_var = "${suite:regex}"
test_var = "${test:regex}"
success = PLATFORM_TEST_SUCCESS_STATUS
exported = PLATFORM_TEST_EXPORT_FILTER
runs_selector = f'suite=~"{suite_var}",{exported}'
@ -2961,6 +2980,7 @@ def build_jobs_dashboard():
coverage_metric_selector = f'__name__=~".*_quality_gate_coverage_percent",suite=~"{suite_var}",{exported}'
workspace_coverage_selector = f'suite=~"{suite_var}",{exported}'
smell_selector = f'suite=~"{suite_var}",{exported}'
test_case_selector = f'suite=~"{suite_var}",test=~"{test_var}",{exported}'
suite_universe = " or ".join(
f'label_replace(vector(1), "suite", "{suite}", "__name__", ".*")'
@ -2987,25 +3007,23 @@ def build_jobs_dashboard():
failures_by_suite_24h = (
f'sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_failure_selector}}}[24h]))'
)
success_history_by_suite_core = (
success_history_by_suite = (
f'100 * (sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_success_selector}}}[$__interval])) '
f'/ clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[$__interval]))), 1))'
)
success_history_by_suite = (
f'({success_history_by_suite_core}) '
f'or on(suite) (0 * sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[$__range])))'
)
coverage_by_suite = (
f'(max by (suite) ({{{coverage_metric_selector}}})) '
f'or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{{{workspace_coverage_selector}}}))'
)
coverage_with_missing = (
f"({coverage_by_suite}) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[30d]))) - 1)"
")"
)
coverage_gap = f"clamp_min(95 - ({coverage_by_suite}), 0)"
smell_by_suite = f'max by (suite) (platform_quality_gate_source_lines_over_500_total{{{smell_selector}}})'
smell_with_missing = (
f"({smell_by_suite}) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{{{runs_selector}}}[30d]))) - 1)"
")"
)
average_coverage = f"(avg(({coverage_by_suite})) or on() vector(0))"
suites_loc_violating = f'(sum((({smell_by_suite}) > bool 0)) or on() vector(0))'
@ -3047,6 +3065,30 @@ def build_jobs_dashboard():
)
return f'({core}) or on(suite) (0 * ({suite_universe}))'
problematic_tests_history = (
f'topk(12, sum by (suite, test) (increase(platform_quality_gate_test_case_result{{suite=~"{suite_var}",status="failed",{exported}}}[$__interval])))'
)
worst_test_per_suite = (
f'topk by (suite) (1, sum by (suite, test) (increase(platform_quality_gate_test_case_result{{suite=~"{suite_var}",status="failed",{exported}}}[30d])))'
)
selected_test_pass_fail = [
{
"refId": "A",
"expr": f'sum by (suite) (increase(platform_quality_gate_test_case_result{{{test_case_selector},status="passed"}}[$__interval])) or on() vector(0)',
"legendFormat": "passed · {{suite}}",
},
{
"refId": "B",
"expr": f'sum by (suite) (increase(platform_quality_gate_test_case_result{{{test_case_selector},status="failed"}}[$__interval])) or on() vector(0)',
"legendFormat": "failed · {{suite}}",
},
{
"refId": "C",
"expr": f'sum by (suite) (increase(platform_quality_gate_test_case_result{{{test_case_selector},status="skipped"}}[$__interval])) or on() vector(0)',
"legendFormat": "skipped · {{suite}}",
},
]
missing_tests_by_suite = (
f'(({suite_universe}) unless on(suite) count by (suite) ({{__name__=~".*_quality_gate_tests_total",{exported}}}))'
)
@ -3589,12 +3631,52 @@ def build_jobs_dashboard():
limit=24,
)
)
panels.append(
timeseries_panel(
145,
"Problematic Tests Over Time (Top failures)",
problematic_tests_history,
{"h": 8, "w": 12, "x": 0, "y": 45},
unit="none",
legend="{{suite}} · {{test}}",
legend_display="list",
legend_placement="bottom",
legend_calcs=["lastNotNull", "max", "sum"],
)
)
panels.append(
timeseries_panel(
146,
"Selected Test Pass/Fail History",
None,
{"h": 8, "w": 8, "x": 12, "y": 45},
unit="none",
targets=selected_test_pass_fail,
legend_display="list",
legend_placement="bottom",
legend_calcs=["lastNotNull", "sum"],
)
)
panels.append(
bargauge_panel(
147,
"Most Problematic Test by Suite (30d)",
worst_test_per_suite,
{"h": 8, "w": 4, "x": 20, "y": 45},
unit="none",
instant=True,
legend="{{suite}} · {{test}}",
sort_order="desc",
thresholds=failures_thresholds,
limit=9,
)
)
coverage_panel = bargauge_panel(
17,
"Coverage by Suite (Latest, gate 95)",
coverage_with_missing,
{"h": 8, "w": 12, "x": 0, "y": 45},
{"h": 8, "w": 12, "x": 0, "y": 53},
unit="percent",
instant=True,
legend="{{suite}}",
@ -3611,7 +3693,7 @@ def build_jobs_dashboard():
18,
"Files >500 LOC by Suite (Latest)",
smell_with_missing,
{"h": 8, "w": 12, "x": 12, "y": 45},
{"h": 8, "w": 12, "x": 12, "y": 53},
unit="none",
instant=True,
legend="{{suite}}",
@ -3628,7 +3710,7 @@ def build_jobs_dashboard():
27,
"Missing Tests Metrics by Suite",
missing_tests_by_suite,
{"h": 7, "w": 6, "x": 0, "y": 53},
{"h": 7, "w": 6, "x": 0, "y": 61},
unit="none",
instant=True,
legend="{{suite}}",
@ -3642,7 +3724,7 @@ def build_jobs_dashboard():
28,
"Missing Checks Metrics by Suite",
missing_checks_by_suite,
{"h": 7, "w": 6, "x": 6, "y": 53},
{"h": 7, "w": 6, "x": 6, "y": 61},
unit="none",
instant=True,
legend="{{suite}}",
@ -3656,7 +3738,7 @@ def build_jobs_dashboard():
29,
"Missing Coverage Metrics by Suite",
missing_coverage_by_suite,
{"h": 7, "w": 6, "x": 12, "y": 53},
{"h": 7, "w": 6, "x": 12, "y": 61},
unit="none",
instant=True,
legend="{{suite}}",
@ -3670,7 +3752,7 @@ def build_jobs_dashboard():
30,
"Missing LOC Metrics by Suite",
missing_loc_by_suite,
{"h": 7, "w": 6, "x": 18, "y": 53},
{"h": 7, "w": 6, "x": 18, "y": 61},
unit="none",
instant=True,
legend="{{suite}}",
@ -3684,7 +3766,7 @@ def build_jobs_dashboard():
31,
"SonarQube API Up",
"(max(sonarqube_up) or on() vector(0))",
{"h": 6, "w": 4, "x": 0, "y": 60},
{"h": 6, "w": 4, "x": 0, "y": 68},
unit="none",
instant=True,
thresholds={
@ -3701,7 +3783,7 @@ def build_jobs_dashboard():
32,
"Sonar Projects (Selected)",
f'(count(sonarqube_project_quality_gate_pass{{project_key=~"{suite_var}"}}) or on() vector(0))',
{"h": 6, "w": 4, "x": 4, "y": 60},
{"h": 6, "w": 4, "x": 4, "y": 68},
unit="none",
instant=True,
thresholds=failures_thresholds,
@ -3712,7 +3794,7 @@ def build_jobs_dashboard():
33,
"Sonar Gate Fetch Errors",
"(max(sonarqube_quality_gate_fetch_errors_total) or on() vector(0))",
{"h": 6, "w": 4, "x": 8, "y": 60},
{"h": 6, "w": 4, "x": 8, "y": 68},
unit="none",
instant=True,
thresholds=failures_thresholds,
@ -3722,7 +3804,7 @@ def build_jobs_dashboard():
34,
"Sonar Gate Status Mix (Selected)",
f'count by (status) (sonarqube_project_quality_gate_pass{{project_key=~"{suite_var}"}})',
{"h": 6, "w": 6, "x": 12, "y": 60},
{"h": 6, "w": 6, "x": 12, "y": 68},
)
sonar_status_mix_panel["targets"][0]["legendFormat"] = "{{status}}"
panels.append(sonar_status_mix_panel)
@ -3731,7 +3813,7 @@ def build_jobs_dashboard():
35,
"Projects Failing Sonar Gate",
f'sort_desc(count by (project_key) (sonarqube_project_quality_gate_pass{{project_key=~"{suite_var}",status!~"OK|ok"}}))',
{"h": 6, "w": 6, "x": 18, "y": 60},
{"h": 6, "w": 6, "x": 18, "y": 68},
unit="none",
instant=True,
legend="{{project_key}}",
@ -3754,6 +3836,7 @@ def build_jobs_dashboard():
"templating": {
"list": [
testing_suite_variable(),
testing_case_variable(),
]
},
}

224
services/monitoring/dashboards/atlas-jobs.json
View File

@ -1210,7 +1210,7 @@
},
"targets": [
{
"expr": "(100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))) or on(suite) (0 * sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__range])))",
"expr": "100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))",
"refId": "A",
"legendFormat": "{{suite}}"
}
@ -2045,9 +2045,9 @@
]
},
{
"id": 17,
"type": "bargauge",
"title": "Coverage by Suite (Latest, gate 95)",
"id": 145,
"type": "timeseries",
"title": "Problematic Tests Over Time (Top failures)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
@ -2060,7 +2060,180 @@
},
"targets": [
{
"expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))",
"expr": "topk(12, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[$__interval])))",
"refId": "A",
"legendFormat": "{{suite}} \u00b7 {{test}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max",
"sum"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 146,
"type": "timeseries",
"title": "Selected Test Pass/Fail History",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 12,
"y": 45
},
"targets": [
{
"refId": "A",
"expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"passed\"}[$__interval])) or on() vector(0)",
"legendFormat": "passed \u00b7 {{suite}}"
},
{
"refId": "B",
"expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"failed\"}[$__interval])) or on() vector(0)",
"legendFormat": "failed \u00b7 {{suite}}"
},
{
"refId": "C",
"expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"skipped\"}[$__interval])) or on() vector(0)",
"legendFormat": "skipped \u00b7 {{suite}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"calcs": [
"lastNotNull",
"sum"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 147,
"type": "bargauge",
"title": "Most Problematic Test by Suite (30d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 8,
"w": 4,
"x": 20,
"y": 45
},
"targets": [
{
"expr": "sort_desc(topk by (suite) (1, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[30d]))))",
"refId": "A",
"legendFormat": "{{suite}} \u00b7 {{test}}",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"min": 0,
"max": null,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "orange",
"value": 3
},
{
"color": "red",
"value": 5
}
]
}
},
"overrides": []
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
},
"transformations": [
{
"id": "sortBy",
"options": {
"fields": [
"Value"
],
"order": "desc"
}
},
{
"id": "limit",
"options": {
"limit": 9
}
}
]
},
{
"id": 17,
"type": "bargauge",
"title": "Coverage by Suite (Latest, gate 95)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 53
},
"targets": [
{
"expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))",
"refId": "A",
"legendFormat": "{{suite}}",
"instant": true
@ -2141,11 +2314,11 @@
"h": 8,
"w": 12,
"x": 12,
"y": 45
"y": 53
},
"targets": [
{
"expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))",
"expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))",
"refId": "A",
"legendFormat": "{{suite}}",
"instant": true
@ -2229,7 +2402,7 @@
"h": 7,
"w": 6,
"x": 0,
"y": 53
"y": 61
},
"targets": [
{
@ -2296,7 +2469,7 @@
"h": 7,
"w": 6,
"x": 6,
"y": 53
"y": 61
},
"targets": [
{
@ -2363,7 +2536,7 @@
"h": 7,
"w": 6,
"x": 12,
"y": 53
"y": 61
},
"targets": [
{
@ -2430,7 +2603,7 @@
"h": 7,
"w": 6,
"x": 18,
"y": 53
"y": 61
},
"targets": [
{
@ -2497,7 +2670,7 @@
"h": 6,
"w": 4,
"x": 0,
"y": 60
"y": 68
},
"targets": [
{
@ -2558,7 +2731,7 @@
"h": 6,
"w": 4,
"x": 4,
"y": 60
"y": 68
},
"targets": [
{
@ -2627,7 +2800,7 @@
"h": 6,
"w": 4,
"x": 8,
"y": 60
"y": 68
},
"targets": [
{
@ -2696,7 +2869,7 @@
"h": 6,
"w": 6,
"x": 12,
"y": 60
"y": 68
},
"targets": [
{
@ -2747,7 +2920,7 @@
"h": 6,
"w": 6,
"x": 18,
"y": 60
"y": 68
},
"targets": [
{
@ -2891,6 +3064,25 @@
"refresh": 1,
"sort": 1,
"skipUrlSync": false
},
{
"name": "test",
"label": "Test Case",
"type": "query",
"query": "label_values(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",job=\"platform-quality-ci\"}, test)",
"current": {
"text": "All",
"value": "$__all",
"selected": true
},
"options": [],
"hide": 0,
"multi": false,
"includeAll": true,
"allValue": ".*",
"refresh": 2,
"sort": 1,
"skipUrlSync": false
}
]
}

224
services/monitoring/grafana-dashboard-jobs.yaml
View File

@ -1219,7 +1219,7 @@ data:
},
"targets": [
{
"expr": "(100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))) or on(suite) (0 * sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__range])))",
"expr": "100 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[$__interval])) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[$__interval]))), 1))",
"refId": "A",
"legendFormat": "{{suite}}"
}
@ -2054,9 +2054,9 @@ data:
]
},
{
"id": 17,
"type": "bargauge",
"title": "Coverage by Suite (Latest, gate 95)",
"id": 145,
"type": "timeseries",
"title": "Problematic Tests Over Time (Top failures)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
@ -2069,7 +2069,180 @@ data:
},
"targets": [
{
"expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))",
"expr": "topk(12, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[$__interval])))",
"refId": "A",
"legendFormat": "{{suite}} \u00b7 {{test}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"calcs": [
"lastNotNull",
"max",
"sum"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 146,
"type": "timeseries",
"title": "Selected Test Pass/Fail History",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 8,
"w": 8,
"x": 12,
"y": 45
},
"targets": [
{
"refId": "A",
"expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"passed\"}[$__interval])) or on() vector(0)",
"legendFormat": "passed \u00b7 {{suite}}"
},
{
"refId": "B",
"expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"failed\"}[$__interval])) or on() vector(0)",
"legendFormat": "failed \u00b7 {{suite}}"
},
{
"refId": "C",
"expr": "sum by (suite) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",test=~\"${test:regex}\",exported_job=\"platform-quality-ci\",status=\"skipped\"}[$__interval])) or on() vector(0)",
"legendFormat": "skipped \u00b7 {{suite}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"calcs": [
"lastNotNull",
"sum"
]
},
"tooltip": {
"mode": "multi"
}
}
},
{
"id": 147,
"type": "bargauge",
"title": "Most Problematic Test by Suite (30d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 8,
"w": 4,
"x": 20,
"y": 45
},
"targets": [
{
"expr": "sort_desc(topk by (suite) (1, sum by (suite, test) (increase(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",status=\"failed\",exported_job=\"platform-quality-ci\"}[30d]))))",
"refId": "A",
"legendFormat": "{{suite}} \u00b7 {{test}}",
"instant": true
}
],
"fieldConfig": {
"defaults": {
"unit": "none",
"min": 0,
"max": null,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "orange",
"value": 3
},
{
"color": "red",
"value": 5
}
]
}
},
"overrides": []
},
"options": {
"displayMode": "gradient",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
}
},
"transformations": [
{
"id": "sortBy",
"options": {
"fields": [
"Value"
],
"order": "desc"
}
},
{
"id": "limit",
"options": {
"limit": 9
}
}
]
},
{
"id": 17,
"type": "bargauge",
"title": "Coverage by Suite (Latest, gate 95)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 53
},
"targets": [
{
"expr": "sort(((max by (suite) ({__name__=~\".*_quality_gate_coverage_percent\",suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))",
"refId": "A",
"legendFormat": "{{suite}}",
"instant": true
@ -2150,11 +2323,11 @@ data:
"h": 8,
"w": 12,
"x": 12,
"y": 45
"y": 53
},
"targets": [
{
"expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1))",
"expr": "sort_desc((max by (suite) (platform_quality_gate_source_lines_over_500_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"})) or on(suite) (0 * (sum by (suite) (increase(platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}[30d]))) - 1)))",
"refId": "A",
"legendFormat": "{{suite}}",
"instant": true
@ -2238,7 +2411,7 @@ data:
"h": 7,
"w": 6,
"x": 0,
"y": 53
"y": 61
},
"targets": [
{
@ -2305,7 +2478,7 @@ data:
"h": 7,
"w": 6,
"x": 6,
"y": 53
"y": 61
},
"targets": [
{
@ -2372,7 +2545,7 @@ data:
"h": 7,
"w": 6,
"x": 12,
"y": 53
"y": 61
},
"targets": [
{
@ -2439,7 +2612,7 @@ data:
"h": 7,
"w": 6,
"x": 18,
"y": 53
"y": 61
},
"targets": [
{
@ -2506,7 +2679,7 @@ data:
"h": 6,
"w": 4,
"x": 0,
"y": 60
"y": 68
},
"targets": [
{
@ -2567,7 +2740,7 @@ data:
"h": 6,
"w": 4,
"x": 4,
"y": 60
"y": 68
},
"targets": [
{
@ -2636,7 +2809,7 @@ data:
"h": 6,
"w": 4,
"x": 8,
"y": 60
"y": 68
},
"targets": [
{
@ -2705,7 +2878,7 @@ data:
"h": 6,
"w": 6,
"x": 12,
"y": 60
"y": 68
},
"targets": [
{
@ -2756,7 +2929,7 @@ data:
"h": 6,
"w": 6,
"x": 18,
"y": 60
"y": 68
},
"targets": [
{
@ -2900,6 +3073,25 @@ data:
"refresh": 1,
"sort": 1,
"skipUrlSync": false
},
{
"name": "test",
"label": "Test Case",
"type": "query",
"query": "label_values(platform_quality_gate_test_case_result{suite=~\"${suite:regex}\",job=\"platform-quality-ci\"}, test)",
"current": {
"text": "All",
"value": "$__all",
"selected": true
},
"options": [],
"hide": 0,
"multi": false,
"includeAll": true,
"allValue": ".*",
"refresh": 2,
"sort": 1,
"skipUrlSync": false
}
]
}