298 lines
11 KiB
Bash
Executable File
298 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Run the Rust test suite, publish CI test metrics when configured, and retain artifacts.
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
|
|
source "${ROOT_DIR}/scripts/ci/headless_run.sh"
|
|
REPORT_DIR="${ROOT_DIR}/target/test-gate"
|
|
TEST_LOG="${REPORT_DIR}/cargo-test.log"
|
|
SUMMARY_JSON="${REPORT_DIR}/summary.json"
|
|
SUMMARY_TXT="${REPORT_DIR}/summary.txt"
|
|
METRICS_FILE="${REPORT_DIR}/metrics.prom"
|
|
PUSHGATEWAY_URL=${QUALITY_GATE_PUSHGATEWAY_URL:-}
|
|
PUSHGATEWAY_JOB=${LESAVKA_TEST_GATE_PUSHGATEWAY_JOB:-platform-quality-ci}
|
|
|
|
mkdir -p "${REPORT_DIR}"
|
|
cd "${ROOT_DIR}"
|
|
|
|
branch=${BRANCH_NAME:-${GIT_BRANCH:-}}
|
|
if [[ -z "${branch}" ]]; then
|
|
branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)
|
|
fi
|
|
commit=${GIT_COMMIT:-}
|
|
if [[ -z "${commit}" ]]; then
|
|
commit=$(git rev-parse --short HEAD 2>/dev/null || echo unknown)
|
|
fi
|
|
build_url=${BUILD_URL:-}
|
|
|
|
start_seconds=$(date +%s)
|
|
status=0
|
|
set +e
|
|
export RUST_TEST_THREADS="${RUST_TEST_THREADS:-1}"
|
|
lesavka_ci_run_headless cargo test --workspace --all-targets --no-fail-fast --color never 2>&1 | tee "${TEST_LOG}"
|
|
status=${PIPESTATUS[0]}
|
|
set -e
|
|
end_seconds=$(date +%s)
|
|
duration_seconds=$((end_seconds - start_seconds))
|
|
|
|
python3 - \
|
|
"${TEST_LOG}" \
|
|
"${SUMMARY_JSON}" \
|
|
"${SUMMARY_TXT}" \
|
|
"${METRICS_FILE}" \
|
|
"${status}" \
|
|
"${duration_seconds}" \
|
|
"${branch}" \
|
|
"${commit}" \
|
|
"${build_url}" \
|
|
"${BUILD_NUMBER:-unknown}" \
|
|
"${JOB_NAME:-lesavka}" <<'PY'
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
log_path = pathlib.Path(sys.argv[1])
|
|
summary_json_path = pathlib.Path(sys.argv[2])
|
|
summary_txt_path = pathlib.Path(sys.argv[3])
|
|
metrics_path = pathlib.Path(sys.argv[4])
|
|
status = int(sys.argv[5])
|
|
duration_seconds = int(sys.argv[6])
|
|
branch = sys.argv[7] or 'unknown'
|
|
commit = sys.argv[8] or 'unknown'
|
|
build_url = sys.argv[9]
|
|
build_number = sys.argv[10] or 'unknown'
|
|
jenkins_job = sys.argv[11] or 'lesavka'
|
|
|
|
result_re = re.compile(
|
|
r'test result: (?:ok|FAILED)\. '
|
|
r'(?P<passed>\d+) passed; '
|
|
r'(?P<failed>\d+) failed; '
|
|
r'(?P<ignored>\d+) ignored; '
|
|
r'(?P<measured>\d+) measured; '
|
|
r'(?P<filtered>\d+) filtered out;'
|
|
)
|
|
running_re = re.compile(r'^\s*Running (?P<target>.+?)(?: \(|$)')
|
|
test_re = re.compile(r'^test (?P<name>.+?) \.\.\. (?P<result>ok|FAILED|ignored)$')
|
|
|
|
counts = {'passed': 0, 'failed': 0, 'ignored': 0, 'measured': 0, 'filtered': 0}
|
|
test_cases = []
|
|
current_target = ''
|
|
|
|
test_category_names = {
|
|
'api', 'chaos', 'compatibility', 'component', 'contract', 'e2e',
|
|
'integration', 'manual', 'performance', 'regression', 'reliability',
|
|
'security', 'smoke', 'system', 'ui', 'unit',
|
|
}
|
|
support_category_names = {'fixtures', 'golden', 'helpers'}
|
|
|
|
def normalize_category(category: str) -> str:
|
|
if category in test_category_names:
|
|
return category
|
|
if category in support_category_names:
|
|
return 'uncategorized'
|
|
return 'uncategorized'
|
|
|
|
manifest_path = pathlib.Path('tests/test-taxonomy-manifest.json')
|
|
category_by_path = {}
|
|
category_by_test_name = {}
|
|
test_categories = set()
|
|
if manifest_path.exists():
|
|
for item in json.loads(manifest_path.read_text(encoding='utf-8')):
|
|
path = item.get('new', '')
|
|
category = normalize_category(item.get('category', ''))
|
|
if path and category:
|
|
category_by_path[path] = category
|
|
category_by_test_name[pathlib.PurePosixPath(path).stem] = category
|
|
if category in test_category_names:
|
|
test_categories.add(category)
|
|
|
|
def category_for_path(path: str) -> str:
|
|
if path in category_by_path:
|
|
return category_by_path[path]
|
|
parts = pathlib.PurePosixPath(path).parts
|
|
if len(parts) >= 2 and parts[0] == 'tests':
|
|
return normalize_category(parts[1])
|
|
if path.startswith('src/'):
|
|
return 'unit'
|
|
return 'uncategorized'
|
|
|
|
cargo_toml_path = pathlib.Path('Cargo.toml')
|
|
if cargo_toml_path.exists():
|
|
current_name = ''
|
|
current_path = ''
|
|
in_test_section = False
|
|
for raw in cargo_toml_path.read_text(encoding='utf-8').splitlines():
|
|
line = raw.strip()
|
|
if line == '[[test]]':
|
|
if current_name and current_path:
|
|
category_by_test_name[current_name] = category_for_path(current_path)
|
|
current_name = ''
|
|
current_path = ''
|
|
in_test_section = True
|
|
continue
|
|
if line.startswith('['):
|
|
if in_test_section and current_name and current_path:
|
|
category_by_test_name[current_name] = category_for_path(current_path)
|
|
current_name = ''
|
|
current_path = ''
|
|
in_test_section = False
|
|
continue
|
|
if not in_test_section or '=' not in line:
|
|
continue
|
|
key, value = [part.strip() for part in line.split('=', 1)]
|
|
value = value.strip('"')
|
|
if key == 'name':
|
|
current_name = value
|
|
elif key == 'path':
|
|
current_path = value
|
|
if in_test_section and current_name and current_path:
|
|
category_by_test_name[current_name] = category_for_path(current_path)
|
|
|
|
def category_for_target(target: str) -> str:
|
|
for prefix in ('unittests ', 'doctests '):
|
|
if target.startswith(prefix):
|
|
target = target[len(prefix):]
|
|
break
|
|
if target in category_by_path:
|
|
return category_by_path[target]
|
|
target_path = pathlib.PurePosixPath(target)
|
|
target_name = target_path.stem
|
|
if target_name in category_by_test_name:
|
|
return category_by_test_name[target_name]
|
|
if len(target_path.parts) >= 3 and target_path.parts[-3:-1] == ('debug', 'deps'):
|
|
binary_name = target_path.name.rsplit('-', 1)[0]
|
|
if binary_name in category_by_test_name:
|
|
return category_by_test_name[binary_name]
|
|
parts = pathlib.PurePosixPath(target).parts
|
|
if len(parts) >= 2 and parts[0] == 'tests':
|
|
return normalize_category(parts[1])
|
|
if target.startswith('src/'):
|
|
return 'unit'
|
|
return 'uncategorized'
|
|
|
|
for raw in log_path.read_text(encoding='utf-8', errors='replace').splitlines():
|
|
running = running_re.search(raw)
|
|
if running:
|
|
current_target = running.group('target')
|
|
continue
|
|
test_match = test_re.search(raw.strip())
|
|
if test_match:
|
|
raw_result = test_match.group('result')
|
|
test_cases.append({
|
|
'test': f'{current_target or "unknown"}::{test_match.group("name")}',
|
|
'category': category_for_target(current_target or 'unknown'),
|
|
'status': {'ok': 'passed', 'FAILED': 'failed', 'ignored': 'skipped'}[raw_result],
|
|
})
|
|
continue
|
|
match = result_re.search(raw)
|
|
if not match:
|
|
continue
|
|
for key in counts:
|
|
counts[key] += int(match.group(key))
|
|
|
|
outcome = 'ok' if status == 0 else 'failed'
|
|
if status != 0 and not test_cases:
|
|
fallback_categories = sorted(test_categories) or ['uncategorized']
|
|
for category in fallback_categories:
|
|
test_cases.append({
|
|
'test': '__test_collection_failed__',
|
|
'category': category,
|
|
'status': 'failed',
|
|
})
|
|
summary = {
|
|
'suite': 'lesavka',
|
|
'branch': branch,
|
|
'commit': commit,
|
|
'build_url': build_url,
|
|
'outcome': outcome,
|
|
'exit_code': status,
|
|
'duration_seconds': duration_seconds,
|
|
'generated_at': datetime.now(timezone.utc).isoformat(),
|
|
'tests': counts,
|
|
}
|
|
summary_json_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + '\n', encoding='utf-8')
|
|
summary_txt_path.write_text(
|
|
'\n'.join([
|
|
f"lesavka test gate: {outcome}",
|
|
f"branch: {branch}",
|
|
f"commit: {commit}",
|
|
f"duration: {duration_seconds}s",
|
|
f"passed: {counts['passed']}",
|
|
f"failed: {counts['failed']}",
|
|
f"ignored: {counts['ignored']}",
|
|
f"filtered: {counts['filtered']}",
|
|
]) + '\n',
|
|
encoding='utf-8',
|
|
)
|
|
|
|
def label_value(value: str) -> str:
|
|
return value.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n')
|
|
|
|
labels = f'suite="lesavka",branch="{label_value(branch)}"'
|
|
build_labels = (
|
|
f'suite="lesavka",branch="{label_value(branch)}",commit="{label_value(commit)}",'
|
|
f'build_number="{label_value(build_number)}",jenkins_job="{label_value(jenkins_job)}"'
|
|
)
|
|
success = 1 if outcome == 'ok' else 0
|
|
failure = 1 - success
|
|
lines = [
|
|
'# HELP lesavka_test_gate_last_run_success Whether the latest lesavka cargo test gate run succeeded.',
|
|
'# TYPE lesavka_test_gate_last_run_success gauge',
|
|
f'lesavka_test_gate_last_run_success{{{labels}}} {success}',
|
|
'# HELP lesavka_test_gate_duration_seconds Duration of the latest lesavka cargo test gate run.',
|
|
'# TYPE lesavka_test_gate_duration_seconds gauge',
|
|
f'lesavka_test_gate_duration_seconds{{{labels}}} {duration_seconds}',
|
|
'# HELP lesavka_test_gate_tests Number of Rust tests reported by the latest lesavka test gate run.',
|
|
'# TYPE lesavka_test_gate_tests gauge',
|
|
]
|
|
for result, value in counts.items():
|
|
lines.append(f'lesavka_test_gate_tests{{{labels},result="{result}"}} {value}')
|
|
lines.extend([
|
|
'# HELP platform_quality_gate_build_info Build metadata for the latest lesavka gate run.',
|
|
'# TYPE platform_quality_gate_build_info gauge',
|
|
f'platform_quality_gate_build_info{{{build_labels}}} 1',
|
|
'# HELP platform_quality_gate_tests_total Test result counts from the latest lesavka gate run.',
|
|
'# TYPE platform_quality_gate_tests_total gauge',
|
|
f'platform_quality_gate_tests_total{{{labels},result="passed"}} {counts["passed"]}',
|
|
f'platform_quality_gate_tests_total{{{labels},result="failed"}} {counts["failed"]}',
|
|
f'platform_quality_gate_tests_total{{{labels},result="ignored"}} {counts["ignored"]}',
|
|
'# HELP platform_quality_gate_checks_total Check outcomes from the latest lesavka gate run.',
|
|
'# TYPE platform_quality_gate_checks_total gauge',
|
|
f'platform_quality_gate_checks_total{{{labels},check="tests",status="ok"}} {success}',
|
|
f'platform_quality_gate_checks_total{{{labels},check="tests",status="failed"}} {failure}',
|
|
'# HELP platform_quality_gate_test_case_result Per-test result from the latest lesavka gate run.',
|
|
'# TYPE platform_quality_gate_test_case_result gauge',
|
|
])
|
|
for case in test_cases:
|
|
for status_name in ('passed', 'failed', 'skipped'):
|
|
value = 1 if case['status'] == status_name else 0
|
|
case_labels = (
|
|
f'{build_labels},category="{label_value(case["category"])}",'
|
|
f'test="{label_value(case["test"])}",status="{status_name}"'
|
|
)
|
|
lines.append(f'platform_quality_gate_test_case_result{{{case_labels}}} {value}')
|
|
metrics_path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
|
PY
|
|
|
|
publish_metrics() {
|
|
if [[ -z "${PUSHGATEWAY_URL}" ]]; then
|
|
echo "Skipping test metrics publish: QUALITY_GATE_PUSHGATEWAY_URL is not set"
|
|
return 0
|
|
fi
|
|
|
|
curl --fail --silent --show-error \
|
|
--request PUT \
|
|
--data-binary @"${METRICS_FILE}" \
|
|
"${PUSHGATEWAY_URL%/}/metrics/job/${PUSHGATEWAY_JOB}/suite/lesavka/gate/tests"
|
|
}
|
|
|
|
publish_status=0
|
|
publish_metrics || publish_status=$?
|
|
|
|
if [[ "${status}" -ne 0 ]]; then
|
|
exit "${status}"
|
|
fi
|
|
exit "${publish_status}"
|