ci(jenkins): skip prep on test runs and retry flaky exec
This commit is contained in:
parent
9db28692bd
commit
1856e1b411
171
Jenkinsfile
vendored
171
Jenkinsfile
vendored
@ -91,6 +91,9 @@ spec:
|
||||
}
|
||||
}
|
||||
stage('Prep toolchain') {
|
||||
when {
|
||||
expression { return params.PUBLISH_IMAGES }
|
||||
}
|
||||
steps {
|
||||
container('builder') {
|
||||
sh '''
|
||||
@ -105,95 +108,98 @@ spec:
|
||||
stage('Run quality gate') {
|
||||
steps {
|
||||
container('tester') {
|
||||
sh '''
|
||||
set -eu
|
||||
apt-get update >/dev/null
|
||||
apt-get install -y --no-install-recommends jq python3 >/dev/null
|
||||
mkdir -p build
|
||||
set +e
|
||||
bash scripts/check.sh
|
||||
gate_rc=$?
|
||||
set -e
|
||||
if [ ! -f build/go-test.json ]; then
|
||||
: > build/go-test.json
|
||||
fi
|
||||
tests_total="$(jq -s '[.[] | select(.Test != null and (.Action=="pass" or .Action=="fail" or .Action=="skip"))] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_failed="$(jq -s '[.[] | select(.Test != null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_skipped="$(jq -s '[.[] | select(.Test != null and .Action=="skip")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_errors="$(jq -s '[.[] | select(.Test == null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_passed=$((tests_total - tests_failed - tests_skipped))
|
||||
if [ "${tests_passed}" -lt 0 ]; then
|
||||
tests_passed=0
|
||||
fi
|
||||
coverage_percent="$(jq -r '.coverage_percent // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
||||
over_500="$(jq -r '.source_lines_over_500 // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
||||
cat > build/test-summary.json <<EOF
|
||||
{
|
||||
"tests": ${tests_total},
|
||||
"passed": ${tests_passed},
|
||||
"failed": ${tests_failed},
|
||||
"errors": ${tests_errors},
|
||||
"skipped": ${tests_skipped},
|
||||
"coverage_percent": ${coverage_percent},
|
||||
"source_lines_over_500": ${over_500}
|
||||
}
|
||||
retry(2) {
|
||||
sh '''
|
||||
set -eu
|
||||
apt-get update >/dev/null
|
||||
apt-get install -y --no-install-recommends jq python3 >/dev/null
|
||||
mkdir -p build
|
||||
set +e
|
||||
bash scripts/check.sh
|
||||
gate_rc=$?
|
||||
set -e
|
||||
if [ ! -f build/go-test.json ]; then
|
||||
: > build/go-test.json
|
||||
fi
|
||||
tests_total="$(jq -s '[.[] | select(.Test != null and (.Action=="pass" or .Action=="fail" or .Action=="skip"))] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_failed="$(jq -s '[.[] | select(.Test != null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_skipped="$(jq -s '[.[] | select(.Test != null and .Action=="skip")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_errors="$(jq -s '[.[] | select(.Test == null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
||||
tests_passed=$((tests_total - tests_failed - tests_skipped))
|
||||
if [ "${tests_passed}" -lt 0 ]; then
|
||||
tests_passed=0
|
||||
fi
|
||||
coverage_percent="$(jq -r '.coverage_percent // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
||||
over_500="$(jq -r '.source_lines_over_500 // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
||||
cat > build/test-summary.json <<EOF
|
||||
{
|
||||
"tests": ${tests_total},
|
||||
"passed": ${tests_passed},
|
||||
"failed": ${tests_failed},
|
||||
"errors": ${tests_errors},
|
||||
"skipped": ${tests_skipped},
|
||||
"coverage_percent": ${coverage_percent},
|
||||
"source_lines_over_500": ${over_500}
|
||||
}
|
||||
EOF
|
||||
printf '%s\n' "${gate_rc}" > build/test.exitcode
|
||||
'''
|
||||
printf '%s\n' "${gate_rc}" > build/test.exitcode
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Publish test metrics') {
|
||||
steps {
|
||||
container('tester') {
|
||||
sh '''
|
||||
set -eu
|
||||
apt-get update >/dev/null
|
||||
apt-get install -y --no-install-recommends curl jq >/dev/null
|
||||
suite="${SUITE_NAME}"
|
||||
gateway="${PUSHGATEWAY_URL}"
|
||||
test_rc="$(cat build/test.exitcode 2>/dev/null || echo 1)"
|
||||
status="ok"
|
||||
if [ "${test_rc}" -ne 0 ]; then
|
||||
status="failed"
|
||||
fi
|
||||
fetch_counter() {
|
||||
status_name="$1"
|
||||
line="$(curl -fsS "${gateway}/metrics" 2>/dev/null | awk -v suite="${suite}" -v status="${status_name}" '
|
||||
/platform_quality_gate_runs_total/ {
|
||||
if (index($0, "job=\\"platform-quality-ci\\"") && index($0, "suite=\\"" suite "\\"") && index($0, "status=\\"" status "\\"")) {
|
||||
print $2
|
||||
exit
|
||||
retry(2) {
|
||||
sh '''
|
||||
set -eu
|
||||
apt-get update >/dev/null
|
||||
apt-get install -y --no-install-recommends curl jq >/dev/null
|
||||
suite="${SUITE_NAME}"
|
||||
gateway="${PUSHGATEWAY_URL}"
|
||||
test_rc="$(cat build/test.exitcode 2>/dev/null || echo 1)"
|
||||
status="ok"
|
||||
if [ "${test_rc}" -ne 0 ]; then
|
||||
status="failed"
|
||||
fi
|
||||
fetch_counter() {
|
||||
status_name="$1"
|
||||
line="$(curl -fsS "${gateway}/metrics" 2>/dev/null | awk -v suite="${suite}" -v status="${status_name}" '
|
||||
/platform_quality_gate_runs_total/ {
|
||||
if (index($0, "job=\\"platform-quality-ci\\"") && index($0, "suite=\\"" suite "\\"") && index($0, "status=\\"" status "\\"")) {
|
||||
print $2
|
||||
exit
|
||||
}
|
||||
}
|
||||
}
|
||||
' || true)"
|
||||
[ -n "${line}" ] && printf '%s\n' "${line}" || printf '0\n'
|
||||
}
|
||||
ok_count="$(fetch_counter ok)"
|
||||
failed_count="$(fetch_counter failed)"
|
||||
if [ "${status}" = "ok" ]; then
|
||||
ok_count=$((ok_count + 1))
|
||||
else
|
||||
failed_count=$((failed_count + 1))
|
||||
fi
|
||||
tests_passed="$(jq -r '.passed // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
tests_failed="$(jq -r '.failed // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
tests_errors="$(jq -r '.errors // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
tests_skipped="$(jq -r '.skipped // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
coverage_percent="$(jq -r '.coverage_percent // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
over_500="$(jq -r '.source_lines_over_500 // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
tests_check="failed"
|
||||
if [ "${test_rc}" -eq 0 ] && [ "${tests_failed}" -eq 0 ] && [ "${tests_errors}" -eq 0 ] && [ "${tests_passed}" -gt 0 ]; then
|
||||
tests_check="ok"
|
||||
fi
|
||||
coverage_check="$(awk -v value="${coverage_percent}" 'BEGIN { if ((value + 0) >= 95) { print "ok" } else { print "failed" } }')"
|
||||
loc_check="failed"
|
||||
if [ "${over_500}" -eq 0 ]; then
|
||||
loc_check="ok"
|
||||
fi
|
||||
if ! cat <<METRICS | curl -fsS --data-binary @- "${gateway}/metrics/job/platform-quality-ci/suite/${suite}" >/dev/null; then
|
||||
echo "warning: metrics push failed for suite=${suite}" >&2
|
||||
fi
|
||||
' || true)"
|
||||
[ -n "${line}" ] && printf '%s\n' "${line}" || printf '0\n'
|
||||
}
|
||||
ok_count="$(fetch_counter ok)"
|
||||
failed_count="$(fetch_counter failed)"
|
||||
if [ "${status}" = "ok" ]; then
|
||||
ok_count=$((ok_count + 1))
|
||||
else
|
||||
failed_count=$((failed_count + 1))
|
||||
fi
|
||||
tests_passed="$(jq -r '.passed // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
tests_failed="$(jq -r '.failed // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
tests_errors="$(jq -r '.errors // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
tests_skipped="$(jq -r '.skipped // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
coverage_percent="$(jq -r '.coverage_percent // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
over_500="$(jq -r '.source_lines_over_500 // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
||||
tests_check="failed"
|
||||
if [ "${test_rc}" -eq 0 ] && [ "${tests_failed}" -eq 0 ] && [ "${tests_errors}" -eq 0 ] && [ "${tests_passed}" -gt 0 ]; then
|
||||
tests_check="ok"
|
||||
fi
|
||||
coverage_check="$(awk -v value="${coverage_percent}" 'BEGIN { if ((value + 0) >= 95) { print "ok" } else { print "failed" } }')"
|
||||
loc_check="failed"
|
||||
if [ "${over_500}" -eq 0 ]; then
|
||||
loc_check="ok"
|
||||
fi
|
||||
if ! cat <<METRICS | curl -fsS --data-binary @- "${gateway}/metrics/job/platform-quality-ci/suite/${suite}" >/dev/null; then
|
||||
echo "warning: metrics push failed for suite=${suite}" >&2
|
||||
fi
|
||||
# TYPE platform_quality_gate_runs_total counter
|
||||
platform_quality_gate_runs_total{suite="${suite}",status="ok"} ${ok_count}
|
||||
platform_quality_gate_runs_total{suite="${suite}",status="failed"} ${failed_count}
|
||||
@ -213,7 +219,8 @@ soteria_quality_gate_checks_total{suite="${suite}",check="tests",result="${tests
|
||||
soteria_quality_gate_checks_total{suite="${suite}",check="coverage",result="${coverage_check}"} 1
|
||||
soteria_quality_gate_checks_total{suite="${suite}",check="loc",result="${loc_check}"} 1
|
||||
METRICS
|
||||
'''
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user