bstein/soteria
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ci(jenkins): skip prep on test runs and retry flaky exec

Browse Source
This commit is contained in:
Brad Stein 2026-04-18 17:29:50 -03:00
parent 9db28692bd
commit 1856e1b411
1 changed files with 89 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
Jenkinsfile vendored
View File

@ -91,6 +91,9 @@ spec:
} }
} }
stage('Prep toolchain') { stage('Prep toolchain') {
when {
expression { return params.PUBLISH_IMAGES }
}
steps { steps {
container('builder') { container('builder') {
sh ''' sh '''
@ -105,95 +108,98 @@ spec:
stage('Run quality gate') { stage('Run quality gate') {
steps { steps {
container('tester') { container('tester') {
sh ''' retry(2) {
set -eu sh '''
apt-get update >/dev/null set -eu
apt-get install -y --no-install-recommends jq python3 >/dev/null apt-get update >/dev/null
mkdir -p build apt-get install -y --no-install-recommends jq python3 >/dev/null
set +e mkdir -p build
bash scripts/check.sh set +e
gate_rc=$? bash scripts/check.sh
set -e gate_rc=$?
if [ ! -f build/go-test.json ]; then set -e
: > build/go-test.json if [ ! -f build/go-test.json ]; then
fi : > build/go-test.json
tests_total="$(jq -s '[.[] | select(.Test != null and (.Action=="pass" or .Action=="fail" or .Action=="skip"))] | length' build/go-test.json 2>/dev/null || echo 0)" fi
tests_failed="$(jq -s '[.[] | select(.Test != null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)" tests_total="$(jq -s '[.[] | select(.Test != null and (.Action=="pass" or .Action=="fail" or .Action=="skip"))] | length' build/go-test.json 2>/dev/null || echo 0)"
tests_skipped="$(jq -s '[.[] | select(.Test != null and .Action=="skip")] | length' build/go-test.json 2>/dev/null || echo 0)" tests_failed="$(jq -s '[.[] | select(.Test != null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
tests_errors="$(jq -s '[.[] | select(.Test == null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)" tests_skipped="$(jq -s '[.[] | select(.Test != null and .Action=="skip")] | length' build/go-test.json 2>/dev/null || echo 0)"
tests_passed=$((tests_total - tests_failed - tests_skipped)) tests_errors="$(jq -s '[.[] | select(.Test == null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
if [ "${tests_passed}" -lt 0 ]; then tests_passed=$((tests_total - tests_failed - tests_skipped))
tests_passed=0 if [ "${tests_passed}" -lt 0 ]; then
fi tests_passed=0
coverage_percent="$(jq -r '.coverage_percent // 0' build/quality-summary.json 2>/dev/null || echo 0)" fi
over_500="$(jq -r '.source_lines_over_500 // 0' build/quality-summary.json 2>/dev/null || echo 0)" coverage_percent="$(jq -r '.coverage_percent // 0' build/quality-summary.json 2>/dev/null || echo 0)"
cat > build/test-summary.json <<EOF over_500="$(jq -r '.source_lines_over_500 // 0' build/quality-summary.json 2>/dev/null || echo 0)"
{ cat > build/test-summary.json <<EOF
"tests": ${tests_total}, {
"passed": ${tests_passed}, "tests": ${tests_total},
"failed": ${tests_failed}, "passed": ${tests_passed},
"errors": ${tests_errors}, "failed": ${tests_failed},
"skipped": ${tests_skipped}, "errors": ${tests_errors},
"coverage_percent": ${coverage_percent}, "skipped": ${tests_skipped},
"source_lines_over_500": ${over_500} "coverage_percent": ${coverage_percent},
} "source_lines_over_500": ${over_500}
}
EOF EOF
printf '%s\n' "${gate_rc}" > build/test.exitcode printf '%s\n' "${gate_rc}" > build/test.exitcode
''' '''
}
} }
} }
} }
stage('Publish test metrics') { stage('Publish test metrics') {
steps { steps {
container('tester') { container('tester') {
sh ''' retry(2) {
set -eu sh '''
apt-get update >/dev/null set -eu
apt-get install -y --no-install-recommends curl jq >/dev/null apt-get update >/dev/null
suite="${SUITE_NAME}" apt-get install -y --no-install-recommends curl jq >/dev/null
gateway="${PUSHGATEWAY_URL}" suite="${SUITE_NAME}"
test_rc="$(cat build/test.exitcode 2>/dev/null || echo 1)" gateway="${PUSHGATEWAY_URL}"
status="ok" test_rc="$(cat build/test.exitcode 2>/dev/null || echo 1)"
if [ "${test_rc}" -ne 0 ]; then status="ok"
status="failed" if [ "${test_rc}" -ne 0 ]; then
fi status="failed"
fetch_counter() { fi
status_name="$1" fetch_counter() {
line="$(curl -fsS "${gateway}/metrics" 2>/dev/null | awk -v suite="${suite}" -v status="${status_name}" ' status_name="$1"
/platform_quality_gate_runs_total/ { line="$(curl -fsS "${gateway}/metrics" 2>/dev/null | awk -v suite="${suite}" -v status="${status_name}" '
if (index($0, "job=\\"platform-quality-ci\\"") && index($0, "suite=\\"" suite "\\"") && index($0, "status=\\"" status "\\"")) { /platform_quality_gate_runs_total/ {
print $2 if (index($0, "job=\\"platform-quality-ci\\"") && index($0, "suite=\\"" suite "\\"") && index($0, "status=\\"" status "\\"")) {
exit print $2
exit
}
} }
} ' || true)"
' || true)" [ -n "${line}" ] && printf '%s\n' "${line}" || printf '0\n'
[ -n "${line}" ] && printf '%s\n' "${line}" || printf '0\n' }
} ok_count="$(fetch_counter ok)"
ok_count="$(fetch_counter ok)" failed_count="$(fetch_counter failed)"
failed_count="$(fetch_counter failed)" if [ "${status}" = "ok" ]; then
if [ "${status}" = "ok" ]; then ok_count=$((ok_count + 1))
ok_count=$((ok_count + 1)) else
else failed_count=$((failed_count + 1))
failed_count=$((failed_count + 1)) fi
fi tests_passed="$(jq -r '.passed // 0' build/test-summary.json 2>/dev/null || echo 0)"
tests_passed="$(jq -r '.passed // 0' build/test-summary.json 2>/dev/null || echo 0)" tests_failed="$(jq -r '.failed // 0' build/test-summary.json 2>/dev/null || echo 0)"
tests_failed="$(jq -r '.failed // 0' build/test-summary.json 2>/dev/null || echo 0)" tests_errors="$(jq -r '.errors // 0' build/test-summary.json 2>/dev/null || echo 0)"
tests_errors="$(jq -r '.errors // 0' build/test-summary.json 2>/dev/null || echo 0)" tests_skipped="$(jq -r '.skipped // 0' build/test-summary.json 2>/dev/null || echo 0)"
tests_skipped="$(jq -r '.skipped // 0' build/test-summary.json 2>/dev/null || echo 0)" coverage_percent="$(jq -r '.coverage_percent // 0' build/test-summary.json 2>/dev/null || echo 0)"
coverage_percent="$(jq -r '.coverage_percent // 0' build/test-summary.json 2>/dev/null || echo 0)" over_500="$(jq -r '.source_lines_over_500 // 0' build/test-summary.json 2>/dev/null || echo 0)"
over_500="$(jq -r '.source_lines_over_500 // 0' build/test-summary.json 2>/dev/null || echo 0)" tests_check="failed"
tests_check="failed" if [ "${test_rc}" -eq 0 ] && [ "${tests_failed}" -eq 0 ] && [ "${tests_errors}" -eq 0 ] && [ "${tests_passed}" -gt 0 ]; then
if [ "${test_rc}" -eq 0 ] && [ "${tests_failed}" -eq 0 ] && [ "${tests_errors}" -eq 0 ] && [ "${tests_passed}" -gt 0 ]; then tests_check="ok"
tests_check="ok" fi
fi coverage_check="$(awk -v value="${coverage_percent}" 'BEGIN { if ((value + 0) >= 95) { print "ok" } else { print "failed" } }')"
coverage_check="$(awk -v value="${coverage_percent}" 'BEGIN { if ((value + 0) >= 95) { print "ok" } else { print "failed" } }')" loc_check="failed"
loc_check="failed" if [ "${over_500}" -eq 0 ]; then
if [ "${over_500}" -eq 0 ]; then loc_check="ok"
loc_check="ok" fi
fi if ! cat <<METRICS | curl -fsS --data-binary @- "${gateway}/metrics/job/platform-quality-ci/suite/${suite}" >/dev/null; then
if ! cat <<METRICS | curl -fsS --data-binary @- "${gateway}/metrics/job/platform-quality-ci/suite/${suite}" >/dev/null; then echo "warning: metrics push failed for suite=${suite}" >&2
echo "warning: metrics push failed for suite=${suite}" >&2 fi
fi
# TYPE platform_quality_gate_runs_total counter # TYPE platform_quality_gate_runs_total counter
platform_quality_gate_runs_total{suite="${suite}",status="ok"} ${ok_count} platform_quality_gate_runs_total{suite="${suite}",status="ok"} ${ok_count}
platform_quality_gate_runs_total{suite="${suite}",status="failed"} ${failed_count} platform_quality_gate_runs_total{suite="${suite}",status="failed"} ${failed_count}
@ -213,7 +219,8 @@ soteria_quality_gate_checks_total{suite="${suite}",check="tests",result="${tests
soteria_quality_gate_checks_total{suite="${suite}",check="coverage",result="${coverage_check}"} 1 soteria_quality_gate_checks_total{suite="${suite}",check="coverage",result="${coverage_check}"} 1
soteria_quality_gate_checks_total{suite="${suite}",check="loc",result="${loc_check}"} 1 soteria_quality_gate_checks_total{suite="${suite}",check="loc",result="${loc_check}"} 1
METRICS METRICS
''' '''
}
} }
} }
} }