From 9103cd22f2fecc7c4fdd405dd831b4eff5c30a52 Mon Sep 17 00:00:00 2001 From: jenkins Date: Mon, 20 Apr 2026 10:49:54 -0300 Subject: [PATCH] ci(data-prepper): add retention and archive quality artifacts --- services/logging/Jenkinsfile.data-prepper | 174 ++++++++++++++++++++++ 1 file changed, 174 insertions(+) diff --git a/services/logging/Jenkinsfile.data-prepper b/services/logging/Jenkinsfile.data-prepper index 4f7c6a77..2c526d19 100644 --- a/services/logging/Jenkinsfile.data-prepper +++ b/services/logging/Jenkinsfile.data-prepper @@ -31,11 +31,21 @@ spec: """ } } + environment { + SUITE_NAME = 'data_prepper' + PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091' + QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json' + QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json' + } parameters { string(name: 'HARBOR_REPO', defaultValue: 'registry.bstein.dev/streaming/data-prepper', description: 'Docker repository for Data Prepper') string(name: 'IMAGE_TAG', defaultValue: '2.8.0', description: 'Image tag to publish') booleanParam(name: 'PUSH_LATEST', defaultValue: true, description: 'Also push the latest tag') } + options { + disableConcurrentBuilds() + buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120')) + } stages { stage('Checkout') { steps { @@ -44,6 +54,79 @@ spec: } } } + stage('Collect quality evidence') { + steps { + container('git') { + sh ''' + set -euo pipefail + apk add --no-cache curl jq >/dev/null 2>&1 || true + mkdir -p build + + sonar_report="${QUALITY_GATE_SONARQUBE_REPORT:-build/sonarqube-quality-gate.json}" + if [ ! -f "${sonar_report}" ]; then + if [ -n "${SONARQUBE_HOST_URL:-}" ] && [ -n "${SONARQUBE_PROJECT_KEY:-}" ]; then + host="${SONARQUBE_HOST_URL%/}" + query="$(printf '%s' "${SONARQUBE_PROJECT_KEY}" | sed 's/ /%20/g')" + sonar_ok=0 + if [ -n "${SONARQUBE_TOKEN:-}" ]; then + auth="$(printf '%s:' "${SONARQUBE_TOKEN}" | base64 | tr -d '\\n')" + if curl -fsS -H "Authorization: Basic ${auth}" "${host}/api/qualitygates/project_status?projectKey=${query}" > "${sonar_report}"; then + sonar_ok=1 + fi + else + if curl -fsS "${host}/api/qualitygates/project_status?projectKey=${query}" > "${sonar_report}"; then + sonar_ok=1 + fi + fi + if [ "${sonar_ok}" -ne 1 ]; then + cat > "${sonar_report}" < "${sonar_report}" < "${ironbank_report}" < "${ironbank_report}" </dev/null 2>&1 || true + suite="${SUITE_NAME}" + gateway="${PUSHGATEWAY_URL}" + status="${QUALITY_OUTCOME:-failed}" + fetch_counter() { + status_name="$1" + line="$(curl -fsS "${gateway}/metrics" 2>/dev/null | awk -v suite="${suite}" -v status="${status_name}" ' + /platform_quality_gate_runs_total/ { + if (index($0, "job=\\"platform-quality-ci\\"") && index($0, "suite=\\"" suite "\\"") && index($0, "status=\\"" status "\\"")) { + print $2 + exit + } + } + ' || true)" + [ -n "${line}" ] && printf '%s\n' "${line}" || printf '0\n' + } + ok_count="$(fetch_counter ok)" + failed_count="$(fetch_counter failed)" + if [ "${status}" = "ok" ]; then + ok_count=$((ok_count + 1)) + else + failed_count=$((failed_count + 1)) + fi + sonarqube_check="not_applicable" + if [ -f build/sonarqube-quality-gate.json ]; then + sonar_status="$(jq -r '.status // .projectStatus.status // .qualityGate.status // empty' build/sonarqube-quality-gate.json 2>/dev/null | tr '[:upper:]' '[:lower:]')" + if [ -n "${sonar_status}" ]; then + case "${sonar_status}" in + ok|pass|passed|success) sonarqube_check="ok" ;; + *) sonarqube_check="failed" ;; + esac + else + sonarqube_check="failed" + fi + fi + supply_chain_check="not_applicable" + if [ -f build/ironbank-compliance.json ]; then + compliant="$(jq -r '.compliant // empty' build/ironbank-compliance.json 2>/dev/null)" + if [ "${compliant}" = "true" ]; then + supply_chain_check="ok" + elif [ "${compliant}" = "false" ]; then + supply_chain_check="failed" + else + ironbank_status="$(jq -r '.status // .result // .compliance // empty' build/ironbank-compliance.json 2>/dev/null | tr '[:upper:]' '[:lower:]')" + case "${ironbank_status}" in + ok|pass|passed|success|compliant) supply_chain_check="ok" ;; + "") supply_chain_check="failed" ;; + *) supply_chain_check="failed" ;; + esac + fi + fi + gate_glue_check="ok" + if [ "${status}" != "ok" ]; then + gate_glue_check="failed" + fi + cat </dev/null || \ + echo "warning: metrics push failed for suite=${suite}" >&2 +# TYPE platform_quality_gate_runs_total counter +platform_quality_gate_runs_total{suite="${suite}",status="ok"} ${ok_count} +platform_quality_gate_runs_total{suite="${suite}",status="failed"} ${failed_count} +# TYPE data_prepper_quality_gate_tests_total gauge +data_prepper_quality_gate_tests_total{suite="${suite}",result="passed"} 0 +data_prepper_quality_gate_tests_total{suite="${suite}",result="failed"} 0 +data_prepper_quality_gate_tests_total{suite="${suite}",result="error"} 0 +data_prepper_quality_gate_tests_total{suite="${suite}",result="skipped"} 0 +# TYPE platform_quality_gate_workspace_line_coverage_percent gauge +platform_quality_gate_workspace_line_coverage_percent{suite="${suite}"} 0 +# TYPE platform_quality_gate_source_lines_over_500_total gauge +platform_quality_gate_source_lines_over_500_total{suite="${suite}"} 0 +# TYPE data_prepper_quality_gate_checks_total gauge +data_prepper_quality_gate_checks_total{suite="${suite}",check="tests",result="not_applicable"} 1 +data_prepper_quality_gate_checks_total{suite="${suite}",check="coverage",result="not_applicable"} 1 +data_prepper_quality_gate_checks_total{suite="${suite}",check="loc",result="not_applicable"} 1 +data_prepper_quality_gate_checks_total{suite="${suite}",check="docs_naming",result="not_applicable"} 1 +data_prepper_quality_gate_checks_total{suite="${suite}",check="gate_glue",result="${gate_glue_check}"} 1 +data_prepper_quality_gate_checks_total{suite="${suite}",check="sonarqube",result="${sonarqube_check}"} 1 +data_prepper_quality_gate_checks_total{suite="${suite}",check="supply_chain",result="${supply_chain_check}"} 1 +METRICS + ''' + } + archiveArtifacts artifacts: 'build/**/*.json,build/**/*.xml,build/**/*.txt,build/**/*.rc', allowEmptyArchive: true, fingerprint: true + } + } }