491 lines
19 KiB
Groovy
491 lines
19 KiB
Groovy
pipeline {
|
|
agent {
|
|
kubernetes {
|
|
defaultContainer 'tester'
|
|
yaml """
|
|
apiVersion: v1
|
|
kind: Pod
|
|
spec:
|
|
nodeSelector:
|
|
kubernetes.io/arch: arm64
|
|
node-role.kubernetes.io/worker: "true"
|
|
containers:
|
|
- name: builder
|
|
image: docker:27
|
|
command:
|
|
- cat
|
|
tty: true
|
|
env:
|
|
- name: DOCKER_HOST
|
|
value: tcp://localhost:2375
|
|
- name: DOCKER_TLS_CERTDIR
|
|
value: ""
|
|
volumeMounts:
|
|
- name: workspace-volume
|
|
mountPath: /home/jenkins/agent
|
|
- name: docker-config-writable
|
|
mountPath: /root/.docker
|
|
- name: harbor-config
|
|
mountPath: /docker-config
|
|
- name: tester
|
|
image: golang:1.25-bookworm
|
|
command:
|
|
- cat
|
|
tty: true
|
|
volumeMounts:
|
|
- name: workspace-volume
|
|
mountPath: /home/jenkins/agent
|
|
volumes:
|
|
- name: docker-config-writable
|
|
emptyDir: {}
|
|
- name: harbor-config
|
|
secret:
|
|
secretName: harbor-robot-pipeline
|
|
items:
|
|
- key: .dockerconfigjson
|
|
path: config.json
|
|
- name: workspace-volume
|
|
emptyDir: {}
|
|
"""
|
|
}
|
|
}
|
|
environment {
|
|
SUITE_NAME = 'soteria'
|
|
PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
|
|
SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
|
|
SONARQUBE_PROJECT_KEY = 'soteria'
|
|
QUALITY_GATE_SONARQUBE_ENFORCE = '0'
|
|
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
|
|
QUALITY_GATE_IRONBANK_ENFORCE = '0'
|
|
QUALITY_GATE_IRONBANK_REQUIRED = '0'
|
|
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
|
|
}
|
|
options {
|
|
disableConcurrentBuilds()
|
|
}
|
|
parameters {
|
|
booleanParam(
|
|
name: 'PUBLISH_IMAGES',
|
|
defaultValue: false,
|
|
description: 'Build and push the Soteria runtime image (enable for release runs).'
|
|
)
|
|
}
|
|
triggers {
|
|
pollSCM('H/5 * * * *')
|
|
}
|
|
stages {
|
|
stage('Checkout') {
|
|
steps {
|
|
checkout scm
|
|
}
|
|
}
|
|
stage('Prep toolchain') {
|
|
steps {
|
|
container('tester') {
|
|
sh '''
|
|
set -eu
|
|
apt-get update >/dev/null
|
|
apt-get install -y --no-install-recommends bash git jq curl python3 ripgrep >/dev/null
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Run quality gate') {
|
|
steps {
|
|
container('tester') {
|
|
sh '''
|
|
set -eu
|
|
apt-get update >/dev/null
|
|
apt-get install -y --no-install-recommends jq python3 ripgrep >/dev/null
|
|
mkdir -p build
|
|
python3 - <<'PY'
|
|
import base64
|
|
import json
|
|
import os
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/')
|
|
project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip()
|
|
token = os.getenv('SONARQUBE_TOKEN', '').strip()
|
|
sonar_report = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json')
|
|
|
|
payload = {"status": "ERROR", "note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY"}
|
|
if host and project_key:
|
|
query = urllib.parse.urlencode({"projectKey": project_key})
|
|
request = urllib.request.Request(f"{host}/api/qualitygates/project_status?{query}", method="GET")
|
|
if token:
|
|
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
|
|
request.add_header("Authorization", f"Basic {encoded}")
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=12) as response:
|
|
payload = json.loads(response.read().decode("utf-8"))
|
|
except Exception as exc: # noqa: BLE001
|
|
payload = {"status": "ERROR", "error": str(exc)}
|
|
Path(sonar_report).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
|
|
|
|
ironbank_report = Path(os.getenv('QUALITY_GATE_IRONBANK_REPORT', 'build/ironbank-compliance.json'))
|
|
if not ironbank_report.exists():
|
|
status = os.getenv('IRONBANK_COMPLIANCE_STATUS', '').strip()
|
|
compliant = os.getenv('IRONBANK_COMPLIANT', '').strip().lower()
|
|
ironbank_payload = {
|
|
"status": status or "unknown",
|
|
"compliant": compliant in {"1", "true", "yes", "on"} if compliant else None,
|
|
}
|
|
ironbank_payload = {k: v for k, v in ironbank_payload.items() if v is not None}
|
|
if "status" not in ironbank_payload:
|
|
ironbank_payload["status"] = "unknown"
|
|
ironbank_payload["note"] = (
|
|
"Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT "
|
|
"or write build/ironbank-compliance.json in image-building repos."
|
|
)
|
|
ironbank_report.parent.mkdir(parents=True, exist_ok=True)
|
|
ironbank_report.write_text(json.dumps(ironbank_payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
|
|
PY
|
|
set +e
|
|
bash scripts/check.sh
|
|
gate_rc=$?
|
|
set -e
|
|
if [ ! -f build/go-test.json ]; then
|
|
: > build/go-test.json
|
|
fi
|
|
tests_total="$(jq -s '[.[] | select(.Test != null and (.Action=="pass" or .Action=="fail" or .Action=="skip"))] | length' build/go-test.json 2>/dev/null || echo 0)"
|
|
tests_failed="$(jq -s '[.[] | select(.Test != null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
|
tests_skipped="$(jq -s '[.[] | select(.Test != null and .Action=="skip")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
|
tests_errors="$(jq -s '[.[] | select(.Test == null and .Action=="fail")] | length' build/go-test.json 2>/dev/null || echo 0)"
|
|
tests_passed=$((tests_total - tests_failed - tests_skipped))
|
|
if [ "${tests_passed}" -lt 0 ]; then
|
|
tests_passed=0
|
|
fi
|
|
coverage_percent="$(jq -r '.coverage_percent // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
|
over_500="$(jq -r '.source_lines_over_500 // 0' build/quality-summary.json 2>/dev/null || echo 0)"
|
|
cat > build/test-summary.json <<EOF
|
|
{
|
|
"tests": ${tests_total},
|
|
"passed": ${tests_passed},
|
|
"failed": ${tests_failed},
|
|
"errors": ${tests_errors},
|
|
"skipped": ${tests_skipped},
|
|
"coverage_percent": ${coverage_percent},
|
|
"source_lines_over_500": ${over_500}
|
|
}
|
|
EOF
|
|
printf '%s\n' "${gate_rc}" > build/test.exitcode
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Publish test metrics') {
|
|
steps {
|
|
container('tester') {
|
|
sh '''
|
|
set -eu
|
|
apt-get update >/dev/null
|
|
apt-get install -y --no-install-recommends curl jq >/dev/null
|
|
suite="${SUITE_NAME}"
|
|
gateway="${PUSHGATEWAY_URL}"
|
|
test_rc="$(cat build/test.exitcode 2>/dev/null || echo 1)"
|
|
status="ok"
|
|
if [ "${test_rc}" -ne 0 ]; then
|
|
status="failed"
|
|
fi
|
|
fetch_counter() {
|
|
status_name="$1"
|
|
line="$(curl -fsS "${gateway}/metrics" 2>/dev/null | awk -v suite="${suite}" -v status="${status_name}" '
|
|
/platform_quality_gate_runs_total/ {
|
|
if (index($0, "job=\\"platform-quality-ci\\"") && index($0, "suite=\\"" suite "\\"") && index($0, "status=\\"" status "\\"")) {
|
|
print $2
|
|
exit
|
|
}
|
|
}
|
|
' || true)"
|
|
[ -n "${line}" ] && printf '%s\n' "${line}" || printf '0\n'
|
|
}
|
|
ok_count="$(fetch_counter ok)"
|
|
failed_count="$(fetch_counter failed)"
|
|
if [ "${status}" = "ok" ]; then
|
|
ok_count=$((ok_count + 1))
|
|
else
|
|
failed_count=$((failed_count + 1))
|
|
fi
|
|
tests_passed="$(jq -r '.passed // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
|
tests_failed="$(jq -r '.failed // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
|
tests_errors="$(jq -r '.errors // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
|
tests_skipped="$(jq -r '.skipped // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
|
coverage_percent="$(jq -r '.coverage_percent // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
|
over_500="$(jq -r '.source_lines_over_500 // 0' build/test-summary.json 2>/dev/null || echo 0)"
|
|
tests_check="failed"
|
|
if [ "${test_rc}" -eq 0 ] && [ "${tests_failed}" -eq 0 ] && [ "${tests_errors}" -eq 0 ] && [ "${tests_passed}" -gt 0 ]; then
|
|
tests_check="ok"
|
|
fi
|
|
coverage_check="$(awk -v value="${coverage_percent}" 'BEGIN { if ((value + 0) >= 95) { print "ok" } else { print "failed" } }')"
|
|
loc_check="failed"
|
|
if [ "${over_500}" -eq 0 ]; then
|
|
loc_check="ok"
|
|
fi
|
|
docs_naming_check="$(cat build/docs-naming.status 2>/dev/null || echo failed)"
|
|
case "${docs_naming_check}" in
|
|
ok|pass|passed|success) docs_naming_check="ok" ;;
|
|
*) docs_naming_check="failed" ;;
|
|
esac
|
|
gate_glue_check="ok"
|
|
sonarqube_check="not_applicable"
|
|
if [ -f build/sonarqube-quality-gate.json ]; then
|
|
sonar_status="$(jq -r '.status // .projectStatus.status // .qualityGate.status // empty' build/sonarqube-quality-gate.json 2>/dev/null | tr '[:upper:]' '[:lower:]')"
|
|
if [ -n "${sonar_status}" ]; then
|
|
case "${sonar_status}" in
|
|
ok|pass|passed|success) sonarqube_check="ok" ;;
|
|
*) sonarqube_check="failed" ;;
|
|
esac
|
|
else
|
|
sonarqube_check="failed"
|
|
fi
|
|
fi
|
|
supply_chain_check="not_applicable"
|
|
if [ -f build/ironbank-compliance.json ]; then
|
|
compliant="$(jq -r '.compliant // empty' build/ironbank-compliance.json 2>/dev/null)"
|
|
if [ "${compliant}" = "true" ]; then
|
|
supply_chain_check="ok"
|
|
elif [ "${compliant}" = "false" ]; then
|
|
supply_chain_check="failed"
|
|
else
|
|
ironbank_status="$(jq -r '.status // .result // .compliance // empty' build/ironbank-compliance.json 2>/dev/null | tr '[:upper:]' '[:lower:]')"
|
|
case "${ironbank_status}" in
|
|
ok|pass|passed|success|compliant) supply_chain_check="ok" ;;
|
|
"") supply_chain_check="failed" ;;
|
|
*) supply_chain_check="failed" ;;
|
|
esac
|
|
fi
|
|
fi
|
|
test_case_metrics_file="build/test-case-metrics.prom"
|
|
: > "${test_case_metrics_file}"
|
|
if [ -s build/go-test.json ]; then
|
|
jq -r '
|
|
select(.Test != null and (.Action=="pass" or .Action=="fail" or .Action=="skip"))
|
|
| [.Test, (if .Action=="pass" then "passed" elif .Action=="fail" then "failed" else "skipped" end)]
|
|
| @tsv
|
|
' build/go-test.json 2>/dev/null \
|
|
| while IFS=$'\t' read -r test_name test_status; do
|
|
[ -n "${test_name}" ] || continue
|
|
escaped_test="$(printf '%s' "${test_name}" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -e ':a;N;$!ba;s/\n/\\n/g')"
|
|
printf 'platform_quality_gate_test_case_result{suite="%s",test="%s",status="%s"} 1\n' \
|
|
"${suite}" "${escaped_test}" "${test_status}" >> "${test_case_metrics_file}"
|
|
done
|
|
fi
|
|
cat > build/pushgateway-metrics.prom <<METRICS
|
|
# TYPE platform_quality_gate_runs_total counter
|
|
platform_quality_gate_runs_total{suite="${suite}",status="ok"} ${ok_count}
|
|
platform_quality_gate_runs_total{suite="${suite}",status="failed"} ${failed_count}
|
|
# TYPE soteria_quality_gate_tests_total gauge
|
|
soteria_quality_gate_tests_total{suite="${suite}",result="passed"} ${tests_passed}
|
|
soteria_quality_gate_tests_total{suite="${suite}",result="failed"} ${tests_failed}
|
|
soteria_quality_gate_tests_total{suite="${suite}",result="error"} ${tests_errors}
|
|
soteria_quality_gate_tests_total{suite="${suite}",result="skipped"} ${tests_skipped}
|
|
# TYPE soteria_quality_gate_coverage_percent gauge
|
|
soteria_quality_gate_coverage_percent{suite="${suite}"} ${coverage_percent}
|
|
# TYPE platform_quality_gate_workspace_line_coverage_percent gauge
|
|
platform_quality_gate_workspace_line_coverage_percent{suite="${suite}"} ${coverage_percent}
|
|
# TYPE platform_quality_gate_source_lines_over_500_total gauge
|
|
platform_quality_gate_source_lines_over_500_total{suite="${suite}"} ${over_500}
|
|
# TYPE soteria_quality_gate_checks_total gauge
|
|
soteria_quality_gate_checks_total{suite="${suite}",check="tests",result="${tests_check}"} 1
|
|
soteria_quality_gate_checks_total{suite="${suite}",check="coverage",result="${coverage_check}"} 1
|
|
soteria_quality_gate_checks_total{suite="${suite}",check="loc",result="${loc_check}"} 1
|
|
soteria_quality_gate_checks_total{suite="${suite}",check="docs_naming",result="${docs_naming_check}"} 1
|
|
soteria_quality_gate_checks_total{suite="${suite}",check="gate_glue",result="${gate_glue_check}"} 1
|
|
soteria_quality_gate_checks_total{suite="${suite}",check="sonarqube",result="${sonarqube_check}"} 1
|
|
soteria_quality_gate_checks_total{suite="${suite}",check="supply_chain",result="${supply_chain_check}"} 1
|
|
METRICS
|
|
if [ -s "${test_case_metrics_file}" ]; then
|
|
{
|
|
echo "# TYPE platform_quality_gate_test_case_result gauge"
|
|
cat "${test_case_metrics_file}"
|
|
} >> build/pushgateway-metrics.prom
|
|
fi
|
|
if ! curl -fsS -X PUT --data-binary @build/pushgateway-metrics.prom "${gateway}/metrics/job/platform-quality-ci/suite/${suite}" >/dev/null; then
|
|
echo "warning: metrics push failed for suite=${suite}" >&2
|
|
fi
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Enforce quality gate') {
|
|
steps {
|
|
container('tester') {
|
|
sh '''
|
|
set -eu
|
|
test_rc="$(cat build/test.exitcode 2>/dev/null || echo 1)"
|
|
fail=0
|
|
if [ "${test_rc}" -ne 0 ]; then
|
|
echo "quality gate failed with rc=${test_rc}" >&2
|
|
fail=1
|
|
fi
|
|
|
|
enabled() {
|
|
case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in
|
|
1|true|yes|on) return 0 ;;
|
|
*) return 1 ;;
|
|
esac
|
|
}
|
|
|
|
if enabled "${QUALITY_GATE_SONARQUBE_ENFORCE:-1}"; then
|
|
sonar_status="$(python3 - <<'PY'
|
|
import json
|
|
from pathlib import Path
|
|
|
|
path = Path("build/sonarqube-quality-gate.json")
|
|
if not path.exists():
|
|
print("missing")
|
|
raise SystemExit(0)
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
except Exception: # noqa: BLE001
|
|
print("error")
|
|
raise SystemExit(0)
|
|
status = (payload.get("status") or payload.get("projectStatus", {}).get("status") or payload.get("qualityGate", {}).get("status") or "").strip().lower()
|
|
print(status or "missing")
|
|
PY
|
|
)"
|
|
case "${sonar_status}" in
|
|
ok|pass|passed|success) ;;
|
|
*)
|
|
echo "sonarqube gate failed: ${sonar_status}" >&2
|
|
fail=1
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
|
|
if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
|
|
ironbank_required=1
|
|
fi
|
|
if enabled "${QUALITY_GATE_IRONBANK_ENFORCE:-1}"; then
|
|
supply_status="$(python3 - <<'PY'
|
|
import json
|
|
from pathlib import Path
|
|
|
|
path = Path("build/ironbank-compliance.json")
|
|
if not path.exists():
|
|
print("missing")
|
|
raise SystemExit(0)
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
except Exception: # noqa: BLE001
|
|
print("error")
|
|
raise SystemExit(0)
|
|
compliant = payload.get("compliant")
|
|
if compliant is True:
|
|
print("ok")
|
|
elif compliant is False:
|
|
print("failed")
|
|
else:
|
|
status = str(payload.get("status") or payload.get("result") or payload.get("compliance") or "").strip().lower()
|
|
print(status or "missing")
|
|
PY
|
|
)"
|
|
case "${supply_status}" in
|
|
ok|pass|passed|success|compliant) ;;
|
|
not_applicable|na|n/a)
|
|
if enabled "${ironbank_required}"; then
|
|
echo "supply chain gate required but status=${supply_status}" >&2
|
|
fail=1
|
|
fi
|
|
;;
|
|
*)
|
|
if enabled "${ironbank_required}"; then
|
|
echo "supply chain gate failed: ${supply_status}" >&2
|
|
fail=1
|
|
else
|
|
echo "supply chain gate not passing (${supply_status}) but not required for this run" >&2
|
|
fi
|
|
;;
|
|
esac
|
|
fi
|
|
|
|
exit "${fail}"
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Compute version') {
|
|
when {
|
|
expression { return params.PUBLISH_IMAGES }
|
|
}
|
|
steps {
|
|
container('builder') {
|
|
script {
|
|
sh 'git config --global --add safe.directory /home/jenkins/agent/workspace/Soteria'
|
|
def semver = sh(returnStdout: true, script: 'git describe --tags --exact-match || true').trim()
|
|
if (!semver) {
|
|
semver = sh(returnStdout: true, script: 'git rev-list --count HEAD').trim()
|
|
semver = "0.1.0-${semver}"
|
|
}
|
|
sh "echo SEMVER=${semver} > build.env"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
stage('Buildx setup') {
|
|
when {
|
|
expression { return params.PUBLISH_IMAGES }
|
|
}
|
|
steps {
|
|
container('builder') {
|
|
sh '''
|
|
set -eu
|
|
seq 1 10 | while read _; do
|
|
docker info && break || sleep 2
|
|
done
|
|
BUILDER_NAME="soteria-${BUILD_NUMBER}"
|
|
docker buildx rm "${BUILDER_NAME}" >/dev/null 2>&1 || true
|
|
docker buildx create --name "${BUILDER_NAME}" --driver docker-container --bootstrap --use
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
stage('Build & push image') {
|
|
when {
|
|
expression { return params.PUBLISH_IMAGES }
|
|
}
|
|
steps {
|
|
container('builder') {
|
|
withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD')]) {
|
|
sh '''
|
|
set -eu
|
|
VERSION_TAG=$(cut -d= -f2 build.env)
|
|
printf '%s' "${HARBOR_PASSWORD}" | docker login registry.bstein.dev -u "${HARBOR_USERNAME}" --password-stdin
|
|
docker buildx build --platform linux/arm64 \
|
|
--provenance=false \
|
|
--tag registry.bstein.dev/bstein/soteria:${VERSION_TAG} \
|
|
--tag registry.bstein.dev/bstein/soteria:latest \
|
|
--push .
|
|
'''
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
post {
|
|
always {
|
|
script {
|
|
try {
|
|
if (fileExists('build.env')) {
|
|
def env = readProperties file: 'build.env'
|
|
echo "Build complete for ${env.SEMVER}"
|
|
}
|
|
} catch (Throwable err) {
|
|
echo "post workspace unavailable for build.env: ${err.class.simpleName}"
|
|
}
|
|
}
|
|
script {
|
|
try {
|
|
archiveArtifacts artifacts: 'build/*', allowEmptyArchive: true
|
|
} catch (Throwable err) {
|
|
echo "archive skipped: ${err.class.simpleName}"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|