ci: add sonar/supply evidence collection and checks metrics
This commit is contained in:
parent
dfbe46b0f8
commit
e4014aba1c
219
Jenkinsfile
vendored
219
Jenkinsfile
vendored
@ -72,6 +72,8 @@ spec:
|
||||
PYTHONUNBUFFERED = '1'
|
||||
SUITE_NAME = 'atlasbot'
|
||||
PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
|
||||
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
|
||||
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
|
||||
}
|
||||
stages {
|
||||
stage('Checkout') {
|
||||
@ -79,6 +81,71 @@ spec:
|
||||
checkout scm
|
||||
}
|
||||
}
|
||||
stage('Collect SonarQube evidence') {
|
||||
steps {
|
||||
container('tester') {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
mkdir -p build
|
||||
python3 - <<'PY'
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/')
|
||||
project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip()
|
||||
token = os.getenv('SONARQUBE_TOKEN', '').strip()
|
||||
report_path = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json')
|
||||
payload = {"status": "ERROR", "note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY"}
|
||||
if host and project_key:
|
||||
query = urllib.parse.urlencode({"projectKey": project_key})
|
||||
request = urllib.request.Request(f"{host}/api/qualitygates/project_status?{query}", method="GET")
|
||||
if token:
|
||||
encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
|
||||
request.add_header("Authorization", f"Basic {encoded}")
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=12) as response:
|
||||
payload = json.loads(response.read().decode("utf-8"))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
payload = {"status": "ERROR", "error": str(exc)}
|
||||
with open(report_path, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, indent=2, sort_keys=True)
|
||||
handle.write("\\n")
|
||||
PY
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Collect Supply Chain evidence') {
|
||||
steps {
|
||||
container('tester') {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
mkdir -p build
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
report_path = Path(os.getenv('QUALITY_GATE_IRONBANK_REPORT', 'build/ironbank-compliance.json'))
|
||||
if report_path.exists():
|
||||
raise SystemExit(0)
|
||||
status = os.getenv('IRONBANK_COMPLIANCE_STATUS', '').strip()
|
||||
compliant = os.getenv('IRONBANK_COMPLIANT', '').strip().lower()
|
||||
payload = {"status": status or "unknown", "compliant": compliant in {"1", "true", "yes", "on"} if compliant else None}
|
||||
payload = {k: v for k, v in payload.items() if v is not None}
|
||||
if "status" not in payload:
|
||||
payload["status"] = "unknown"
|
||||
payload["note"] = "Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT or write build/ironbank-compliance.json in image-building repos."
|
||||
report_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
report_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
|
||||
PY
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Prep toolchain') {
|
||||
steps {
|
||||
container('builder') {
|
||||
@ -133,31 +200,65 @@ spec:
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Unit tests') {
|
||||
stage('Run quality gate') {
|
||||
steps {
|
||||
container('builder') {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
mkdir -p build
|
||||
docker buildx build --platform linux/arm64 --target test --load -t atlasbot-test .
|
||||
docker run --rm -v "$PWD/build:/out" atlasbot-test \
|
||||
python -m ruff check atlasbot --select E9,F63,F7,F82
|
||||
docker run --rm -v "$PWD/build:/out" atlasbot-test \
|
||||
python -m slipcover --json --out /out/coverage.json --source atlasbot --fail-under 90 \
|
||||
-m pytest -q --junitxml /out/junit.xml
|
||||
case "${TEST_PLATFORM:-$(uname -m)}" in
|
||||
x86_64|amd64|linux/amd64)
|
||||
TEST_PLATFORM_RESOLVED=linux/amd64
|
||||
;;
|
||||
aarch64|arm64|linux/arm64)
|
||||
TEST_PLATFORM_RESOLVED=linux/arm64
|
||||
;;
|
||||
*)
|
||||
TEST_PLATFORM_RESOLVED=linux/amd64
|
||||
;;
|
||||
esac
|
||||
set +e
|
||||
docker buildx build --platform "${TEST_PLATFORM_RESOLVED}" --target test --load -t atlasbot-test . \
|
||||
&& docker run --rm -v "$PWD/build:/out" atlasbot-test \
|
||||
python -m ruff check atlasbot testing scripts --select E,F,W,B,C90,I,PLR,RUF,SIM,UP,ARG --ignore E501 \
|
||||
&& docker run --rm -v "$PWD/build:/out" atlasbot-test \
|
||||
python scripts/check_file_sizes.py --root atlasbot --max-lines 500 \
|
||||
&& docker run --rm -v "$PWD/build:/out" atlasbot-test \
|
||||
python scripts/check_docstrings.py --root atlasbot \
|
||||
&& docker run --rm -v "$PWD/build:/out" atlasbot-test \
|
||||
python -m slipcover --json --out /out/coverage.json --source atlasbot --fail-under 95 \
|
||||
-m pytest -q --junitxml /out/junit.xml \
|
||||
&& docker run --rm -v "$PWD/build:/out" atlasbot-test \
|
||||
python scripts/check_coverage.py /out/coverage.json --root atlasbot --threshold 95
|
||||
gate_rc=$?
|
||||
set -e
|
||||
printf '%s\n' "${gate_rc}" > build/quality-gate.rc
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Publish test metrics') {
|
||||
steps {
|
||||
container('builder') {
|
||||
container('tester') {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
docker run --rm -v "$PWD/build:/out" \
|
||||
-e JUNIT_PATH=/out/junit.xml \
|
||||
-e COVERAGE_PATH=/out/coverage.json \
|
||||
atlasbot-test python scripts/publish_test_metrics.py
|
||||
export JUNIT_PATH='build/junit.xml'
|
||||
export COVERAGE_PATH='build/coverage.json'
|
||||
export SOURCE_ROOT='atlasbot'
|
||||
export QUALITY_GATE_RC_PATH='build/quality-gate.rc'
|
||||
python scripts/publish_test_metrics.py || true
|
||||
'''
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Enforce quality gate') {
|
||||
steps {
|
||||
container('tester') {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
test "$(cat build/quality-gate.rc 2>/dev/null || echo 1)" -eq 0
|
||||
'''
|
||||
}
|
||||
}
|
||||
@ -179,101 +280,11 @@ spec:
|
||||
}
|
||||
}
|
||||
post {
|
||||
success {
|
||||
container('tester') {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
export QUALITY_STATUS=ok
|
||||
python - <<'PY'
|
||||
import os
|
||||
import re
|
||||
import urllib.request
|
||||
|
||||
suite = os.environ.get("SUITE_NAME", "atlasbot")
|
||||
status = os.environ.get("QUALITY_STATUS", "failed")
|
||||
gateway = os.environ.get("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091").rstrip("/")
|
||||
text = urllib.request.urlopen(f"{gateway}/metrics", timeout=10).read().decode("utf-8", errors="replace")
|
||||
|
||||
def counter(name: str) -> float:
|
||||
pattern = re.compile(
|
||||
rf'^platform_quality_gate_runs_total\\{{[^}}]*job="platform-quality-ci"[^}}]*suite="{re.escape(suite)}"[^}}]*status="{name}"[^}}]*\\}}\\s+([0-9]+(?:\\.[0-9]+)?)$',
|
||||
re.M,
|
||||
)
|
||||
match = pattern.search(text)
|
||||
return float(match.group(1)) if match else 0.0
|
||||
|
||||
ok = counter("ok")
|
||||
failed = counter("failed")
|
||||
if status == "ok":
|
||||
ok += 1
|
||||
else:
|
||||
failed += 1
|
||||
payload = (
|
||||
"# TYPE platform_quality_gate_runs_total counter\\n"
|
||||
f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {int(ok)}\\n'
|
||||
f'platform_quality_gate_runs_total{{suite="{suite}",status="failed"}} {int(failed)}\\n'
|
||||
)
|
||||
req = urllib.request.Request(
|
||||
f"{gateway}/metrics/job/platform-quality-ci/suite/{suite}",
|
||||
data=payload.encode("utf-8"),
|
||||
method="POST",
|
||||
headers={"Content-Type": "text/plain"},
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=10).read()
|
||||
PY
|
||||
'''
|
||||
}
|
||||
}
|
||||
failure {
|
||||
container('tester') {
|
||||
sh '''
|
||||
set -euo pipefail
|
||||
export QUALITY_STATUS=failed
|
||||
python - <<'PY'
|
||||
import os
|
||||
import re
|
||||
import urllib.request
|
||||
|
||||
suite = os.environ.get("SUITE_NAME", "atlasbot")
|
||||
status = os.environ.get("QUALITY_STATUS", "failed")
|
||||
gateway = os.environ.get("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091").rstrip("/")
|
||||
text = urllib.request.urlopen(f"{gateway}/metrics", timeout=10).read().decode("utf-8", errors="replace")
|
||||
|
||||
def counter(name: str) -> float:
|
||||
pattern = re.compile(
|
||||
rf'^platform_quality_gate_runs_total\\{{[^}}]*job="platform-quality-ci"[^}}]*suite="{re.escape(suite)}"[^}}]*status="{name}"[^}}]*\\}}\\s+([0-9]+(?:\\.[0-9]+)?)$',
|
||||
re.M,
|
||||
)
|
||||
match = pattern.search(text)
|
||||
return float(match.group(1)) if match else 0.0
|
||||
|
||||
ok = counter("ok")
|
||||
failed = counter("failed")
|
||||
if status == "ok":
|
||||
ok += 1
|
||||
else:
|
||||
failed += 1
|
||||
payload = (
|
||||
"# TYPE platform_quality_gate_runs_total counter\\n"
|
||||
f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {int(ok)}\\n'
|
||||
f'platform_quality_gate_runs_total{{suite="{suite}",status="failed"}} {int(failed)}\\n'
|
||||
)
|
||||
req = urllib.request.Request(
|
||||
f"{gateway}/metrics/job/platform-quality-ci/suite/{suite}",
|
||||
data=payload.encode("utf-8"),
|
||||
method="POST",
|
||||
headers={"Content-Type": "text/plain"},
|
||||
)
|
||||
urllib.request.urlopen(req, timeout=10).read()
|
||||
PY
|
||||
'''
|
||||
}
|
||||
}
|
||||
always {
|
||||
script {
|
||||
if (fileExists('build.env')) {
|
||||
def env = readProperties file: 'build.env'
|
||||
echo "Build complete for ${env.SEMVER}"
|
||||
def envFile = readProperties file: 'build.env'
|
||||
echo "Build complete for ${envFile.SEMVER}"
|
||||
}
|
||||
}
|
||||
archiveArtifacts artifacts: 'build/*', allowEmptyArchive: true
|
||||
|
||||
95
scripts/publish_test_metrics.py
Normal file → Executable file
95
scripts/publish_test_metrics.py
Normal file → Executable file
@ -8,6 +8,8 @@ Outputs:
|
||||
- platform_quality_gate_runs_total{suite="atlasbot",status="ok|failed"}
|
||||
- atlasbot_quality_gate_tests_total{suite="atlasbot",result=*}
|
||||
- atlasbot_quality_gate_coverage_percent{suite="atlasbot"}
|
||||
- platform_quality_gate_workspace_line_coverage_percent{suite="atlasbot"}
|
||||
- platform_quality_gate_source_lines_over_500_total{suite="atlasbot"}
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@ -18,6 +20,8 @@ import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
QUALITY_SUCCESS_STATES = {"ok", "pass", "passed", "success", "compliant"}
|
||||
|
||||
|
||||
def _as_int(node: ET.Element, name: str) -> int:
|
||||
raw = node.attrib.get(name) or "0"
|
||||
@ -61,6 +65,70 @@ def _load_coverage_percent(path: Path) -> float:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _load_gate_rc(path: Path) -> int | None:
|
||||
if not path.exists():
|
||||
return None
|
||||
raw = path.read_text(encoding="utf-8").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return int(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _count_source_lines_over_500(root: Path) -> int:
|
||||
if not root.exists():
|
||||
return 0
|
||||
over = 0
|
||||
for path in root.rglob("*.py"):
|
||||
if not path.is_file():
|
||||
continue
|
||||
line_count = sum(1 for _ in path.open("r", encoding="utf-8"))
|
||||
if line_count > 500:
|
||||
over += 1
|
||||
return over
|
||||
|
||||
|
||||
def _load_json(path: Path) -> dict | None:
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
return None
|
||||
return payload if isinstance(payload, dict) else None
|
||||
|
||||
|
||||
def _sonarqube_check_status(build_dir: Path) -> str:
|
||||
report = _load_json(Path(os.getenv("QUALITY_GATE_SONARQUBE_REPORT", str(build_dir / "sonarqube-quality-gate.json"))))
|
||||
if not report:
|
||||
return "not_applicable"
|
||||
status_candidates = [
|
||||
report.get("status"),
|
||||
((report.get("projectStatus") or {}).get("status") if isinstance(report.get("projectStatus"), dict) else None),
|
||||
((report.get("qualityGate") or {}).get("status") if isinstance(report.get("qualityGate"), dict) else None),
|
||||
]
|
||||
for value in status_candidates:
|
||||
if isinstance(value, str):
|
||||
return "ok" if value.strip().lower() in QUALITY_SUCCESS_STATES else "failed"
|
||||
return "failed"
|
||||
|
||||
|
||||
def _supply_chain_check_status(build_dir: Path) -> str:
|
||||
report = _load_json(Path(os.getenv("QUALITY_GATE_IRONBANK_REPORT", str(build_dir / "ironbank-compliance.json"))))
|
||||
if not report:
|
||||
return "not_applicable"
|
||||
compliant = report.get("compliant")
|
||||
if isinstance(compliant, bool):
|
||||
return "ok" if compliant else "failed"
|
||||
status_candidates = [report.get("status"), report.get("result"), report.get("compliance")]
|
||||
for value in status_candidates:
|
||||
if isinstance(value, str):
|
||||
return "ok" if value.strip().lower() in QUALITY_SUCCESS_STATES else "failed"
|
||||
return "failed"
|
||||
|
||||
|
||||
def _read_text(url: str) -> str:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=10) as resp:
|
||||
@ -73,7 +141,7 @@ def _counter(metrics: str, suite: str, status: str) -> float:
|
||||
for line in metrics.splitlines():
|
||||
if not line.startswith("platform_quality_gate_runs_total{"):
|
||||
continue
|
||||
if f'job="platform-quality-ci"' not in line:
|
||||
if 'job="platform-quality-ci"' not in line:
|
||||
continue
|
||||
if f'suite="{suite}"' not in line:
|
||||
continue
|
||||
@ -109,11 +177,27 @@ def main() -> int:
|
||||
|
||||
junit_path = Path(os.getenv("JUNIT_PATH", "build/junit.xml"))
|
||||
coverage_path = Path(os.getenv("COVERAGE_PATH", "build/coverage.json"))
|
||||
gate_rc_path = Path(os.getenv("QUALITY_GATE_RC_PATH", "build/quality-gate.rc"))
|
||||
source_root = Path(os.getenv("SOURCE_ROOT", "atlasbot"))
|
||||
build_dir = Path(os.getenv("BUILD_DIR", "build"))
|
||||
|
||||
totals = _load_junit(junit_path)
|
||||
coverage_pct = _load_coverage_percent(coverage_path)
|
||||
gate_rc = _load_gate_rc(gate_rc_path)
|
||||
source_lines_over_500 = _count_source_lines_over_500(source_root)
|
||||
passed = max(totals["tests"] - totals["failures"] - totals["errors"] - totals["skipped"], 0)
|
||||
outcome = "ok" if totals["tests"] > 0 and totals["failures"] == 0 and totals["errors"] == 0 else "failed"
|
||||
if gate_rc is not None and gate_rc != 0:
|
||||
outcome = "failed"
|
||||
checks = {
|
||||
"tests": "ok" if outcome == "ok" else "failed",
|
||||
"coverage": "ok" if coverage_pct >= 95.0 else "failed",
|
||||
"loc": "ok" if source_lines_over_500 == 0 else "failed",
|
||||
"docs_naming": "not_applicable",
|
||||
"gate_glue": "ok",
|
||||
"sonarqube": _sonarqube_check_status(build_dir),
|
||||
"supply_chain": _supply_chain_check_status(build_dir),
|
||||
}
|
||||
|
||||
metrics = _read_text(f"{pushgateway_url}/metrics")
|
||||
ok_count = _counter(metrics, suite, "ok")
|
||||
@ -135,8 +219,17 @@ def main() -> int:
|
||||
f'atlasbot_quality_gate_tests_total{{suite="{suite}",result="skipped"}} {totals["skipped"]}',
|
||||
"# TYPE atlasbot_quality_gate_coverage_percent gauge",
|
||||
f'atlasbot_quality_gate_coverage_percent{{suite="{suite}"}} {coverage_pct:.3f}',
|
||||
"# TYPE platform_quality_gate_workspace_line_coverage_percent gauge",
|
||||
f'platform_quality_gate_workspace_line_coverage_percent{{suite="{suite}"}} {coverage_pct:.3f}',
|
||||
"# TYPE platform_quality_gate_source_lines_over_500_total gauge",
|
||||
f'platform_quality_gate_source_lines_over_500_total{{suite="{suite}"}} {source_lines_over_500}',
|
||||
"# TYPE atlasbot_quality_gate_checks_total gauge",
|
||||
]
|
||||
) + "\n"
|
||||
payload += "\n".join(
|
||||
f'atlasbot_quality_gate_checks_total{{suite="{suite}",check="{check_name}",result="{check_status}"}} 1'
|
||||
for check_name, check_status in checks.items()
|
||||
) + "\n"
|
||||
|
||||
_post_text(f"{pushgateway_url}/metrics/job/platform-quality-ci/suite/{suite}", payload)
|
||||
return 0
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user