From 5d7679f1836858ba63131472a3501e60b8aca00f Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Sun, 19 Apr 2026 14:11:43 -0300 Subject: [PATCH] ci: add sonar/supply evidence collection and checks metrics --- Jenkinsfile | 129 ++++++++++++++++++++++++++++---- scripts/publish_test_metrics.py | 113 +++++++++++++++++++++------- 2 files changed, 200 insertions(+), 42 deletions(-) mode change 100644 => 100755 scripts/publish_test_metrics.py diff --git a/Jenkinsfile b/Jenkinsfile index 88fe4df..dacaf65 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -72,6 +72,8 @@ spec: PYTHONUNBUFFERED = '1' SUITE_NAME = 'atlasbot' PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091' + QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json' + QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json' } stages { stage('Checkout') { @@ -79,6 +81,71 @@ spec: checkout scm } } + stage('Collect SonarQube evidence') { + steps { + container('tester') { + sh ''' + set -euo pipefail + mkdir -p build + python3 - <<'PY' +import base64 +import json +import os +import urllib.parse +import urllib.request + +host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/') +project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip() +token = os.getenv('SONARQUBE_TOKEN', '').strip() +report_path = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json') +payload = {"status": "ERROR", "note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY"} +if host and project_key: + query = urllib.parse.urlencode({"projectKey": project_key}) + request = urllib.request.Request(f"{host}/api/qualitygates/project_status?{query}", method="GET") + if token: + encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8") + request.add_header("Authorization", f"Basic {encoded}") + try: + with urllib.request.urlopen(request, timeout=12) as response: + payload = json.loads(response.read().decode("utf-8")) + except Exception as exc: # noqa: BLE001 + payload = {"status": "ERROR", "error": str(exc)} +with open(report_path, "w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\\n") +PY + ''' + } + } + } + stage('Collect Supply Chain evidence') { + steps { + container('tester') { + sh ''' + set -euo pipefail + mkdir -p build + python3 - <<'PY' +import json +import os +from pathlib import Path + +report_path = Path(os.getenv('QUALITY_GATE_IRONBANK_REPORT', 'build/ironbank-compliance.json')) +if report_path.exists(): + raise SystemExit(0) +status = os.getenv('IRONBANK_COMPLIANCE_STATUS', '').strip() +compliant = os.getenv('IRONBANK_COMPLIANT', '').strip().lower() +payload = {"status": status or "unknown", "compliant": compliant in {"1", "true", "yes", "on"} if compliant else None} +payload = {k: v for k, v in payload.items() if v is not None} +if "status" not in payload: + payload["status"] = "unknown" +payload["note"] = "Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT or write build/ironbank-compliance.json in image-building repos." +report_path.parent.mkdir(parents=True, exist_ok=True) +report_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8") +PY + ''' + } + } + } stage('Prep toolchain') { steps { container('builder') { @@ -133,31 +200,65 @@ spec: } } } - stage('Unit tests') { + stage('Run quality gate') { steps { container('builder') { sh ''' set -euo pipefail mkdir -p build - docker buildx build --platform linux/arm64 --target test --load -t atlasbot-test . - docker run --rm -v "$PWD/build:/out" atlasbot-test \ - python -m ruff check atlasbot --select E9,F63,F7,F82 - docker run --rm -v "$PWD/build:/out" atlasbot-test \ - python -m slipcover --json --out /out/coverage.json --source atlasbot --fail-under 90 \ - -m pytest -q --junitxml /out/junit.xml + case "${TEST_PLATFORM:-$(uname -m)}" in + x86_64|amd64|linux/amd64) + TEST_PLATFORM_RESOLVED=linux/amd64 + ;; + aarch64|arm64|linux/arm64) + TEST_PLATFORM_RESOLVED=linux/arm64 + ;; + *) + TEST_PLATFORM_RESOLVED=linux/amd64 + ;; + esac + set +e + docker buildx build --platform "${TEST_PLATFORM_RESOLVED}" --target test --load -t atlasbot-test . \ + && docker run --rm -v "$PWD/build:/out" atlasbot-test \ + python -m ruff check atlasbot testing scripts --select E,F,W,B,C90,I,PLR,RUF,SIM,UP,ARG --ignore E501 \ + && docker run --rm -v "$PWD/build:/out" atlasbot-test \ + python scripts/check_file_sizes.py --root atlasbot --max-lines 500 \ + && docker run --rm -v "$PWD/build:/out" atlasbot-test \ + python scripts/check_docstrings.py --root atlasbot \ + && docker run --rm -v "$PWD/build:/out" atlasbot-test \ + python -m slipcover --json --out /out/coverage.json --source atlasbot --fail-under 95 \ + -m pytest -q --junitxml /out/junit.xml \ + && docker run --rm -v "$PWD/build:/out" atlasbot-test \ + python scripts/check_coverage.py /out/coverage.json --root atlasbot --threshold 95 + gate_rc=$? + set -e + printf '%s\n' "${gate_rc}" > build/quality-gate.rc ''' } } } + stage('Publish test metrics') { steps { - container('builder') { + container('tester') { sh ''' set -euo pipefail - docker run --rm -v "$PWD/build:/out" \ - -e JUNIT_PATH=/out/junit.xml \ - -e COVERAGE_PATH=/out/coverage.json \ - atlasbot-test python scripts/publish_test_metrics.py + export JUNIT_PATH='build/junit.xml' + export COVERAGE_PATH='build/coverage.json' + export SOURCE_ROOT='atlasbot' + export QUALITY_GATE_RC_PATH='build/quality-gate.rc' + python scripts/publish_test_metrics.py || true + ''' + } + } + } + + stage('Enforce quality gate') { + steps { + container('tester') { + sh ''' + set -euo pipefail + test "$(cat build/quality-gate.rc 2>/dev/null || echo 1)" -eq 0 ''' } } @@ -182,8 +283,8 @@ spec: always { script { if (fileExists('build.env')) { - def env = readProperties file: 'build.env' - echo "Build complete for ${env.SEMVER}" + def envFile = readProperties file: 'build.env' + echo "Build complete for ${envFile.SEMVER}" } } archiveArtifacts artifacts: 'build/*', allowEmptyArchive: true diff --git a/scripts/publish_test_metrics.py b/scripts/publish_test_metrics.py old mode 100644 new mode 100755 index 582f767..e70d91c --- a/scripts/publish_test_metrics.py +++ b/scripts/publish_test_metrics.py @@ -8,6 +8,8 @@ Outputs: - platform_quality_gate_runs_total{suite="atlasbot",status="ok|failed"} - atlasbot_quality_gate_tests_total{suite="atlasbot",result=*} - atlasbot_quality_gate_coverage_percent{suite="atlasbot"} +- platform_quality_gate_workspace_line_coverage_percent{suite="atlasbot"} +- platform_quality_gate_source_lines_over_500_total{suite="atlasbot"} """ from __future__ import annotations @@ -18,9 +20,7 @@ import urllib.request import xml.etree.ElementTree as ET from pathlib import Path - -SOURCE_SUFFIXES = {".py", ".js", ".mjs", ".ts", ".tsx", ".json", ".yaml", ".yml", ".sh"} -SKIP_DIRS = {".git", ".venv", "venv", "node_modules", "build", "dist", "__pycache__", ".pytest_cache"} +QUALITY_SUCCESS_STATES = {"ok", "pass", "passed", "success", "compliant"} def _as_int(node: ET.Element, name: str) -> int: @@ -65,23 +65,68 @@ def _load_coverage_percent(path: Path) -> float: return 0.0 -def _count_lines_over_limit(root: Path, *, max_lines: int = 500) -> int: - count = 0 - for path in root.rglob("*"): +def _load_gate_rc(path: Path) -> int | None: + if not path.exists(): + return None + raw = path.read_text(encoding="utf-8").strip() + if not raw: + return None + try: + return int(raw) + except ValueError: + return None + + +def _count_source_lines_over_500(root: Path) -> int: + if not root.exists(): + return 0 + over = 0 + for path in root.rglob("*.py"): if not path.is_file(): continue - if any(part in SKIP_DIRS for part in path.parts): - continue - if path.name != "Jenkinsfile" and path.suffix.lower() not in SOURCE_SUFFIXES: - continue - try: - with path.open("r", encoding="utf-8", errors="ignore") as handle: - lines = sum(1 for _ in handle) - except OSError: - continue - if lines > max_lines: - count += 1 - return count + line_count = sum(1 for _ in path.open("r", encoding="utf-8")) + if line_count > 500: + over += 1 + return over + + +def _load_json(path: Path) -> dict | None: + if not path.exists(): + return None + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return None + return payload if isinstance(payload, dict) else None + + +def _sonarqube_check_status(build_dir: Path) -> str: + report = _load_json(Path(os.getenv("QUALITY_GATE_SONARQUBE_REPORT", str(build_dir / "sonarqube-quality-gate.json")))) + if not report: + return "not_applicable" + status_candidates = [ + report.get("status"), + ((report.get("projectStatus") or {}).get("status") if isinstance(report.get("projectStatus"), dict) else None), + ((report.get("qualityGate") or {}).get("status") if isinstance(report.get("qualityGate"), dict) else None), + ] + for value in status_candidates: + if isinstance(value, str): + return "ok" if value.strip().lower() in QUALITY_SUCCESS_STATES else "failed" + return "failed" + + +def _supply_chain_check_status(build_dir: Path) -> str: + report = _load_json(Path(os.getenv("QUALITY_GATE_IRONBANK_REPORT", str(build_dir / "ironbank-compliance.json")))) + if not report: + return "not_applicable" + compliant = report.get("compliant") + if isinstance(compliant, bool): + return "ok" if compliant else "failed" + status_candidates = [report.get("status"), report.get("result"), report.get("compliance")] + for value in status_candidates: + if isinstance(value, str): + return "ok" if value.strip().lower() in QUALITY_SUCCESS_STATES else "failed" + return "failed" def _read_text(url: str) -> str: @@ -96,7 +141,7 @@ def _counter(metrics: str, suite: str, status: str) -> float: for line in metrics.splitlines(): if not line.startswith("platform_quality_gate_runs_total{"): continue - if f'job="platform-quality-ci"' not in line: + if 'job="platform-quality-ci"' not in line: continue if f'suite="{suite}"' not in line: continue @@ -132,16 +177,27 @@ def main() -> int: junit_path = Path(os.getenv("JUNIT_PATH", "build/junit.xml")) coverage_path = Path(os.getenv("COVERAGE_PATH", "build/coverage.json")) + gate_rc_path = Path(os.getenv("QUALITY_GATE_RC_PATH", "build/quality-gate.rc")) + source_root = Path(os.getenv("SOURCE_ROOT", "atlasbot")) + build_dir = Path(os.getenv("BUILD_DIR", "build")) - repo_root = Path(__file__).resolve().parents[1] totals = _load_junit(junit_path) coverage_pct = _load_coverage_percent(coverage_path) - over_500 = _count_lines_over_limit(repo_root) + gate_rc = _load_gate_rc(gate_rc_path) + source_lines_over_500 = _count_source_lines_over_500(source_root) passed = max(totals["tests"] - totals["failures"] - totals["errors"] - totals["skipped"], 0) outcome = "ok" if totals["tests"] > 0 and totals["failures"] == 0 and totals["errors"] == 0 else "failed" - tests_check = "ok" if outcome == "ok" else "failed" - coverage_check = "ok" if coverage_pct >= 95.0 else "failed" - loc_check = "ok" if over_500 == 0 else "failed" + if gate_rc is not None and gate_rc != 0: + outcome = "failed" + checks = { + "tests": "ok" if outcome == "ok" else "failed", + "coverage": "ok" if coverage_pct >= 95.0 else "failed", + "loc": "ok" if source_lines_over_500 == 0 else "failed", + "docs_naming": "not_applicable", + "gate_glue": "ok", + "sonarqube": _sonarqube_check_status(build_dir), + "supply_chain": _supply_chain_check_status(build_dir), + } metrics = _read_text(f"{pushgateway_url}/metrics") ok_count = _counter(metrics, suite, "ok") @@ -166,13 +222,14 @@ def main() -> int: "# TYPE platform_quality_gate_workspace_line_coverage_percent gauge", f'platform_quality_gate_workspace_line_coverage_percent{{suite="{suite}"}} {coverage_pct:.3f}', "# TYPE platform_quality_gate_source_lines_over_500_total gauge", - f'platform_quality_gate_source_lines_over_500_total{{suite="{suite}"}} {over_500}', + f'platform_quality_gate_source_lines_over_500_total{{suite="{suite}"}} {source_lines_over_500}', "# TYPE atlasbot_quality_gate_checks_total gauge", - f'atlasbot_quality_gate_checks_total{{suite="{suite}",check="tests",result="{tests_check}"}} 1', - f'atlasbot_quality_gate_checks_total{{suite="{suite}",check="coverage",result="{coverage_check}"}} 1', - f'atlasbot_quality_gate_checks_total{{suite="{suite}",check="loc",result="{loc_check}"}} 1', ] ) + "\n" + payload += "\n".join( + f'atlasbot_quality_gate_checks_total{{suite="{suite}",check="{check_name}",result="{check_status}"}} 1' + for check_name, check_status in checks.items() + ) + "\n" _post_text(f"{pushgateway_url}/metrics/job/platform-quality-ci/suite/{suite}", payload) return 0