diff --git a/Jenkinsfile b/Jenkinsfile index 4d6b23e6..9b782bae 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,6 +23,8 @@ spec: environment { PIP_DISABLE_PIP_VERSION_CHECK = '1' PYTHONUNBUFFERED = '1' + SUITE_NAME = 'titan-iac' + PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091' } stages { stage('Checkout') { @@ -37,7 +39,31 @@ spec: } stage('Glue tests') { steps { - sh 'pytest -q ci/tests/glue' + sh ''' + set -eu + mkdir -p build + set +e + pytest -q ci/tests/glue --junitxml=build/junit-glue.xml + glue_rc=$? + set -e + printf '%s\n' "${glue_rc}" > build/glue.rc + ''' + } + } + stage('Publish test metrics') { + steps { + sh ''' + set -eu + python3 ci/scripts/publish_test_metrics.py + ''' + } + } + stage('Enforce glue tests') { + steps { + sh ''' + set -eu + test "$(cat build/glue.rc 2>/dev/null || echo 1)" -eq 0 + ''' } } stage('Resolve Flux branch') { @@ -74,4 +100,18 @@ spec: } } } + post { + always { + script { + if (fileExists('build/junit-glue.xml')) { + try { + junit allowEmptyResults: true, testResults: 'build/junit-glue.xml' + } catch (Throwable err) { + echo "junit step unavailable: ${err.class.simpleName}" + } + } + } + archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true + } + } } diff --git a/ci/Jenkinsfile.titan-iac b/ci/Jenkinsfile.titan-iac index 2f6d26d4..49078a11 100644 --- a/ci/Jenkinsfile.titan-iac +++ b/ci/Jenkinsfile.titan-iac @@ -41,7 +41,27 @@ spec: sh ''' set -eu mkdir -p build + set +e pytest -q ci/tests/glue --junitxml=build/junit-glue.xml + glue_rc=$? + set -e + printf '%s\n' "${glue_rc}" > build/glue.rc + ''' + } + } + stage('Publish test metrics') { + steps { + sh ''' + set -eu + python3 ci/scripts/publish_test_metrics.py + ''' + } + } + stage('Enforce glue tests') { + steps { + sh ''' + set -eu + test "$(cat build/glue.rc 2>/dev/null || echo 1)" -eq 0 ''' } } @@ -82,94 +102,15 @@ spec: post { always { script { - env.QUALITY_STATUS = currentBuild.currentResult == 'SUCCESS' ? 'ok' : 'failed' + if (fileExists('build/junit-glue.xml')) { + try { + junit allowEmptyResults: true, testResults: 'build/junit-glue.xml' + } catch (Throwable err) { + echo "junit step unavailable: ${err.class.simpleName}" + } + } } - sh ''' - set -eu - python - <<'PY' -import os -import urllib.request -import xml.etree.ElementTree as ET -from pathlib import Path - -suite = os.getenv("SUITE_NAME", "titan-iac") -status = os.getenv("QUALITY_STATUS", "failed") -gateway = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091").rstrip("/") -junit_path = Path("build/junit-glue.xml") - -totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} -if junit_path.exists(): - root = ET.parse(junit_path).getroot() - suites = [root] if root.tag == "testsuite" else list(root.findall("testsuite")) if root.tag == "testsuites" else [] - for node in suites: - for key in totals: - raw = node.attrib.get(key) or "0" - try: - totals[key] += int(float(raw)) - except ValueError: - pass - -passed = max(totals["tests"] - totals["failures"] - totals["errors"] - totals["skipped"], 0) - -def read_metrics() -> str: - try: - with urllib.request.urlopen(f"{gateway}/metrics", timeout=10) as resp: - return resp.read().decode("utf-8", errors="replace") - except Exception: - return "" - -def read_counter(text: str, counter_status: str) -> float: - for line in text.splitlines(): - if not line.startswith("platform_quality_gate_runs_total{"): - continue - if 'job="platform-quality-ci"' not in line: - continue - if f'suite="{suite}"' not in line: - continue - if f'status="{counter_status}"' not in line: - continue - parts = line.split() - if len(parts) < 2: - continue - try: - return float(parts[1]) - except ValueError: - return 0.0 - return 0.0 - -metrics = read_metrics() -ok_count = read_counter(metrics, "ok") -failed_count = read_counter(metrics, "failed") -if status == "ok": - ok_count += 1 -else: - failed_count += 1 - -payload = "\n".join( - [ - "# TYPE platform_quality_gate_runs_total counter", - f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count:.0f}', - f'platform_quality_gate_runs_total{{suite="{suite}",status="failed"}} {failed_count:.0f}', - "# TYPE titan_iac_quality_gate_tests_total gauge", - f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="passed"}} {passed}', - f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="failed"}} {totals["failures"]}', - f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="error"}} {totals["errors"]}', - f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="skipped"}} {totals["skipped"]}', - ] -) + "\n" - -req = urllib.request.Request( - f"{gateway}/metrics/job/platform-quality-ci/suite/{suite}", - data=payload.encode("utf-8"), - method="POST", - headers={"Content-Type": "text/plain"}, -) -with urllib.request.urlopen(req, timeout=10) as resp: - if resp.status >= 400: - raise RuntimeError(f"push failed: {resp.status}") -PY - ''' - archiveArtifacts artifacts: 'build/junit-glue.xml', allowEmptyArchive: true + archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true } } } diff --git a/ci/scripts/publish_test_metrics.py b/ci/scripts/publish_test_metrics.py new file mode 100644 index 00000000..cdd25241 --- /dev/null +++ b/ci/scripts/publish_test_metrics.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Publish titan-iac Jenkins glue test results to Pushgateway.""" + +from __future__ import annotations + +import json +import os +import re +import urllib.error +import urllib.request +import xml.etree.ElementTree as ET + + +def _escape_label(value: str) -> str: + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +def _label_str(labels: dict[str, str]) -> str: + parts = [f'{key}="{_escape_label(val)}"' for key, val in labels.items() if val] + return "{" + ",".join(parts) + "}" if parts else "" + + +def _read_text(url: str) -> str: + with urllib.request.urlopen(url, timeout=10) as response: + return response.read().decode("utf-8") + + +def _post_text(url: str, payload: str) -> None: + request = urllib.request.Request( + url, + data=payload.encode("utf-8"), + method="POST", + headers={"Content-Type": "text/plain"}, + ) + with urllib.request.urlopen(request, timeout=10) as response: + if response.status >= 400: + raise RuntimeError(f"push failed with status={response.status}") + + +def _parse_junit(path: str) -> dict[str, int]: + if not os.path.exists(path): + return {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} + + tree = ET.parse(path) + root = tree.getroot() + totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} + + suites: list[ET.Element] + if root.tag == "testsuite": + suites = [root] + elif root.tag == "testsuites": + suites = [elem for elem in root if elem.tag == "testsuite"] + else: + suites = [] + + for suite in suites: + for key in totals: + raw_value = suite.attrib.get(key, "0") + try: + totals[key] += int(float(raw_value)) + except ValueError: + totals[key] += 0 + return totals + + +def _read_exit_code(path: str) -> int: + try: + with open(path, "r", encoding="utf-8") as handle: + return int(handle.read().strip()) + except (FileNotFoundError, ValueError): + return 1 + + +def _fetch_existing_counter(pushgateway_url: str, metric: str, labels: dict[str, str]) -> float: + text = _read_text(f"{pushgateway_url.rstrip('/')}/metrics") + for line in text.splitlines(): + if not line.startswith(metric + "{"): + continue + if any(f'{key}="{value}"' not in line for key, value in labels.items()): + continue + parts = line.split() + if len(parts) < 2: + continue + try: + return float(parts[1]) + except ValueError: + return 0.0 + return 0.0 + + +def _build_payload( + suite: str, + status: str, + tests: dict[str, int], + ok_count: int, + failed_count: int, + branch: str, + build_number: str, +) -> str: + passed = max(tests["tests"] - tests["failures"] - tests["errors"] - tests["skipped"], 0) + build_labels = _label_str( + { + "suite": suite, + "branch": branch or "unknown", + "build_number": build_number or "unknown", + } + ) + lines = [ + "# TYPE platform_quality_gate_runs_total counter", + f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count}', + f'platform_quality_gate_runs_total{{suite="{suite}",status="failed"}} {failed_count}', + "# TYPE titan_iac_quality_gate_tests_total gauge", + f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="passed"}} {passed}', + f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="failed"}} {tests["failures"]}', + f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="error"}} {tests["errors"]}', + f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="skipped"}} {tests["skipped"]}', + "# TYPE titan_iac_quality_gate_run_status gauge", + f'titan_iac_quality_gate_run_status{{suite="{suite}",status="ok"}} {1 if status == "ok" else 0}', + f'titan_iac_quality_gate_run_status{{suite="{suite}",status="failed"}} {1 if status == "failed" else 0}', + "# TYPE titan_iac_quality_gate_build_info gauge", + f"titan_iac_quality_gate_build_info{build_labels} 1", + ] + return "\n".join(lines) + "\n" + + +def main() -> int: + suite = os.getenv("SUITE_NAME", "titan-iac") + pushgateway_url = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091") + job_name = os.getenv("QUALITY_GATE_JOB_NAME", "platform-quality-ci") + junit_path = os.getenv("JUNIT_PATH", "build/junit-glue.xml") + exit_code_path = os.getenv("GLUE_EXIT_CODE_PATH", "build/glue.rc") + branch = os.getenv("BRANCH_NAME", os.getenv("GIT_BRANCH", "")) + build_number = os.getenv("BUILD_NUMBER", "") + + tests = _parse_junit(junit_path) + exit_code = _read_exit_code(exit_code_path) + status = "ok" if exit_code == 0 else "failed" + + ok_count = int( + _fetch_existing_counter( + pushgateway_url, + "platform_quality_gate_runs_total", + {"job": job_name, "suite": suite, "status": "ok"}, + ) + ) + failed_count = int( + _fetch_existing_counter( + pushgateway_url, + "platform_quality_gate_runs_total", + {"job": job_name, "suite": suite, "status": "failed"}, + ) + ) + if status == "ok": + ok_count += 1 + else: + failed_count += 1 + + payload = _build_payload( + suite=suite, + status=status, + tests=tests, + ok_count=ok_count, + failed_count=failed_count, + branch=branch, + build_number=build_number, + ) + push_url = f"{pushgateway_url.rstrip('/')}/metrics/job/{job_name}/suite/{suite}" + _post_text(push_url, payload) + + summary = { + "suite": suite, + "status": status, + "tests_total": tests["tests"], + "tests_failed": tests["failures"], + "tests_error": tests["errors"], + "tests_skipped": tests["skipped"], + "ok_count": ok_count, + "failed_count": failed_count, + } + print(json.dumps(summary, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index c3dcea81..9eff8e5c 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -125,32 +125,6 @@ data: } } } - pipelineJob('atlasbot') { - properties { - pipelineTriggers { - triggers { - scmTrigger { - scmpoll_spec('H/5 * * * *') - ignorePostCommitHooks(false) - } - } - } - } - definition { - cpsScm { - scm { - git { - remote { - url('https://scm.bstein.dev/bstein/atlasbot.git') - credentials('gitea-pat') - } - branches('*/main') - } - } - scriptPath('Jenkinsfile') - } - } - } pipelineJob('metis') { properties { pipelineTriggers { @@ -177,6 +151,58 @@ data: } } } + pipelineJob('ananke') { + properties { + pipelineTriggers { + triggers { + scmTrigger { + scmpoll_spec('H/5 * * * *') + ignorePostCommitHooks(false) + } + } + } + } + definition { + cpsScm { + scm { + git { + remote { + url('https://scm.bstein.dev/bstein/ananke.git') + credentials('gitea-pat') + } + branches('*/main') + } + } + scriptPath('Jenkinsfile') + } + } + } + pipelineJob('lesavka') { + properties { + pipelineTriggers { + triggers { + scmTrigger { + scmpoll_spec('H/5 * * * *') + ignorePostCommitHooks(false) + } + } + } + } + definition { + cpsScm { + scm { + git { + remote { + url('https://scm.bstein.dev/bstein/lesavka.git') + credentials('gitea-pat') + } + branches('*/master') + } + } + scriptPath('Jenkinsfile') + } + } + } pipelineJob('pegasus') { properties { pipelineTriggers { @@ -222,39 +248,13 @@ data: url('https://scm.bstein.dev/bstein/titan-iac.git') credentials('gitea-pat') } - branches('*/main') + branches('*/feature/sso-hardening') } } scriptPath('services/logging/Jenkinsfile.data-prepper') } } } - pipelineJob('Soteria') { - properties { - pipelineTriggers { - triggers { - scmTrigger { - scmpoll_spec('H/5 * * * *') - ignorePostCommitHooks(false) - } - } - } - } - definition { - cpsScm { - scm { - git { - remote { - url('https://scm.bstein.dev/bstein/soteria.git') - credentials('gitea-pat') - } - branches('*/main') - } - } - scriptPath('Jenkinsfile') - } - } - } multibranchPipelineJob('titan-iac-quality-gate') { branchSources { branchSource {