From c38b6c5e277ac0cd3d3792b80277b1ae50b4eacf Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Fri, 10 Apr 2026 16:38:55 -0300 Subject: [PATCH] ci: publish titan-iac tests and seed ananke/lesavka jobs --- Jenkinsfile | 42 +++++- ci/Jenkinsfile.titan-iac | 42 +++++- ci/scripts/publish_test_metrics.py | 185 ++++++++++++++++++++++++++ services/jenkins/configmap-jcasc.yaml | 120 +++++++++++------ 4 files changed, 345 insertions(+), 44 deletions(-) create mode 100644 ci/scripts/publish_test_metrics.py diff --git a/Jenkinsfile b/Jenkinsfile index 4d6b23e6..9b782bae 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,6 +23,8 @@ spec: environment { PIP_DISABLE_PIP_VERSION_CHECK = '1' PYTHONUNBUFFERED = '1' + SUITE_NAME = 'titan-iac' + PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091' } stages { stage('Checkout') { @@ -37,7 +39,31 @@ spec: } stage('Glue tests') { steps { - sh 'pytest -q ci/tests/glue' + sh ''' + set -eu + mkdir -p build + set +e + pytest -q ci/tests/glue --junitxml=build/junit-glue.xml + glue_rc=$? + set -e + printf '%s\n' "${glue_rc}" > build/glue.rc + ''' + } + } + stage('Publish test metrics') { + steps { + sh ''' + set -eu + python3 ci/scripts/publish_test_metrics.py + ''' + } + } + stage('Enforce glue tests') { + steps { + sh ''' + set -eu + test "$(cat build/glue.rc 2>/dev/null || echo 1)" -eq 0 + ''' } } stage('Resolve Flux branch') { @@ -74,4 +100,18 @@ spec: } } } + post { + always { + script { + if (fileExists('build/junit-glue.xml')) { + try { + junit allowEmptyResults: true, testResults: 'build/junit-glue.xml' + } catch (Throwable err) { + echo "junit step unavailable: ${err.class.simpleName}" + } + } + } + archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true + } + } } diff --git a/ci/Jenkinsfile.titan-iac b/ci/Jenkinsfile.titan-iac index 77990d77..49078a11 100644 --- a/ci/Jenkinsfile.titan-iac +++ b/ci/Jenkinsfile.titan-iac @@ -22,6 +22,8 @@ spec: environment { PIP_DISABLE_PIP_VERSION_CHECK = '1' PYTHONUNBUFFERED = '1' + SUITE_NAME = 'titan-iac' + PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091' } stages { stage('Checkout') { @@ -36,7 +38,31 @@ spec: } stage('Glue tests') { steps { - sh 'pytest -q ci/tests/glue' + sh ''' + set -eu + mkdir -p build + set +e + pytest -q ci/tests/glue --junitxml=build/junit-glue.xml + glue_rc=$? + set -e + printf '%s\n' "${glue_rc}" > build/glue.rc + ''' + } + } + stage('Publish test metrics') { + steps { + sh ''' + set -eu + python3 ci/scripts/publish_test_metrics.py + ''' + } + } + stage('Enforce glue tests') { + steps { + sh ''' + set -eu + test "$(cat build/glue.rc 2>/dev/null || echo 1)" -eq 0 + ''' } } stage('Resolve Flux branch') { @@ -73,4 +99,18 @@ spec: } } } + post { + always { + script { + if (fileExists('build/junit-glue.xml')) { + try { + junit allowEmptyResults: true, testResults: 'build/junit-glue.xml' + } catch (Throwable err) { + echo "junit step unavailable: ${err.class.simpleName}" + } + } + } + archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true + } + } } diff --git a/ci/scripts/publish_test_metrics.py b/ci/scripts/publish_test_metrics.py new file mode 100644 index 00000000..cdd25241 --- /dev/null +++ b/ci/scripts/publish_test_metrics.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Publish titan-iac Jenkins glue test results to Pushgateway.""" + +from __future__ import annotations + +import json +import os +import re +import urllib.error +import urllib.request +import xml.etree.ElementTree as ET + + +def _escape_label(value: str) -> str: + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +def _label_str(labels: dict[str, str]) -> str: + parts = [f'{key}="{_escape_label(val)}"' for key, val in labels.items() if val] + return "{" + ",".join(parts) + "}" if parts else "" + + +def _read_text(url: str) -> str: + with urllib.request.urlopen(url, timeout=10) as response: + return response.read().decode("utf-8") + + +def _post_text(url: str, payload: str) -> None: + request = urllib.request.Request( + url, + data=payload.encode("utf-8"), + method="POST", + headers={"Content-Type": "text/plain"}, + ) + with urllib.request.urlopen(request, timeout=10) as response: + if response.status >= 400: + raise RuntimeError(f"push failed with status={response.status}") + + +def _parse_junit(path: str) -> dict[str, int]: + if not os.path.exists(path): + return {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} + + tree = ET.parse(path) + root = tree.getroot() + totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} + + suites: list[ET.Element] + if root.tag == "testsuite": + suites = [root] + elif root.tag == "testsuites": + suites = [elem for elem in root if elem.tag == "testsuite"] + else: + suites = [] + + for suite in suites: + for key in totals: + raw_value = suite.attrib.get(key, "0") + try: + totals[key] += int(float(raw_value)) + except ValueError: + totals[key] += 0 + return totals + + +def _read_exit_code(path: str) -> int: + try: + with open(path, "r", encoding="utf-8") as handle: + return int(handle.read().strip()) + except (FileNotFoundError, ValueError): + return 1 + + +def _fetch_existing_counter(pushgateway_url: str, metric: str, labels: dict[str, str]) -> float: + text = _read_text(f"{pushgateway_url.rstrip('/')}/metrics") + for line in text.splitlines(): + if not line.startswith(metric + "{"): + continue + if any(f'{key}="{value}"' not in line for key, value in labels.items()): + continue + parts = line.split() + if len(parts) < 2: + continue + try: + return float(parts[1]) + except ValueError: + return 0.0 + return 0.0 + + +def _build_payload( + suite: str, + status: str, + tests: dict[str, int], + ok_count: int, + failed_count: int, + branch: str, + build_number: str, +) -> str: + passed = max(tests["tests"] - tests["failures"] - tests["errors"] - tests["skipped"], 0) + build_labels = _label_str( + { + "suite": suite, + "branch": branch or "unknown", + "build_number": build_number or "unknown", + } + ) + lines = [ + "# TYPE platform_quality_gate_runs_total counter", + f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count}', + f'platform_quality_gate_runs_total{{suite="{suite}",status="failed"}} {failed_count}', + "# TYPE titan_iac_quality_gate_tests_total gauge", + f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="passed"}} {passed}', + f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="failed"}} {tests["failures"]}', + f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="error"}} {tests["errors"]}', + f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="skipped"}} {tests["skipped"]}', + "# TYPE titan_iac_quality_gate_run_status gauge", + f'titan_iac_quality_gate_run_status{{suite="{suite}",status="ok"}} {1 if status == "ok" else 0}', + f'titan_iac_quality_gate_run_status{{suite="{suite}",status="failed"}} {1 if status == "failed" else 0}', + "# TYPE titan_iac_quality_gate_build_info gauge", + f"titan_iac_quality_gate_build_info{build_labels} 1", + ] + return "\n".join(lines) + "\n" + + +def main() -> int: + suite = os.getenv("SUITE_NAME", "titan-iac") + pushgateway_url = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091") + job_name = os.getenv("QUALITY_GATE_JOB_NAME", "platform-quality-ci") + junit_path = os.getenv("JUNIT_PATH", "build/junit-glue.xml") + exit_code_path = os.getenv("GLUE_EXIT_CODE_PATH", "build/glue.rc") + branch = os.getenv("BRANCH_NAME", os.getenv("GIT_BRANCH", "")) + build_number = os.getenv("BUILD_NUMBER", "") + + tests = _parse_junit(junit_path) + exit_code = _read_exit_code(exit_code_path) + status = "ok" if exit_code == 0 else "failed" + + ok_count = int( + _fetch_existing_counter( + pushgateway_url, + "platform_quality_gate_runs_total", + {"job": job_name, "suite": suite, "status": "ok"}, + ) + ) + failed_count = int( + _fetch_existing_counter( + pushgateway_url, + "platform_quality_gate_runs_total", + {"job": job_name, "suite": suite, "status": "failed"}, + ) + ) + if status == "ok": + ok_count += 1 + else: + failed_count += 1 + + payload = _build_payload( + suite=suite, + status=status, + tests=tests, + ok_count=ok_count, + failed_count=failed_count, + branch=branch, + build_number=build_number, + ) + push_url = f"{pushgateway_url.rstrip('/')}/metrics/job/{job_name}/suite/{suite}" + _post_text(push_url, payload) + + summary = { + "suite": suite, + "status": status, + "tests_total": tests["tests"], + "tests_failed": tests["failures"], + "tests_error": tests["errors"], + "tests_skipped": tests["skipped"], + "ok_count": ok_count, + "failed_count": failed_count, + } + print(json.dumps(summary, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index 0da9ebc3..9eff8e5c 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -73,48 +73,6 @@ data: } } } - pipelineJob('jellyfin-oidc-plugin') { - definition { - cpsScm { - scm { - git { - remote { - url('https://scm.bstein.dev/bstein/titan-iac.git') - credentials('gitea-pat') - } - branches('*/main') - } - } - scriptPath('services/jellyfin/oidc/Jenkinsfile') - } - } - } - pipelineJob('ci-demo') { - properties { - pipelineTriggers { - triggers { - scmTrigger { - scmpoll_spec('H/1 * * * *') - ignorePostCommitHooks(false) - } - } - } - } - definition { - cpsScm { - scm { - git { - remote { - url('https://scm.bstein.dev/bstein/ci-demo.git') - credentials('gitea-pat') - } - branches('*/master') - } - } - scriptPath('Jenkinsfile') - } - } - } pipelineJob('bstein-dev-home') { properties { pipelineTriggers { @@ -193,6 +151,84 @@ data: } } } + pipelineJob('ananke') { + properties { + pipelineTriggers { + triggers { + scmTrigger { + scmpoll_spec('H/5 * * * *') + ignorePostCommitHooks(false) + } + } + } + } + definition { + cpsScm { + scm { + git { + remote { + url('https://scm.bstein.dev/bstein/ananke.git') + credentials('gitea-pat') + } + branches('*/main') + } + } + scriptPath('Jenkinsfile') + } + } + } + pipelineJob('lesavka') { + properties { + pipelineTriggers { + triggers { + scmTrigger { + scmpoll_spec('H/5 * * * *') + ignorePostCommitHooks(false) + } + } + } + } + definition { + cpsScm { + scm { + git { + remote { + url('https://scm.bstein.dev/bstein/lesavka.git') + credentials('gitea-pat') + } + branches('*/master') + } + } + scriptPath('Jenkinsfile') + } + } + } + pipelineJob('pegasus') { + properties { + pipelineTriggers { + triggers { + scmTrigger { + scmpoll_spec('H/5 * * * *') + ignorePostCommitHooks(false) + } + } + } + } + definition { + cpsScm { + scm { + git { + remote { + url('https://scm.bstein.dev/bstein/pegasus.git') + credentials('gitea-pat') + } + branches('*/main') + } + } + scriptPath('Jenkinsfile') + } + } + } pipelineJob('data-prepper') { properties { pipelineTriggers {