maintenance(ariadne): activate jenkins workspace cleanup deletes

Jenkinsfile

@@ -7,6 +7,7 @@ pipeline {
 apiVersion: v1
 kind: Pod
 spec:
+  serviceAccountName: "jenkins"
   nodeSelector:
     hardware: rpi5
     kubernetes.io/arch: arm64
@@ -23,6 +24,9 @@ spec:
   environment {
     PIP_DISABLE_PIP_VERSION_CHECK = '1'
     PYTHONUNBUFFERED = '1'
+    SUITE_NAME = 'titan-iac'
+    PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
+    VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
   }
   stages {
     stage('Checkout') {
@@ -35,9 +39,36 @@ spec:
         sh 'pip install --no-cache-dir -r ci/requirements.txt'
       }
     }
-    stage('Glue tests') {
+    stage('Run quality gate') {
       steps {
-        sh 'pytest -q ci/tests/glue'
+        sh '''
+          set -eu
+          mkdir -p build
+          set +e
+          python3 -m testing.quality_gate --profile jenkins --build-dir build
+          quality_gate_rc=$?
+          set -e
+          printf '%s\n' "${quality_gate_rc}" > build/quality-gate.rc
+        '''
+      }
+    }
+    stage('Publish test metrics') {
+      steps {
+        sh '''
+          set -eu
+          export JUNIT_GLOB='build/junit-*.xml'
+          export QUALITY_GATE_EXIT_CODE_PATH='build/quality-gate.rc'
+          export QUALITY_GATE_SUMMARY_PATH='build/quality-gate-summary.json'
+          python3 ci/scripts/publish_test_metrics.py
+        '''
+      }
+    }
+    stage('Enforce quality gate') {
+      steps {
+        sh '''
+          set -eu
+          test "$(cat build/quality-gate.rc 2>/dev/null || echo 1)" -eq 0
+        '''
       }
     }
     stage('Resolve Flux branch') {
@@ -45,7 +76,7 @@ spec:
         script {
           env.FLUX_BRANCH = sh(
             returnStdout: true,
-            script: "awk '/branch:/{print $2; exit}' clusters/atlas/flux-system/gotk-sync.yaml"
+            script: '''awk '/branch:/{print $2; exit}' clusters/atlas/flux-system/gotk-sync.yaml'''
           ).trim()
           if (!env.FLUX_BRANCH) {
             error('Flux branch not found in gotk-sync.yaml')
@@ -62,16 +93,32 @@ spec:
         }
       }
       steps {
-        withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
+        container('jnlp') {
-          sh '''
+          withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
-            set +x
+            sh '''
-            git config user.email "jenkins@bstein.dev"
+              set +x
-            git config user.name "jenkins"
+              git config user.email "jenkins@bstein.dev"
-            git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
+              git config user.name "jenkins"
-            git push origin HEAD:${FLUX_BRANCH}
+              git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
-          '''
+              git push origin HEAD:${FLUX_BRANCH}
+            '''
+          }
         }
       }
     }
   }
+  post {
+    always {
+      script {
+        if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
+          try {
+            junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
+          } catch (Throwable err) {
+            echo "junit step unavailable: ${err.class.simpleName}"
+          }
+        }
+      }
+      archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
+    }
+  }
 }

ci/Jenkinsfile.titan-iac

@@ -6,6 +6,7 @@ pipeline {
 apiVersion: v1
 kind: Pod
 spec:
+  serviceAccountName: "jenkins"
   nodeSelector:
     hardware: rpi5
     kubernetes.io/arch: arm64
@@ -24,6 +25,7 @@ spec:
     PYTHONUNBUFFERED = '1'
     SUITE_NAME = 'titan-iac'
     PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
+    VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
   }
   stages {
     stage('Checkout') {
@@ -36,12 +38,35 @@ spec:
         sh 'pip install --no-cache-dir -r ci/requirements.txt'
       }
     }
-    stage('Glue tests') {
+    stage('Run quality gate') {
       steps {
         sh '''
           set -eu
           mkdir -p build
-          pytest -q ci/tests/glue --junitxml=build/junit-glue.xml
+          set +e
+          python3 -m testing.quality_gate --profile jenkins --build-dir build
+          quality_gate_rc=$?
+          set -e
+          printf '%s\n' "${quality_gate_rc}" > build/quality-gate.rc
+        '''
+      }
+    }
+    stage('Publish test metrics') {
+      steps {
+        sh '''
+          set -eu
+          export JUNIT_GLOB='build/junit-*.xml'
+          export QUALITY_GATE_EXIT_CODE_PATH='build/quality-gate.rc'
+          export QUALITY_GATE_SUMMARY_PATH='build/quality-gate-summary.json'
+          python3 ci/scripts/publish_test_metrics.py
+        '''
+      }
+    }
+    stage('Enforce quality gate') {
+      steps {
+        sh '''
+          set -eu
+          test "$(cat build/quality-gate.rc 2>/dev/null || echo 1)" -eq 0
         '''
       }
     }
@@ -50,7 +75,7 @@ spec:
         script {
           env.FLUX_BRANCH = sh(
             returnStdout: true,
-            script: "awk '/branch:/{print $2; exit}' clusters/atlas/flux-system/gotk-sync.yaml"
+            script: '''awk '/branch:/{print $2; exit}' clusters/atlas/flux-system/gotk-sync.yaml'''
           ).trim()
           if (!env.FLUX_BRANCH) {
             error('Flux branch not found in gotk-sync.yaml')
@@ -67,14 +92,16 @@ spec:
         }
       }
       steps {
-        withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
+        container('jnlp') {
-          sh '''
+          withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
-            set +x
+            sh '''
-            git config user.email "jenkins@bstein.dev"
+              set +x
-            git config user.name "jenkins"
+              git config user.email "jenkins@bstein.dev"
-            git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
+              git config user.name "jenkins"
-            git push origin HEAD:${FLUX_BRANCH}
+              git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
-          '''
+              git push origin HEAD:${FLUX_BRANCH}
+            '''
+          }
         }
       }
     }
@@ -82,94 +109,15 @@ spec:
   post {
     always {
       script {
-        env.QUALITY_STATUS = currentBuild.currentResult == 'SUCCESS' ? 'ok' : 'failed'
+        if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
+          try {
+            junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
+          } catch (Throwable err) {
+            echo "junit step unavailable: ${err.class.simpleName}"
+          }
+        }
       }
-      sh '''
+      archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
-        set -eu
-        python - <<'PY'
-import os
-import urllib.request
-import xml.etree.ElementTree as ET
-from pathlib import Path
-
-suite = os.getenv("SUITE_NAME", "titan-iac")
-status = os.getenv("QUALITY_STATUS", "failed")
-gateway = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091").rstrip("/")
-junit_path = Path("build/junit-glue.xml")
-
-totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
-if junit_path.exists():
-    root = ET.parse(junit_path).getroot()
-    suites = [root] if root.tag == "testsuite" else list(root.findall("testsuite")) if root.tag == "testsuites" else []
-    for node in suites:
-        for key in totals:
-            raw = node.attrib.get(key) or "0"
-            try:
-                totals[key] += int(float(raw))
-            except ValueError:
-                pass
-
-passed = max(totals["tests"] - totals["failures"] - totals["errors"] - totals["skipped"], 0)
-
-def read_metrics() -> str:
-    try:
-        with urllib.request.urlopen(f"{gateway}/metrics", timeout=10) as resp:
-            return resp.read().decode("utf-8", errors="replace")
-    except Exception:
-        return ""
-
-def read_counter(text: str, counter_status: str) -> float:
-    for line in text.splitlines():
-        if not line.startswith("platform_quality_gate_runs_total{"):
-            continue
-        if 'job="platform-quality-ci"' not in line:
-            continue
-        if f'suite="{suite}"' not in line:
-            continue
-        if f'status="{counter_status}"' not in line:
-            continue
-        parts = line.split()
-        if len(parts) < 2:
-            continue
-        try:
-            return float(parts[1])
-        except ValueError:
-            return 0.0
-    return 0.0
-
-metrics = read_metrics()
-ok_count = read_counter(metrics, "ok")
-failed_count = read_counter(metrics, "failed")
-if status == "ok":
-    ok_count += 1
-else:
-    failed_count += 1
-
-payload = "\n".join(
-    [
-        "# TYPE platform_quality_gate_runs_total counter",
-        f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count:.0f}',
-        f'platform_quality_gate_runs_total{{suite="{suite}",status="failed"}} {failed_count:.0f}',
-        "# TYPE titan_iac_quality_gate_tests_total gauge",
-        f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="passed"}} {passed}',
-        f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="failed"}} {totals["failures"]}',
-        f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="error"}} {totals["errors"]}',
-        f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="skipped"}} {totals["skipped"]}',
-    ]
-) + "\n"
-
-req = urllib.request.Request(
-    f"{gateway}/metrics/job/platform-quality-ci/suite/{suite}",
-    data=payload.encode("utf-8"),
-    method="POST",
-    headers={"Content-Type": "text/plain"},
-)
-with urllib.request.urlopen(req, timeout=10) as resp:
-    if resp.status >= 400:
-        raise RuntimeError(f"push failed: {resp.status}")
-PY
-      '''
-      archiveArtifacts artifacts: 'build/junit-glue.xml', allowEmptyArchive: true
     }
   }
 }

ci/requirements.txt

@@ -1,4 +1,7 @@
 pytest==8.3.4
+pytest-cov==6.0.0
+coverage==7.6.10
 kubernetes==30.1.0
 PyYAML==6.0.2
 requests==2.32.3
+ruff==0.8.4

ci/scripts/publish_test_metrics.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+"""Publish titan-iac quality-gate results to Pushgateway."""
+
+from __future__ import annotations
+
+import json
+import os
+from glob import glob
+import urllib.error
+import urllib.request
+import xml.etree.ElementTree as ET
+
+
+def _escape_label(value: str) -> str:
+    return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
+
+
+def _label_str(labels: dict[str, str]) -> str:
+    parts = [f'{key}="{_escape_label(val)}"' for key, val in labels.items() if val]
+    return "{" + ",".join(parts) + "}" if parts else ""
+
+
+def _read_text(url: str) -> str:
+    with urllib.request.urlopen(url, timeout=10) as response:
+        return response.read().decode("utf-8")
+
+
+def _post_text(url: str, payload: str) -> None:
+    request = urllib.request.Request(
+        url,
+        data=payload.encode("utf-8"),
+        method="POST",
+        headers={"Content-Type": "text/plain"},
+    )
+    with urllib.request.urlopen(request, timeout=10) as response:
+        if response.status >= 400:
+            raise RuntimeError(f"push failed with status={response.status}")
+
+
+def _parse_junit(path: str) -> dict[str, int]:
+    if not os.path.exists(path):
+        return {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
+
+    tree = ET.parse(path)
+    root = tree.getroot()
+    totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
+
+    suites: list[ET.Element]
+    if root.tag == "testsuite":
+        suites = [root]
+    elif root.tag == "testsuites":
+        suites = [elem for elem in root if elem.tag == "testsuite"]
+    else:
+        suites = []
+
+    for suite in suites:
+        for key in totals:
+            raw_value = suite.attrib.get(key, "0")
+            try:
+                totals[key] += int(float(raw_value))
+            except ValueError:
+                totals[key] += 0
+    return totals
+
+
+def _collect_junit_totals(pattern: str) -> dict[str, int]:
+    totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
+    for path in sorted(glob(pattern)):
+        parsed = _parse_junit(path)
+        for key in totals:
+            totals[key] += parsed[key]
+    return totals
+
+
+def _read_exit_code(path: str) -> int:
+    try:
+        with open(path, "r", encoding="utf-8") as handle:
+            return int(handle.read().strip())
+    except (FileNotFoundError, ValueError):
+        return 1
+
+
+def _load_summary(path: str) -> dict:
+    try:
+        with open(path, "r", encoding="utf-8") as handle:
+            return json.load(handle)
+    except (FileNotFoundError, json.JSONDecodeError):
+        return {}
+
+
+def _fetch_existing_counter(pushgateway_url: str, metric: str, labels: dict[str, str]) -> float:
+    text = _read_text(f"{pushgateway_url.rstrip('/')}/metrics")
+    for line in text.splitlines():
+        if not line.startswith(metric + "{"):
+            continue
+        if any(f'{key}="{value}"' not in line for key, value in labels.items()):
+            continue
+        parts = line.split()
+        if len(parts) < 2:
+            continue
+        try:
+            return float(parts[1])
+        except ValueError:
+            return 0.0
+    return 0.0
+
+
+def _build_payload(
+    suite: str,
+    status: str,
+    tests: dict[str, int],
+    ok_count: int,
+    failed_count: int,
+    branch: str,
+    build_number: str,
+    summary: dict | None = None,
+) -> str:
+    passed = max(tests["tests"] - tests["failures"] - tests["errors"] - tests["skipped"], 0)
+    build_labels = _label_str(
+        {
+            "suite": suite,
+            "branch": branch or "unknown",
+            "build_number": build_number or "unknown",
+        }
+    )
+    lines = [
+        "# TYPE platform_quality_gate_runs_total counter",
+        f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count}',
+        f'platform_quality_gate_runs_total{{suite="{suite}",status="failed"}} {failed_count}',
+        "# TYPE titan_iac_quality_gate_tests_total gauge",
+        f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="passed"}} {passed}',
+        f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="failed"}} {tests["failures"]}',
+        f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="error"}} {tests["errors"]}',
+        f'titan_iac_quality_gate_tests_total{{suite="{suite}",result="skipped"}} {tests["skipped"]}',
+        "# TYPE titan_iac_quality_gate_run_status gauge",
+        f'titan_iac_quality_gate_run_status{{suite="{suite}",status="ok"}} {1 if status == "ok" else 0}',
+        f'titan_iac_quality_gate_run_status{{suite="{suite}",status="failed"}} {1 if status == "failed" else 0}',
+        "# TYPE titan_iac_quality_gate_build_info gauge",
+        f"titan_iac_quality_gate_build_info{build_labels} 1",
+    ]
+    results = summary.get("results", []) if isinstance(summary, dict) else []
+    if results:
+        lines.append("# TYPE titan_iac_quality_gate_checks_total gauge")
+        for result in results:
+            check_name = result.get("name")
+            check_status = result.get("status")
+            if not check_name or not check_status:
+                continue
+            lines.append(
+                f'titan_iac_quality_gate_checks_total{{suite="{suite}",check="{_escape_label(str(check_name))}",result="{_escape_label(str(check_status))}"}} 1'
+            )
+    return "\n".join(lines) + "\n"
+
+
+def main() -> int:
+    suite = os.getenv("SUITE_NAME", "titan-iac")
+    pushgateway_url = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091")
+    job_name = os.getenv("QUALITY_GATE_JOB_NAME", "platform-quality-ci")
+    junit_glob = os.getenv("JUNIT_GLOB", os.getenv("JUNIT_PATH", "build/junit-*.xml"))
+    exit_code_path = os.getenv("QUALITY_GATE_EXIT_CODE_PATH", os.getenv("GLUE_EXIT_CODE_PATH", "build/quality-gate.rc"))
+    summary_path = os.getenv("QUALITY_GATE_SUMMARY_PATH", "build/quality-gate-summary.json")
+    branch = os.getenv("BRANCH_NAME", os.getenv("GIT_BRANCH", ""))
+    build_number = os.getenv("BUILD_NUMBER", "")
+
+    tests = _collect_junit_totals(junit_glob)
+    exit_code = _read_exit_code(exit_code_path)
+    status = "ok" if exit_code == 0 else "failed"
+    summary = _load_summary(summary_path)
+
+    ok_count = int(
+        _fetch_existing_counter(
+            pushgateway_url,
+            "platform_quality_gate_runs_total",
+            {"job": job_name, "suite": suite, "status": "ok"},
+        )
+    )
+    failed_count = int(
+        _fetch_existing_counter(
+            pushgateway_url,
+            "platform_quality_gate_runs_total",
+            {"job": job_name, "suite": suite, "status": "failed"},
+        )
+    )
+    if status == "ok":
+        ok_count += 1
+    else:
+        failed_count += 1
+
+    payload = _build_payload(
+        suite=suite,
+        status=status,
+        tests=tests,
+        ok_count=ok_count,
+        failed_count=failed_count,
+        branch=branch,
+        build_number=build_number,
+        summary=summary,
+    )
+    push_url = f"{pushgateway_url.rstrip('/')}/metrics/job/{job_name}/suite/{suite}"
+    _post_text(push_url, payload)
+
+    summary = {
+        "suite": suite,
+        "status": status,
+        "tests_total": tests["tests"],
+        "tests_failed": tests["failures"],
+        "tests_error": tests["errors"],
+        "tests_skipped": tests["skipped"],
+        "ok_count": ok_count,
+        "failed_count": failed_count,
+        "checks_recorded": len(summary.get("results", [])) if isinstance(summary, dict) else 0,
+    }
+    print(json.dumps(summary, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

ci/tests/glue/config.yaml

@@ -1,6 +1,7 @@
 max_success_age_hours: 48
 allow_suspended:
   - bstein-dev-home/vaultwarden-cred-sync
+  - comms/guest-name-randomizer
   - comms/othrys-room-reset
   - comms/pin-othrys-invite
   - comms/seed-othrys-room
@@ -9,6 +10,7 @@ allow_suspended:
   - health/wger-user-sync
   - mailu-mailserver/mailu-sync-nightly
   - nextcloud/nextcloud-mail-sync
+  - vault/vault-oidc-config
 ariadne_schedule_tasks:
   - schedule.mailu_sync
   - schedule.nextcloud_sync

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+addopts = -ra
+norecursedirs = .git .venv .venv-ci __pycache__ tmp

scripts/dashboards_render_atlas.py

@@ -35,6 +35,7 @@ data:
 PROM_DS = {"type": "prometheus", "uid": "atlas-vm"}
 PUBLIC_FOLDER = "overview"
 PRIVATE_FOLDER = "atlas-internal"
+ASTRAIOS_MOUNTPOINT = "/mnt/astraios"
 
 PERCENT_THRESHOLDS = {
     "mode": "absolute",
@@ -156,6 +157,10 @@ def root_usage_expr(scope=""):
     return filesystem_usage_expr("/", scope)
 
 
+def astraios_usage_expr(scope=""):
+    return filesystem_usage_expr(ASTRAIOS_MOUNTPOINT, scope)
+
+
 def astreae_usage_expr(mount):
     return (
         f"100 - (sum(node_filesystem_avail_bytes{{mountpoint=\"{mount}\",fstype!~\"tmpfs|overlay\"}}) / "
@@ -419,75 +424,10 @@ ARIADNE_SCHEDULE_LAST_ERROR_RANGE_HOURS = (
     "(time() - max_over_time(ariadne_schedule_last_error_timestamp_seconds[$__range])) / 3600"
 )
 ARIADNE_ACCESS_REQUESTS = "ariadne_access_requests_total"
-PLATFORM_TEST_SUCCESS_EVENTS_30D = (
+PLATFORM_TEST_SUITE_NAMES = [
-    '(sum(increase(ariadne_task_runs_total{status="ok"}[30d])) or on() vector(0)) + '
+    "ariadne",
-    '(sum(increase(metis_builds_total{status="ok"}[30d])) or on() vector(0)) + '
+    "metis",
-    '(sum(increase(metis_flashes_total{status="ok"}[30d])) or on() vector(0)) + '
+    "ananke",
-    '(sum(increase(ananke_quality_gate_runs_total{suite="ananke",status="ok"}[30d])) or on() vector(0))'
-)
-PLATFORM_TEST_TOTAL_EVENTS_30D = (
-    "(sum(increase(ariadne_task_runs_total[30d])) or on() vector(0)) + "
-    "(sum(increase(metis_builds_total[30d])) or on() vector(0)) + "
-    "(sum(increase(metis_flashes_total[30d])) or on() vector(0)) + "
-    "(sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[30d])) or on() vector(0))"
-)
-TEST_SUCCESS_RATE = (
-    f"100 * ({PLATFORM_TEST_SUCCESS_EVENTS_30D}) / clamp_min(({PLATFORM_TEST_TOTAL_EVENTS_30D}), 1)"
-)
-TEST_FAILURES_24H_TOTAL = (
-    '(sum(increase(ariadne_task_runs_total{status!="ok"}[24h])) or on() vector(0)) + '
-    '(sum(increase(metis_builds_total{status="error"}[24h])) or on() vector(0)) + '
-    '(sum(increase(metis_flashes_total{status="error"}[24h])) or on() vector(0)) + '
-    '(sum(increase(ananke_quality_gate_runs_total{suite="ananke",status="failed"}[24h])) or on() vector(0)) + '
-    '(sum(increase(platform_quality_gate_runs_total{status!~"ok|passed|success"}[24h])) or on() vector(0))'
-)
-PLATFORM_TEST_FAILURES_24H_BY_SUITE = (
-    'sort_desc(sum by (suite) ('
-    'label_replace(increase(ariadne_task_runs_total{status!="ok"}[24h]), "suite", "ariadne", "__name__", ".*") '
-    'or label_replace(increase(metis_builds_total{status="error"}[24h]), "suite", "metis", "__name__", ".*") '
-    'or label_replace(increase(metis_flashes_total{status="error"}[24h]), "suite", "metis", "__name__", ".*") '
-    'or label_replace(increase(ananke_quality_gate_runs_total{suite="ananke",status="failed"}[24h]), "suite", "ananke", "__name__", ".*") '
-    'or increase(platform_quality_gate_runs_total{status!~"ok|passed|success"}[24h])'
-    '))'
-)
-PLATFORM_TEST_ACTIVITY_30D = (
-    'label_replace(sum by (status) (increase(ariadne_task_runs_total[30d])), "source", "ariadne", "__name__", ".*") '
-    'or label_replace(sum by (status) (increase(metis_builds_total[30d])), "source", "metis-build", "__name__", ".*") '
-    'or label_replace(sum by (status) (increase(metis_flashes_total[30d])), "source", "metis-flash", "__name__", ".*") '
-    'or label_replace(sum by (status) (increase(ananke_quality_gate_runs_total{suite="ananke"}[30d])), "source", "ananke-quality", "__name__", ".*")'
-)
-PLATFORM_TEST_POINT_WINDOW = "1h"
-ARIADNE_SUITE_OK_INTERVAL = f'sum(increase(ariadne_task_runs_total{{status="ok"}}[{PLATFORM_TEST_POINT_WINDOW}]))'
-ARIADNE_SUITE_TOTAL_INTERVAL = f'sum(increase(ariadne_task_runs_total[{PLATFORM_TEST_POINT_WINDOW}]))'
-METIS_SUITE_OK_INTERVAL = (
-    f'(sum(increase(metis_builds_total{{status="ok"}}[{PLATFORM_TEST_POINT_WINDOW}])) + '
-    f'sum(increase(metis_flashes_total{{status="ok"}}[{PLATFORM_TEST_POINT_WINDOW}])))'
-)
-METIS_SUITE_TOTAL_INTERVAL = (
-    f'(sum(increase(metis_builds_total[{PLATFORM_TEST_POINT_WINDOW}])) + '
-    f'sum(increase(metis_flashes_total[{PLATFORM_TEST_POINT_WINDOW}])))'
-)
-ANANKE_SUITE_OK_INTERVAL = (
-    f'sum(increase(ananke_quality_gate_runs_total{{suite="ananke",status="ok"}}[{PLATFORM_TEST_POINT_WINDOW}]))'
-)
-ANANKE_SUITE_TOTAL_INTERVAL = (
-    f'sum(increase(ananke_quality_gate_runs_total{{suite="ananke"}}[{PLATFORM_TEST_POINT_WINDOW}]))'
-)
-
-PLATFORM_TEST_SUCCESS_RATE_ARIADNE_SERIES = (
-    f'(100 * ({ARIADNE_SUITE_OK_INTERVAL}) / clamp_min(({ARIADNE_SUITE_TOTAL_INTERVAL}), 1)) '
-    f'and on() (({ARIADNE_SUITE_TOTAL_INTERVAL}) > 0)'
-)
-PLATFORM_TEST_SUCCESS_RATE_METIS_SERIES = (
-    f'(100 * ({METIS_SUITE_OK_INTERVAL}) / clamp_min(({METIS_SUITE_TOTAL_INTERVAL}), 1)) '
-    f'and on() (({METIS_SUITE_TOTAL_INTERVAL}) > 0)'
-)
-PLATFORM_TEST_SUCCESS_RATE_ANANKE_SERIES = (
-    f'(100 * ({ANANKE_SUITE_OK_INTERVAL}) / clamp_min(({ANANKE_SUITE_TOTAL_INTERVAL}), 1)) '
-    f'and on() (({ANANKE_SUITE_TOTAL_INTERVAL}) > 0)'
-)
-
-PLATFORM_TEST_GENERIC_SUITE_NAMES = [
     "atlasbot",
     "lesavka",
     "pegasus",
@@ -495,52 +435,47 @@ PLATFORM_TEST_GENERIC_SUITE_NAMES = [
     "titan-iac",
     "bstein-home",
     "arcanagon",
+    "data-prepper",
 ]
-PLATFORM_TEST_GENERIC_CI_SELECTOR = 'exported_job="platform-quality-ci"'
+PLATFORM_TEST_SUITE_MATCHER = "|".join(PLATFORM_TEST_SUITE_NAMES)
-PLATFORM_TEST_GENERIC_SUITE_TARGETS = [
+PLATFORM_TEST_SUCCESS_EVENTS_30D = (
+    f'(sum(increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}",status=~"ok|passed|success"}}[30d])) or on() vector(0))'
+)
+PLATFORM_TEST_TOTAL_EVENTS_30D = (
+    f'(sum(increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}"}}[30d])) or on() vector(0))'
+)
+TEST_SUCCESS_RATE = (
+    f"100 * ({PLATFORM_TEST_SUCCESS_EVENTS_30D}) / clamp_min(({PLATFORM_TEST_TOTAL_EVENTS_30D}), 1)"
+)
+TEST_FAILURES_24H_TOTAL = (
+    f'(sum(increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}",status!~"ok|passed|success"}}[24h])) or on() vector(0))'
+)
+PLATFORM_TEST_FAILURES_24H_BY_SUITE = (
+    f'sort_desc(sum by (suite) (increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}",status!~"ok|passed|success"}}[24h])))'
+)
+PLATFORM_TEST_ACTIVITY_30D = (
+    f'sum by (suite, status) (increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}"}}[30d]))'
+)
+PLATFORM_TEST_POINT_WINDOW = "1h"
+PLATFORM_TEST_SUCCESS_RATE_SUITE_TARGETS = [
     {
-        "refId": chr(ord("D") + index),
+        "refId": chr(ord("A") + index),
         "expr": (
-            f'(100 * (sum(increase(platform_quality_gate_runs_total{{{PLATFORM_TEST_GENERIC_CI_SELECTOR},suite="{suite}",status=~"ok|passed|success"}}'
+            f'(100 * (sum(increase(platform_quality_gate_runs_total{{suite="{suite}",status=~"ok|passed|success"}}'
             f'[{PLATFORM_TEST_POINT_WINDOW}]))) / '
-            f'clamp_min((sum(increase(platform_quality_gate_runs_total{{{PLATFORM_TEST_GENERIC_CI_SELECTOR},suite="{suite}"}}[{PLATFORM_TEST_POINT_WINDOW}]))), 1)) '
+            f'clamp_min((sum(increase(platform_quality_gate_runs_total{{suite="{suite}"}}[{PLATFORM_TEST_POINT_WINDOW}]))), 1)) '
-            f'and on() ((sum(increase(platform_quality_gate_runs_total{{{PLATFORM_TEST_GENERIC_CI_SELECTOR},suite="{suite}"}}[{PLATFORM_TEST_POINT_WINDOW}]))) > 0)'
+            f'and on() ((sum(increase(platform_quality_gate_runs_total{{suite="{suite}"}}[{PLATFORM_TEST_POINT_WINDOW}]))) > 0)'
         ),
         "legendFormat": suite,
     }
-    for index, suite in enumerate(PLATFORM_TEST_GENERIC_SUITE_NAMES)
+    for index, suite in enumerate(PLATFORM_TEST_SUITE_NAMES)
 ]
-
-PLATFORM_TEST_SUCCESS_RATE_SUITE_TARGETS = [
-    {"refId": "A", "expr": PLATFORM_TEST_SUCCESS_RATE_ARIADNE_SERIES, "legendFormat": "ariadne"},
-    {"refId": "B", "expr": PLATFORM_TEST_SUCCESS_RATE_METIS_SERIES, "legendFormat": "metis"},
-    {"refId": "C", "expr": PLATFORM_TEST_SUCCESS_RATE_ANANKE_SERIES, "legendFormat": "ananke"},
-] + PLATFORM_TEST_GENERIC_SUITE_TARGETS
-
-PLATFORM_TEST_SUCCESS_RATE_24H_NATIVE_BY_SUITE = (
-    'label_replace('
-    '(100 * (sum(increase(ariadne_task_runs_total{status="ok"}[24h]))) / clamp_min((sum(increase(ariadne_task_runs_total[24h]))), 1)) '
-    'and on() ((sum(increase(ariadne_task_runs_total[24h]))) > 0), '
-    '"suite", "ariadne", "__name__", ".*") '
-    'or label_replace('
-    '(100 * ((sum(increase(metis_builds_total{status="ok"}[24h])) + sum(increase(metis_flashes_total{status="ok"}[24h])))) '
-    '/ clamp_min(((sum(increase(metis_builds_total[24h])) + sum(increase(metis_flashes_total[24h])))), 1)) '
-    'and on() (((sum(increase(metis_builds_total[24h])) + sum(increase(metis_flashes_total[24h])))) > 0), '
-    '"suite", "metis", "__name__", ".*") '
-    'or label_replace('
-    '(100 * (sum(increase(ananke_quality_gate_runs_total{suite="ananke",status="ok"}[24h]))) '
-    '/ clamp_min((sum(increase(ananke_quality_gate_runs_total{suite="ananke"}[24h]))), 1)) '
-    'and on() ((sum(increase(ananke_quality_gate_runs_total{suite="ananke"}[24h]))) > 0), '
-    '"suite", "ananke", "__name__", ".*")'
-)
-PLATFORM_TEST_SUCCESS_RATE_24H_GENERIC_BY_SUITE = (
-    '(100 * (sum by (suite) (increase(platform_quality_gate_runs_total{exported_job="platform-quality-ci",status=~"ok|passed|success"}[24h]))) '
-    '/ clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{exported_job="platform-quality-ci"}[24h]))), 1)) '
-    'and on(suite) ((sum by (suite) (increase(platform_quality_gate_runs_total{exported_job="platform-quality-ci"}[24h]))) > 0)'
-)
 PLATFORM_TEST_SUCCESS_RATE_24H_BY_SUITE = (
-    f'sort_desc(({PLATFORM_TEST_SUCCESS_RATE_24H_NATIVE_BY_SUITE}) or ({PLATFORM_TEST_SUCCESS_RATE_24H_GENERIC_BY_SUITE}))'
+    f'sort_desc((100 * (sum by (suite) (increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}",status=~"ok|passed|success"}}[24h]))) '
+    f'/ clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}"}}[24h]))), 1)) '
+    f'and on(suite) ((sum by (suite) (increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}"}}[24h]))) > 0))'
 )
+PVC_BACKUP_AGE_HOURS_BY_PVC = "sort_desc(max by (namespace, pvc) (pvc_backup_age_hours))"
 ANANKE_SELECTOR = 'job="ananke-power"'
 ANANKE_UPS_DB_NAME = "Pyrphoros"
 ANANKE_UPS_DB_NODE = "titan-db"
@@ -1611,26 +1546,27 @@ def build_overview():
     panels.append(
         bargauge_panel(
             47,
-            "Platform Suite Pass Rate (24h)",
+            "PVC Backup Health / Age",
-            PLATFORM_TEST_SUCCESS_RATE_24H_BY_SUITE,
+            PVC_BACKUP_AGE_HOURS_BY_PVC,
             {"h": 5, "w": 6, "x": 18, "y": 7},
-            unit="percent",
+            unit="h",
             instant=True,
-            legend="{{suite}}",
+            legend="{{namespace}}/{{pvc}}",
             sort_order="desc",
             thresholds={
                 "mode": "absolute",
                 "steps": [
-                    {"color": "red", "value": None},
+                    {"color": "green", "value": None},
-                    {"color": "yellow", "value": 80},
+                    {"color": "yellow", "value": 6},
-                    {"color": "green", "value": 95},
+                    {"color": "orange", "value": 12},
+                    {"color": "red", "value": 24},
                 ],
             },
         )
     )
-    panels[-1]["links"] = link_to("atlas-jobs")
+    panels[-1]["links"] = link_to("atlas-storage")
     panels[-1]["description"] = (
-        "24-hour per-suite pass-rate snapshot. This complements the 7-day trend by showing each suite's current quality posture."
+        "Oldest backup age in hours by PVC. This panel is reserved for the upcoming PVC backup health feed and will show no data until those metrics are published."
     )
 
     panels.append(
@@ -1916,13 +1852,17 @@ def build_overview():
         )
     )
     panels.append(
-        bargauge_panel(
+        timeseries_panel(
             22,
-            "Nodes Closest to Full Root Disks",
+            "Nodes Closest to Full Astraios Disks",
-            f"topk(12, {root_usage_expr()})",
+            astraios_usage_expr(),
             {"h": 16, "w": 12, "x": 12, "y": 71},
             unit="percent",
-            thresholds=PERCENT_THRESHOLDS,
+            legend="{{node}}",
+            legend_calcs=["last"],
+            legend_display="table",
+            legend_placement="right",
+            time_from="1w",
             links=link_to("atlas-storage"),
         )
     )
@@ -2292,6 +2232,19 @@ def build_nodes_dashboard():
             time_from="30d",
         )
     )
+    panels.append(
+        timeseries_panel(
+            9,
+            "Astraios Usage",
+            astraios_usage_expr(),
+            {"h": 9, "w": 24, "x": 0, "y": 44},
+            unit="percent",
+            legend="{{node}}",
+            legend_display="table",
+            legend_placement="right",
+            time_from="30d",
+        )
+    )
     return {
         "uid": "atlas-nodes",
         "title": "Atlas Nodes",

scripts/tests/test_mailu_sync.py

@@ -1,5 +1,7 @@
 import importlib.util
 import pathlib
+import sys
+import types
 
 import pytest
 
@@ -20,6 +22,26 @@ def load_sync_module(monkeypatch):
     }
     for k, v in env.items():
         monkeypatch.setenv(k, v)
+    fake_psycopg2 = types.ModuleType("psycopg2")
+    fake_psycopg2.Error = Exception
+    fake_psycopg2.connect = lambda **kwargs: None
+    fake_psycopg2_extras = types.ModuleType("psycopg2.extras")
+    fake_psycopg2_extras.RealDictCursor = object
+    fake_passlib = types.ModuleType("passlib")
+    fake_passlib_hash = types.ModuleType("passlib.hash")
+
+    class _FakeBcryptSha256:
+        @staticmethod
+        def hash(password):
+            return f"stub:{password}"
+
+    fake_passlib_hash.bcrypt_sha256 = _FakeBcryptSha256
+    fake_passlib.hash = fake_passlib_hash
+
+    monkeypatch.setitem(sys.modules, "psycopg2", fake_psycopg2)
+    monkeypatch.setitem(sys.modules, "psycopg2.extras", fake_psycopg2_extras)
+    monkeypatch.setitem(sys.modules, "passlib", fake_passlib)
+    monkeypatch.setitem(sys.modules, "passlib.hash", fake_passlib_hash)
     module_path = (
         pathlib.Path(__file__).resolve().parents[2]
         / "services"

scripts/verify_jenkins_workspace_cleanup_rollout.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MODE="${1:-dry-run}"
+if [[ "$MODE" != "dry-run" && "$MODE" != "active" ]]; then
+  echo "usage: $0 [dry-run|active]" >&2
+  exit 2
+fi
+
+EXPECTED_DRY_RUN="true"
+PROM_MODE="dry_run"
+if [[ "$MODE" == "active" ]]; then
+  EXPECTED_DRY_RUN="false"
+  PROM_MODE="delete"
+fi
+
+KUSTOMIZATION="${KUSTOMIZATION:-maintenance}"
+NAMESPACE="${NAMESPACE:-maintenance}"
+DEPLOYMENT="${DEPLOYMENT:-ariadne}"
+LOCAL_METRICS_PORT="${LOCAL_METRICS_PORT:-18080}"
+
+for cmd in flux kubectl curl grep awk; do
+  if ! command -v "$cmd" >/dev/null 2>&1; then
+    echo "missing required command: $cmd" >&2
+    exit 2
+  fi
+done
+
+echo "[1/5] reconcile Flux kustomization: ${KUSTOMIZATION}"
+flux reconcile kustomization "$KUSTOMIZATION" --namespace flux-system --with-source
+
+echo "[2/5] wait for deployment rollout"
+kubectl -n "$NAMESPACE" rollout status "deployment/$DEPLOYMENT" --timeout=5m
+
+echo "[3/5] verify ariadne env wiring"
+ENV_DUMP="$(kubectl -n "$NAMESPACE" get deployment "$DEPLOYMENT" -o jsonpath='{range .spec.template.spec.containers[0].env[*]}{.name}={.value}{"\n"}{end}')"
+echo "$ENV_DUMP" | grep -F "ARIADNE_SCHEDULE_JENKINS_WORKSPACE_CLEANUP=45 */6 * * *"
+echo "$ENV_DUMP" | grep -F "JENKINS_WORKSPACE_NAMESPACE=jenkins"
+echo "$ENV_DUMP" | grep -F "JENKINS_WORKSPACE_PVC_PREFIX=pvc-workspace-"
+echo "$ENV_DUMP" | grep -F "JENKINS_WORKSPACE_CLEANUP_MIN_AGE_HOURS=24"
+echo "$ENV_DUMP" | grep -F "JENKINS_WORKSPACE_CLEANUP_DRY_RUN=${EXPECTED_DRY_RUN}"
+echo "$ENV_DUMP" | grep -F "JENKINS_WORKSPACE_CLEANUP_MAX_DELETIONS_PER_RUN=20"
+
+echo "[4/5] scrape /metrics and confirm cleanup metrics are exported"
+PF_LOG="$(mktemp)"
+METRICS_FILE="$(mktemp)"
+cleanup() {
+  if [[ -n "${PF_PID:-}" ]]; then
+    kill "$PF_PID" >/dev/null 2>&1 || true
+    wait "$PF_PID" 2>/dev/null || true
+  fi
+  rm -f "$PF_LOG" "$METRICS_FILE"
+}
+trap cleanup EXIT
+
+kubectl -n "$NAMESPACE" port-forward "deployment/$DEPLOYMENT" "${LOCAL_METRICS_PORT}:8080" >"$PF_LOG" 2>&1 &
+PF_PID=$!
+sleep 2
+curl -fsS "http://127.0.0.1:${LOCAL_METRICS_PORT}/metrics" >"$METRICS_FILE"
+grep -F "# HELP ariadne_jenkins_workspace_cleanup_runs_total" "$METRICS_FILE"
+grep -F "# HELP ariadne_jenkins_workspace_cleanup_objects_total" "$METRICS_FILE"
+
+echo "[5/5] show recent cleanup signal"
+if grep -q "ariadne_jenkins_workspace_cleanup_runs_total" "$METRICS_FILE"; then
+  grep "ariadne_jenkins_workspace_cleanup_runs_total" "$METRICS_FILE" | grep "mode=\"${PROM_MODE}\"" || true
+else
+  echo "No run counter sample yet for mode=${PROM_MODE}; wait for schedule window and re-run." >&2
+fi
+
+echo "Recent cleanup logs (if any):"
+kubectl -n "$NAMESPACE" logs "deployment/$DEPLOYMENT" --tail=500 | grep -i "jenkins workspace cleanup" | tail -n 20 || true
+
+echo "verification complete for mode=${MODE}"

services/comms/element-call-deployment.yaml

@@ -17,6 +17,7 @@ spec:
     spec:
       nodeSelector:
         hardware: rpi5
+        node-role.kubernetes.io/worker: "true"
       containers:
         - name: element-call
           image: ghcr.io/element-hq/element-call@sha256:e6897c7818331714eae19d83ef8ea94a8b41115f0d8d3f62c2fed2d02c65c9bc

services/comms/helmrelease.yaml

@@ -119,6 +119,7 @@ spec:
           > /synapse/config/conf.d/runtime-secrets.yaml
       nodeSelector:
         hardware: rpi5
+        node-role.kubernetes.io/worker: "true"
       affinity:
         nodeAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -417,6 +418,7 @@ spec:
 
     nodeSelector:
       hardware: rpi5
+      node-role.kubernetes.io/worker: "true"
 
     affinity:
       nodeAffinity:

services/jellyfin/oidc/Jenkinsfile

@@ -1,684 +0,0 @@
-pipeline {
-  agent {
-    kubernetes {
-      yaml """
-apiVersion: v1
-kind: Pod
-spec:
-  restartPolicy: Never
-  containers:
-    - name: dotnet
-      image: mcr.microsoft.com/dotnet/sdk:9.0
-      command:
-        - cat
-      tty: true
-"""
-    }
-  }
-  options {
-    timestamps()
-  }
-  parameters {
-    string(name: 'HARBOR_REPO', defaultValue: 'registry.bstein.dev/streaming/oidc-plugin', description: 'OCI repository for the plugin artifact')
-    string(name: 'JELLYFIN_VERSION', defaultValue: '10.11.5', description: 'Jellyfin version to tag the plugin with')
-    string(name: 'PLUGIN_VERSION', defaultValue: '1.0.2.0', description: 'Plugin version')
-  }
-  environment {
-    ORAS_VERSION = "1.2.0"
-    DOTNET_CLI_TELEMETRY_OPTOUT = "1"
-    DOTNET_SKIP_FIRST_TIME_EXPERIENCE = "1"
-    SUITE_NAME = "jellyfin-oidc-plugin"
-    PUSHGATEWAY_URL = "http://platform-quality-gateway.monitoring.svc.cluster.local:9091"
-  }
-  stages {
-    stage('Checkout') {
-      steps {
-        container('dotnet') {
-          checkout scm
-        }
-      }
-    }
-    stage('Build plugin') {
-      steps {
-        container('dotnet') {
-          sh '''
-            set -euo pipefail
-            apt-get update
-            apt-get install -y --no-install-recommends zip curl ca-certificates git
-            WORKDIR="$(pwd)/build"
-            SRC_DIR="${WORKDIR}/src"
-            DIST_DIR="${WORKDIR}/dist"
-            ART_DIR="${WORKDIR}/artifact"
-            rm -rf "${SRC_DIR}" "${DIST_DIR}" "${ART_DIR}"
-            mkdir -p "${SRC_DIR}" "${DIST_DIR}" "${ART_DIR}"
-            git clone https://github.com/lolerskatez/JellyfinOIDCPlugin.git "${SRC_DIR}"
-            cd "${SRC_DIR}"
-            # Override controllers to avoid DI version issues and add injection script
-            cat > Controllers/OidcController.cs <<'EOF'
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using IdentityModel.OidcClient;
-using MediaBrowser.Controller.Library;
-using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.DependencyInjection;
-
-namespace JellyfinOIDCPlugin.Controllers;
-
-#nullable enable
-
-[ApiController]
-[Route("api/oidc")]
-public class OidcController : ControllerBase
-{
-    private IUserManager UserManager => HttpContext.RequestServices.GetRequiredService<IUserManager>();
-    private static readonly Dictionary<string, object> StateManager = new(); // Store AuthorizeState objects
-
-    [HttpGet("start")]
-    public async Task<IActionResult> Start()
-    {
-        var config = Plugin.Instance?.Configuration;
-        if (config == null)
-        {
-            return BadRequest("Plugin not initialized");
-        }
-
-        var options = new OidcClientOptions
-        {
-            Authority = config.OidEndpoint?.Trim(),
-            ClientId = config.OidClientId?.Trim(),
-            ClientSecret = config.OidSecret?.Trim(),
-            RedirectUri = GetRedirectUri(),
-            Scope = string.Join(" ", config.OidScopes)
-        };
-
-        try
-        {
-            var client = new OidcClient(options);
-            var result = await client.PrepareLoginAsync().ConfigureAwait(false);
-
-            // Store the authorize state for the callback
-            var stateString = (string)result.GetType().GetProperty("State")?.GetValue(result);
-            if (!string.IsNullOrEmpty(stateString))
-            {
-                StateManager[stateString] = result;
-            }
-
-            var startUrl = (string)result.GetType().GetProperty("StartUrl")?.GetValue(result);
-            if (string.IsNullOrEmpty(startUrl))
-            {
-                Console.WriteLine("OIDC: Could not get StartUrl from OIDC result");
-                return BadRequest("OIDC initialization failed");
-            }
-
-            return Redirect(startUrl);
-        }
-        catch (Exception ex)
-        {
-            Console.WriteLine($"OIDC start error: {ex}");
-            return BadRequest("OIDC error: " + ex.Message);
-        }
-    }
-
-    [HttpGet("callback")]
-    public async Task<IActionResult> Callback()
-    {
-        var config = Plugin.Instance?.Configuration;
-        if (config == null)
-        {
-            return BadRequest("Plugin not initialized");
-        }
-
-        try
-        {
-            var stateParam = Request.Query["state"].ToString();
-            if (string.IsNullOrEmpty(stateParam) || !StateManager.TryGetValue(stateParam, out var storedState))
-            {
-                Console.WriteLine($"OIDC: Invalid state {stateParam}");
-                return BadRequest("Invalid state");
-            }
-
-            var options = new OidcClientOptions
-            {
-                Authority = config.OidEndpoint?.Trim(),
-                ClientId = config.OidClientId?.Trim(),
-                ClientSecret = config.OidSecret?.Trim(),
-                RedirectUri = GetRedirectUri(),
-                Scope = string.Join(" ", config.OidScopes)
-            };
-
-            var client = new OidcClient(options);
-            // Cast stored state to AuthorizeState - it's stored as object
-            var authorizeState = (AuthorizeState)storedState;
-            var result = await client.ProcessResponseAsync(Request.QueryString.Value, authorizeState).ConfigureAwait(false);
-
-            if (result.IsError)
-            {
-                Console.WriteLine($"OIDC callback failed: {result.Error} - {result.ErrorDescription}");
-                return BadRequest("OIDC authentication failed");
-            }
-
-            // Get email from claims
-            var email = result.User?.FindFirst("email")?.Value ?? 
-                       result.User?.FindFirst("preferred_username")?.Value ??
-                       result.User?.FindFirst("sub")?.Value;
-            
-            if (string.IsNullOrEmpty(email))
-            {
-                Console.WriteLine("OIDC: No email/username found in OIDC response");
-                return BadRequest("No email/username found in OIDC response");
-            }
-
-            // Get or create user
-            var user = UserManager.GetUserByName(email);
-            if (user == null)
-            {
-                Console.WriteLine($"OIDC: Creating new user {email}");
-                user = await UserManager.CreateUserAsync(email).ConfigureAwait(false);
-            }
-
-            // Set authentication provider
-            user.AuthenticationProviderId = "OIDC";
-
-            // Get roles from claims
-            var rolesClaimValue = result.User?.FindFirst(config.RoleClaim)?.Value;
-            var roles = string.IsNullOrEmpty(rolesClaimValue)
-                ? Array.Empty<string>()
-                : rolesClaimValue.Split(new[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries);
-
-            // Set permissions based on groups
-            var isAdmin = roles.Any(r => r.Equals("admin", StringComparison.OrdinalIgnoreCase));
-            var isPowerUser = roles.Any(r => r.Equals("Power User", StringComparison.OrdinalIgnoreCase)) && !isAdmin;
-
-            Console.WriteLine($"OIDC: User {email} authenticated. Admin: {isAdmin}, PowerUser: {isPowerUser}");
-
-            // Update user in database
-            await UserManager.UpdateUserAsync(user).ConfigureAwait(false);
-
-            StateManager.Remove(stateParam);
-
-            // Redirect to Jellyfin main page
-            return Redirect("/");
-        }
-        catch (Exception ex)
-        {
-            Console.WriteLine($"OIDC callback error: {ex}");
-            return BadRequest("OIDC error: " + ex.Message);
-        }
-    }
-
-    [HttpPost("token")]
-    public async Task<IActionResult> ExchangeToken([FromBody] TokenExchangeRequest request)
-    {
-        var config = Plugin.Instance?.Configuration;
-        if (config == null)
-        {
-            Console.WriteLine("OIDC: Plugin not initialized");
-            return BadRequest("Plugin not initialized");
-        }
-
-        if (string.IsNullOrEmpty(request?.AccessToken))
-        {
-            Console.WriteLine("OIDC: No access token provided");
-            return BadRequest("Access token is required");
-        }
-
-        try
-        {
-            Console.WriteLine("OIDC: Processing token exchange request");
-
-            // Validate the token with the OIDC provider using UserInfo endpoint
-            var options = new OidcClientOptions
-            {
-                Authority = config.OidEndpoint?.Trim(),
-                ClientId = config.OidClientId?.Trim(),
-                ClientSecret = config.OidSecret?.Trim(),
-                Scope = string.Join(" ", config.OidScopes)
-            };
-
-            var client = new OidcClient(options);
-            
-            // Use the access token to get user info
-            var userInfoResult = await client.GetUserInfoAsync(request.AccessToken).ConfigureAwait(false);
-            
-            if (userInfoResult.IsError)
-            {
-                Console.WriteLine($"OIDC: Failed to get user info: {userInfoResult.Error}");
-                return Unauthorized("Invalid access token");
-            }
-
-            // Extract email/username from user info
-            var email = userInfoResult.Claims.FirstOrDefault(c => c.Type == "email")?.Value ??
-                       userInfoResult.Claims.FirstOrDefault(c => c.Type == "preferred_username")?.Value ??
-                       userInfoResult.Claims.FirstOrDefault(c => c.Type == "sub")?.Value;
-
-            if (string.IsNullOrEmpty(email))
-            {
-                Console.WriteLine("OIDC: No email/username found in token");
-                return BadRequest("No email/username found in token");
-            }
-
-            // Get or create user
-            var user = UserManager.GetUserByName(email);
-            if (user == null)
-            {
-                if (!config.AutoCreateUser)
-                {
-                    Console.WriteLine($"OIDC: User {email} not found and auto-create disabled");
-                    return Unauthorized("User does not exist and auto-creation is disabled");
-                }
-
-                Console.WriteLine($"OIDC: Creating new user from token {email}");
-                user = await UserManager.CreateUserAsync(email).ConfigureAwait(false);
-            }
-
-            // Update user authentication provider
-            user.AuthenticationProviderId = "OIDC";
-
-            // Get roles from claims
-            var rolesClaimName = config.RoleClaim ?? "groups";
-            var rolesClaimValue = userInfoResult.Claims.FirstOrDefault(c => c.Type == rolesClaimName)?.Value;
-            var roles = string.IsNullOrEmpty(rolesClaimValue)
-                ? Array.Empty<string>()
-                : rolesClaimValue.Split(new[] { ',', ' ' }, StringSplitOptions.RemoveEmptyEntries);
-
-            // Set permissions based on groups
-            var isAdmin = roles.Any(r => r.Equals("admin", StringComparison.OrdinalIgnoreCase));
-            var isPowerUser = roles.Any(r => r.Equals("Power User", StringComparison.OrdinalIgnoreCase)) && !isAdmin;
-
-            Console.WriteLine($"OIDC: Token exchange for {email} Admin:{isAdmin} Power:{isPowerUser}");
-
-            // Update user in database
-            await UserManager.UpdateUserAsync(user).ConfigureAwait(false);
-
-            // Return success with user info
-            return Ok(new TokenExchangeResponse
-            {
-                Success = true,
-                UserId = user.Id.ToString(),
-                Username = user.Username,
-                Email = email,
-                IsAdmin = isAdmin,
-                Message = "User authenticated successfully"
-            });
-        }
-        catch (Exception ex)
-        {
-            Console.WriteLine($"OIDC token exchange error: {ex}");
-            return StatusCode(500, $"Token exchange failed: {ex.Message}");
-        }
-    }
-
-    private string GetRedirectUri()
-    {
-        var configured = Plugin.Instance?.Configuration?.RedirectUri;
-        if (!string.IsNullOrWhiteSpace(configured))
-        {
-            return configured!;
-        }
-
-        return $"{Request.Scheme}://{Request.Host}/api/oidc/callback";
-    }
-}
-
-public class TokenExchangeRequest
-{
-    public string? AccessToken { get; set; }
-    public string? IdToken { get; set; }
-}
-
-public class TokenExchangeResponse
-{
-    public bool Success { get; set; }
-    public string? UserId { get; set; }
-    public string? Username { get; set; }
-    public string? Email { get; set; }
-    public bool IsAdmin { get; set; }
-    public string? Message { get; set; }
-}
-EOF
-
-            cat > Controllers/OidcStaticController.cs <<'EOF'
-using System;
-using System.IO;
-using System.Reflection;
-using MediaBrowser.Common.Plugins;
-using Microsoft.AspNetCore.Mvc;
-
-namespace JellyfinOIDCPlugin.Controllers;
-
-[ApiController]
-[Route("api/oidc")]
-public class OidcStaticController : ControllerBase
-{
-    [HttpGet("login.js")]
-    public IActionResult GetLoginScript()
-    {
-        try
-        {
-            var assembly = Assembly.GetExecutingAssembly();
-            using var stream = assembly.GetManifestResourceStream("JellyfinOIDCPlugin.web.oidc-login.js");
-            if (stream == null)
-            {
-                Console.WriteLine("OIDC: Login script resource not found");
-                return NotFound();
-            }
-
-            using var reader = new StreamReader(stream);
-            var content = reader.ReadToEnd();
-            
-            return Content(content, "application/javascript");
-        }
-        catch (Exception ex)
-        {
-            Console.WriteLine($"OIDC: Error serving login script {ex}");
-            return StatusCode(500, "Error loading login script");
-        }
-    }
-
-    [HttpGet("loader.js")]
-    public IActionResult GetLoader()
-    {
-        try
-        {
-            var assembly = Assembly.GetExecutingAssembly();
-            using var stream = assembly.GetManifestResourceStream("JellyfinOIDCPlugin.web.oidc-loader.js");
-            if (stream == null)
-            {
-                Console.WriteLine("OIDC: Loader script resource not found");
-                return NotFound();
-            }
-
-            using var reader = new StreamReader(stream);
-            var content = reader.ReadToEnd();
-            
-            return Content(content, "application/javascript");
-        }
-        catch (Exception ex)
-        {
-            Console.WriteLine($"OIDC: Error serving loader script {ex}");
-            return StatusCode(500, "Error loading loader script");
-        }
-    }
-
-    [HttpGet("inject")]
-    public IActionResult GetInject()
-    {
-        try
-        {
-            var script = @"
-(function() {
-    console.log('[OIDC Plugin] Bootstrap inject started');
-    
-    // Load oidc-loader.js dynamically
-    const loaderScript = document.createElement('script');
-    loaderScript.src = '/api/oidc/loader.js';
-    loaderScript.type = 'application/javascript';
-    loaderScript.onerror = function() {
-        console.error('[OIDC Plugin] Failed to load loader.js');
-    };
-    loaderScript.onload = function() {
-        console.log('[OIDC Plugin] Loader.js loaded successfully');
-    };
-    
-    // Append to head or body
-    const target = document.head || document.documentElement;
-    target.appendChild(loaderScript);
-    
-    console.log('[OIDC Plugin] Bootstrap script appended to page');
-})();
-";
-            return Content(script, "application/javascript");
-        }
-        catch (Exception ex)
-        {
-            Console.WriteLine($"OIDC: Error serving inject script {ex}");
-            return StatusCode(500, "Error loading inject script");
-        }
-    }
-
-    [HttpGet("global.js")]
-    public IActionResult GetGlobalInjector()
-    {
-        try
-        {
-            var assembly = Assembly.GetExecutingAssembly();
-            using var stream = assembly.GetManifestResourceStream("JellyfinOIDCPlugin.web.oidc-global-injector.js");
-            if (stream == null)
-            {
-                Console.WriteLine("OIDC: Global injector resource not found");
-                return NotFound();
-            }
-
-            using var reader = new StreamReader(stream);
-            var content = reader.ReadToEnd();
-            
-            return Content(content, "application/javascript");
-        }
-        catch (Exception ex)
-        {
-            Console.WriteLine($"OIDC: Error serving global injector {ex}");
-            return StatusCode(500, "Error loading global injector");
-        }
-    }
-
-    [HttpGet("config")]
-    public IActionResult GetConfigurationPage()
-    {
-        try
-        {
-            var assembly = Assembly.GetExecutingAssembly();
-            using var stream = assembly.GetManifestResourceStream("JellyfinOIDCPlugin.web.configurationpage.html");
-            if (stream == null)
-            {
-                Console.WriteLine("OIDC: Configuration page resource not found");
-                return NotFound("Configuration page resource not found");
-            }
-
-            using var reader = new StreamReader(stream);
-            var content = reader.ReadToEnd();
-            
-            return Content(content, "text/html");
-        }
-        catch (Exception ex)
-        {
-            Console.WriteLine($"OIDC: Error serving configuration page {ex}");
-            return StatusCode(500, $"Error loading configuration page: {ex.Message}");
-        }
-    }
-}
-EOF
-            cat > JellyfinOIDCPlugin.csproj <<'EOF'
-<Project Sdk="Microsoft.NET.Sdk">
-  <PropertyGroup>
-    <TargetFramework>net9.0</TargetFramework>
-    <AssemblyName>JellyfinOIDCPlugin.v2</AssemblyName>
-    <RootNamespace>JellyfinOIDCPlugin</RootNamespace>
-    <LangVersion>latest</LangVersion>
-    <Nullable>enable</Nullable>
-    <ImplicitUsings>enable</ImplicitUsings>
-    <AssemblyVersion>1.0.2.0</AssemblyVersion>
-    <FileVersion>1.0.2.0</FileVersion>
-    <CopyLocalLockFileAssemblies>false</CopyLocalLockFileAssemblies>
-  </PropertyGroup>
-  <ItemGroup>
-    <PackageReference Include="Jellyfin.Controller" Version="10.11.5">
-      <ExcludeAssets>runtime</ExcludeAssets>
-    </PackageReference>
-    <PackageReference Include="Jellyfin.Model" Version="10.11.5">
-      <ExcludeAssets>runtime</ExcludeAssets>
-    </PackageReference>
-    <PackageReference Include="Jellyfin.Common" Version="10.11.5">
-      <ExcludeAssets>runtime</ExcludeAssets>
-    </PackageReference>
-    <PackageReference Include="Jellyfin.Data" Version="10.11.5">
-      <ExcludeAssets>runtime</ExcludeAssets>
-    </PackageReference>
-    <PackageReference Include="Jellyfin.Database.Implementations" Version="10.11.5">
-      <ExcludeAssets>runtime</ExcludeAssets>
-    </PackageReference>
-    <PackageReference Include="IdentityModel.OidcClient" Version="5.2.1">
-      <PrivateAssets>none</PrivateAssets>
-    </PackageReference>
-    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.11">
-      <ExcludeAssets>runtime</ExcludeAssets>
-    </PackageReference>
-  </ItemGroup>
-  <ItemGroup>
-    <EmbeddedResource Include="web\\*.html" />
-    <EmbeddedResource Include="web\\*.js" />
-    <EmbeddedResource Include="web\\*.css" />
-  </ItemGroup>
-</Project>
-EOF
-            dotnet restore
-            dotnet publish -c Release --no-self-contained -o "${DIST_DIR}"
-            cd "${DIST_DIR}"
-            zip -r "${ART_DIR}/OIDC_Authentication_${PLUGIN_VERSION}-net9.zip" .
-          '''
-        }
-      }
-    }
-    stage('Quality gate smoke tests') {
-      steps {
-        container('dotnet') {
-          sh '''
-            set -euo pipefail
-            apt-get update
-            apt-get install -y --no-install-recommends unzip
-            WORKDIR="$(pwd)/build"
-            ARTIFACT="${WORKDIR}/artifact/OIDC_Authentication_${PLUGIN_VERSION}-net9.zip"
-            test -s "${ARTIFACT}"
-            unzip -l "${ARTIFACT}" > "${WORKDIR}/artifact-list.txt"
-            grep -q 'JellyfinOIDCPlugin.v2.dll' "${WORKDIR}/artifact-list.txt"
-            cat > "${WORKDIR}/quality-summary.env" <<'EOF'
-tests=1
-passed=1
-failed=0
-errors=0
-skipped=0
-EOF
-          '''
-        }
-      }
-    }
-    stage('Push to Harbor') {
-      steps {
-        container('dotnet') {
-          withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD')]) {
-            sh '''
-              set -euo pipefail
-              WORKDIR="$(pwd)/build"
-              ORAS_BIN="/usr/local/bin/oras"
-              curl -sSL "https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz" | tar -xz -C /usr/local/bin oras
-              ref_host="$(echo "${HARBOR_REPO}" | cut -d/ -f1)"
-              "${ORAS_BIN}" login "${ref_host}" -u "${HARBOR_USERNAME}" -p "${HARBOR_PASSWORD}"
-              artifact="${WORKDIR}/artifact/OIDC_Authentication_${PLUGIN_VERSION}-net9.zip"
-              "${ORAS_BIN}" push "${HARBOR_REPO}:${JELLYFIN_VERSION}" "${artifact}:application/zip" --artifact-type application/zip
-              "${ORAS_BIN}" push "${HARBOR_REPO}:latest" "${artifact}:application/zip" --artifact-type application/zip
-            '''
-          }
-        }
-      }
-    }
-  }
-  post {
-    always {
-      script {
-        env.QUALITY_STATUS = currentBuild.currentResult == 'SUCCESS' ? 'ok' : 'failed'
-      }
-      container('dotnet') {
-        sh '''
-          set -euo pipefail
-          python - <<'PY'
-import os
-import urllib.request
-from pathlib import Path
-
-suite = os.getenv("SUITE_NAME", "jellyfin-oidc-plugin")
-status = os.getenv("QUALITY_STATUS", "failed")
-gateway = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091").rstrip("/")
-summary_path = Path("build/quality-summary.env")
-
-totals = {"tests": 1, "passed": 0, "failed": 1, "errors": 0, "skipped": 0}
-if summary_path.exists():
-    for raw in summary_path.read_text(encoding="utf-8").splitlines():
-        if "=" not in raw:
-            continue
-        key, value = raw.split("=", 1)
-        key = key.strip()
-        if key in totals:
-            try:
-                totals[key] = int(float(value.strip()))
-            except ValueError:
-                pass
-if status != "ok":
-    totals["failed"] = max(totals["failed"], 1)
-    totals["passed"] = 0
-
-def read_metrics() -> str:
-    try:
-        with urllib.request.urlopen(f"{gateway}/metrics", timeout=10) as resp:
-            return resp.read().decode("utf-8", errors="replace")
-    except Exception:
-        return ""
-
-def read_counter(text: str, counter_status: str) -> float:
-    for line in text.splitlines():
-        if not line.startswith("platform_quality_gate_runs_total{"):
-            continue
-        if 'job="platform-quality-ci"' not in line:
-            continue
-        if f'suite="{suite}"' not in line:
-            continue
-        if f'status="{counter_status}"' not in line:
-            continue
-        parts = line.split()
-        if len(parts) < 2:
-            continue
-        try:
-            return float(parts[1])
-        except ValueError:
-            return 0.0
-    return 0.0
-
-metrics = read_metrics()
-ok_count = read_counter(metrics, "ok")
-failed_count = read_counter(metrics, "failed")
-if status == "ok":
-    ok_count += 1
-else:
-    failed_count += 1
-
-payload = "\n".join(
-    [
-        "# TYPE platform_quality_gate_runs_total counter",
-        f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count:.0f}',
-        f'platform_quality_gate_runs_total{{suite="{suite}",status="failed"}} {failed_count:.0f}',
-        "# TYPE jellyfin_oidc_quality_gate_tests_total gauge",
-        f'jellyfin_oidc_quality_gate_tests_total{{suite="{suite}",result="passed"}} {totals["passed"]}',
-        f'jellyfin_oidc_quality_gate_tests_total{{suite="{suite}",result="failed"}} {totals["failed"]}',
-        f'jellyfin_oidc_quality_gate_tests_total{{suite="{suite}",result="error"}} {totals["errors"]}',
-        f'jellyfin_oidc_quality_gate_tests_total{{suite="{suite}",result="skipped"}} {totals["skipped"]}',
-    ]
-) + "\n"
-
-req = urllib.request.Request(
-    f"{gateway}/metrics/job/platform-quality-ci/suite/{suite}",
-    data=payload.encode("utf-8"),
-    method="POST",
-    headers={"Content-Type": "text/plain"},
-)
-with urllib.request.urlopen(req, timeout=10) as resp:
-    if resp.status >= 400:
-        raise RuntimeError(f"push failed: {resp.status}")
-PY
-        '''
-      }
-      container('dotnet') {
-        archiveArtifacts artifacts: 'build/artifact/*.zip,build/artifact-list.txt,build/quality-summary.env', allowEmptyArchive: true
-      }
-    }
-  }
-}

services/jenkins/configmap-jcasc.yaml

@@ -73,58 +73,6 @@ data:
               }
             }
           }
-          pipelineJob('jellyfin-oidc-plugin') {
-            properties {
-              pipelineTriggers {
-                triggers {
-                  scmTrigger {
-                    scmpoll_spec('H/5 * * * *')
-                    ignorePostCommitHooks(false)
-                  }
-                }
-              }
-            }
-            definition {
-              cpsScm {
-                scm {
-                  git {
-                    remote {
-                      url('https://scm.bstein.dev/bstein/titan-iac.git')
-                      credentials('gitea-pat')
-                    }
-                    branches('*/main')
-                  }
-                }
-                scriptPath('services/jellyfin/oidc/Jenkinsfile')
-              }
-            }
-          }
-          pipelineJob('ci-demo') {
-            properties {
-              pipelineTriggers {
-                triggers {
-                  scmTrigger {
-                    scmpoll_spec('H/1 * * * *')
-                    ignorePostCommitHooks(false)
-                  }
-                }
-              }
-            }
-            definition {
-              cpsScm {
-                scm {
-                  git {
-                    remote {
-                      url('https://scm.bstein.dev/bstein/ci-demo.git')
-                      credentials('gitea-pat')
-                    }
-                    branches('*/master')
-                  }
-                }
-                scriptPath('Jenkinsfile')
-              }
-            }
-          }
           pipelineJob('bstein-dev-home') {
             properties {
               pipelineTriggers {
@@ -177,32 +125,6 @@ data:
               }
             }
           }
-          pipelineJob('atlasbot') {
-            properties {
-              pipelineTriggers {
-                triggers {
-                  scmTrigger {
-                    scmpoll_spec('H/5 * * * *')
-                    ignorePostCommitHooks(false)
-                  }
-                }
-              }
-            }
-            definition {
-              cpsScm {
-                scm {
-                  git {
-                    remote {
-                      url('https://scm.bstein.dev/bstein/atlasbot.git')
-                      credentials('gitea-pat')
-                    }
-                    branches('*/main')
-                  }
-                }
-                scriptPath('Jenkinsfile')
-              }
-            }
-          }
           pipelineJob('metis') {
             properties {
               pipelineTriggers {
@@ -229,6 +151,58 @@ data:
               }
             }
           }
+          pipelineJob('ananke') {
+            properties {
+              pipelineTriggers {
+                triggers {
+                  scmTrigger {
+                    scmpoll_spec('H/5 * * * *')
+                    ignorePostCommitHooks(false)
+                  }
+                }
+              }
+            }
+            definition {
+              cpsScm {
+                scm {
+                  git {
+                    remote {
+                      url('https://scm.bstein.dev/bstein/ananke.git')
+                      credentials('gitea-pat')
+                    }
+                    branches('*/main')
+                  }
+                }
+                scriptPath('Jenkinsfile')
+              }
+            }
+          }
+          pipelineJob('lesavka') {
+            properties {
+              pipelineTriggers {
+                triggers {
+                  scmTrigger {
+                    scmpoll_spec('H/5 * * * *')
+                    ignorePostCommitHooks(false)
+                  }
+                }
+              }
+            }
+            definition {
+              cpsScm {
+                scm {
+                  git {
+                    remote {
+                      url('https://scm.bstein.dev/bstein/lesavka.git')
+                      credentials('gitea-pat')
+                    }
+                    branches('*/master')
+                  }
+                }
+                scriptPath('Jenkinsfile')
+              }
+            }
+          }
           pipelineJob('pegasus') {
             properties {
               pipelineTriggers {
@@ -255,6 +229,58 @@ data:
               }
             }
           }
+          pipelineJob('atlasbot') {
+            properties {
+              pipelineTriggers {
+                triggers {
+                  scmTrigger {
+                    scmpoll_spec('H/5 * * * *')
+                    ignorePostCommitHooks(false)
+                  }
+                }
+              }
+            }
+            definition {
+              cpsScm {
+                scm {
+                  git {
+                    remote {
+                      url('https://scm.bstein.dev/bstein/atlasbot.git')
+                      credentials('gitea-pat')
+                    }
+                    branches('*/main')
+                  }
+                }
+                scriptPath('Jenkinsfile')
+              }
+            }
+          }
+          pipelineJob('soteria') {
+            properties {
+              pipelineTriggers {
+                triggers {
+                  scmTrigger {
+                    scmpoll_spec('H/5 * * * *')
+                    ignorePostCommitHooks(false)
+                  }
+                }
+              }
+            }
+            definition {
+              cpsScm {
+                scm {
+                  git {
+                    remote {
+                      url('https://scm.bstein.dev/bstein/soteria.git')
+                      credentials('gitea-pat')
+                    }
+                    branches('*/main')
+                  }
+                }
+                scriptPath('Jenkinsfile')
+              }
+            }
+          }
           pipelineJob('data-prepper') {
             properties {
               pipelineTriggers {
@@ -281,7 +307,7 @@ data:
               }
             }
           }
-          pipelineJob('Soteria') {
+          pipelineJob('titan-iac') {
             properties {
               pipelineTriggers {
                 triggers {
@@ -297,7 +323,7 @@ data:
                 scm {
                   git {
                     remote {
-                      url('https://scm.bstein.dev/bstein/soteria.git')
+                      url('https://scm.bstein.dev/bstein/titan-iac.git')
                       credentials('gitea-pat')
                     }
                     branches('*/main')
@@ -373,10 +399,8 @@ data:
             - name: "default"
               namespace: "jenkins"
               workspaceVolume:
-                dynamicPVC:
+                emptyDirWorkspaceVolume:
-                  accessModes: "ReadWriteOnce"
+                  memory: false
-                  requestsSize: "20Gi"
-                  storageClassName: "astreae"
               containers:
               - name: "jnlp"
                 args: "^${computer.jnlpmac} ^${computer.name}"
@@ -394,7 +418,7 @@ data:
                 workingDir: /home/jenkins/agent
               idleMinutes: 0
               instanceCap: 2147483647
-              label: "jenkins-jenkins-agent "
+              label: "jenkins-jenkins-agent"
               nodeUsageMode: "NORMAL"
               podRetention: Never
               serviceAccount: "jenkins"

services/jenkins/serviceaccount.yaml

@@ -39,3 +39,28 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: jenkins-agent
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: jenkins-glue-observer
+rules:
+  - apiGroups: ["batch"]
+    resources:
+      - cronjobs
+    verbs: ["get", "list", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: jenkins-glue-observer
+subjects:
+  - kind: ServiceAccount
+    name: jenkins
+    namespace: jenkins
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: jenkins-glue-observer

services/logging/Jenkinsfile.data-prepper

@@ -36,8 +36,9 @@ spec:
     PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
   }
   parameters {
-    string(name: 'HARBOR_REPO', defaultValue: 'registry.bstein.dev/streaming/data-prepper', description: 'Docker repository for Data Prepper')
+    string(name: 'HARBOR_REPO', defaultValue: 'registry.bstein.dev/monitoring/data-prepper', description: 'Docker repository for Data Prepper')
     string(name: 'IMAGE_TAG', defaultValue: '2.8.0', description: 'Image tag to publish')
+    booleanParam(name: 'PUSH_IMAGE', defaultValue: false, description: 'Publish image artifacts (manual release only)')
     booleanParam(name: 'PUSH_LATEST', defaultValue: true, description: 'Also push the latest tag')
   }
   stages {
@@ -48,14 +49,17 @@ spec:
         }
       }
     }
-    stage('Build & Push') {
+    stage('Build & Push (optional)') {
+      when {
+        expression { return params.PUSH_IMAGE }
+      }
       steps {
         container('kaniko') {
           withCredentials([usernamePassword(credentialsId: 'harbor-robot', usernameVariable: 'HARBOR_USERNAME', passwordVariable: 'HARBOR_PASSWORD')]) {
             sh '''
               set -euo pipefail
-              if [ -z "${HARBOR_REPO:-}" ] || [ "${HARBOR_REPO}" = "registry.bstein.dev/monitoring/data-prepper" ]; then
+              if [ -z "${HARBOR_REPO:-}" ]; then
-                HARBOR_REPO="registry.bstein.dev/streaming/data-prepper"
+                HARBOR_REPO="registry.bstein.dev/monitoring/data-prepper"
               fi
               IMAGE_TAG_SAFE="${IMAGE_TAG:-2.8.0}"
               mkdir -p /kaniko/.docker
@@ -71,7 +75,7 @@ spec:
               }
 EOF
               dest_args="--destination ${HARBOR_REPO}:${IMAGE_TAG_SAFE}"
-              if [ "${PUSH_LATEST}" = "true" ]; then
+              if [ "${PUSH_LATEST:-true}" = "true" ]; then
                 dest_args="${dest_args} --destination ${HARBOR_REPO}:latest"
               fi
               /kaniko/executor \
@@ -84,7 +88,7 @@ EOF
         }
       }
     }
-    stage('Quality gate smoke build') {
+    stage('Smoke test suite') {
       steps {
         container('kaniko') {
           sh '''
@@ -110,7 +114,7 @@ EOF
           fetch_counter() {
             status="$1"
             line="$(curl -fsS "${gateway}/metrics" 2>/dev/null | awk -v suite="${suite}" -v status="${status}" '
-              /^platform_quality_gate_runs_total{/ {
+              /platform_quality_gate_runs_total/ {
                 if (index($0, "job=\\"platform-quality-ci\\"") && index($0, "suite=\\"" suite "\\"") && index($0, "status=\\"" status "\\"")) {
                   print $2
                   exit
@@ -122,10 +126,15 @@ EOF
           ok_count="$(fetch_counter ok)"
           failed_count="$(fetch_counter failed)"
           ok_count=$((ok_count + 1))
+          tests_passed=1
+          tests_failed=0
           cat <<METRICS | curl -fsS --data-binary @- "${gateway}/metrics/job/platform-quality-ci/suite/${suite}" >/dev/null
 # TYPE platform_quality_gate_runs_total counter
 platform_quality_gate_runs_total{suite="${suite}",status="ok"} ${ok_count}
 platform_quality_gate_runs_total{suite="${suite}",status="failed"} ${failed_count}
+# TYPE data_prepper_quality_gate_tests_total gauge
+data_prepper_quality_gate_tests_total{suite="${suite}",result="passed"} ${tests_passed}
+data_prepper_quality_gate_tests_total{suite="${suite}",result="failed"} ${tests_failed}
 METRICS
         '''
       }
@@ -140,7 +149,7 @@ METRICS
           fetch_counter() {
             status="$1"
             line="$(curl -fsS "${gateway}/metrics" 2>/dev/null | awk -v suite="${suite}" -v status="${status}" '
-              /^platform_quality_gate_runs_total{/ {
+              /platform_quality_gate_runs_total/ {
                 if (index($0, "job=\\"platform-quality-ci\\"") && index($0, "suite=\\"" suite "\\"") && index($0, "status=\\"" status "\\"")) {
                   print $2
                   exit
@@ -152,10 +161,15 @@ METRICS
           ok_count="$(fetch_counter ok)"
           failed_count="$(fetch_counter failed)"
           failed_count=$((failed_count + 1))
+          tests_passed=0
+          tests_failed=1
           cat <<METRICS | curl -fsS --data-binary @- "${gateway}/metrics/job/platform-quality-ci/suite/${suite}" >/dev/null
 # TYPE platform_quality_gate_runs_total counter
 platform_quality_gate_runs_total{suite="${suite}",status="ok"} ${ok_count}
 platform_quality_gate_runs_total{suite="${suite}",status="failed"} ${failed_count}
+# TYPE data_prepper_quality_gate_tests_total gauge
+data_prepper_quality_gate_tests_total{suite="${suite}",result="passed"} ${tests_passed}
+data_prepper_quality_gate_tests_total{suite="${suite}",result="failed"} ${tests_failed}
 METRICS
         '''
       }

services/mailu/helmrelease.yaml

@@ -176,6 +176,7 @@ spec:
       logLevel: DEBUG
       nodeSelector:
         hardware: rpi5
+        node-role.kubernetes.io/worker: "true"
       resources:
         requests:
           cpu: 200m

services/mailu/scripts/mailu_sync.py

@@ -7,7 +7,6 @@ Sync Keycloak users to Mailu mailboxes.
 
 import os
 import sys
-import json
 import time
 import secrets
 import string

services/maintenance/ariadne-deployment.yaml

@@ -345,6 +345,18 @@ spec:
               value: "15"
             - name: ARIADNE_SCHEDULE_METIS_SENTINEL_WATCH
               value: "*/30 * * * *"
+            - name: ARIADNE_SCHEDULE_JENKINS_WORKSPACE_CLEANUP
+              value: "45 */6 * * *"
+            - name: JENKINS_WORKSPACE_NAMESPACE
+              value: jenkins
+            - name: JENKINS_WORKSPACE_PVC_PREFIX
+              value: pvc-workspace-
+            - name: JENKINS_WORKSPACE_CLEANUP_MIN_AGE_HOURS
+              value: "24"
+            - name: JENKINS_WORKSPACE_CLEANUP_DRY_RUN
+              value: "false"
+            - name: JENKINS_WORKSPACE_CLEANUP_MAX_DELETIONS_PER_RUN
+              value: "20"
             - name: METRICS_PATH
               value: "/metrics"
           resources:

services/maintenance/ariadne-rbac.yaml

@@ -21,6 +21,23 @@ rules:
       - list
       - watch
       - delete
+  - apiGroups: [""]
+    resources:
+      - persistentvolumeclaims
+      - persistentvolumes
+    verbs:
+      - get
+      - list
+      - watch
+      - delete
+  - apiGroups: ["longhorn.io"]
+    resources:
+      - volumes
+    verbs:
+      - get
+      - list
+      - watch
+      - delete
   - apiGroups: [""]
     resources:
       - nodes

services/maintenance/vault-sync-deployment.yaml

@@ -14,6 +14,8 @@ spec:
       labels:
         app: maintenance-vault-sync
     spec:
+      nodeSelector:
+        node-role.kubernetes.io/worker: "true"
       serviceAccountName: maintenance-vault-sync
       containers:
         - name: sync

services/monitoring/dashboards/atlas-jobs.json

@@ -1138,7 +1138,7 @@
       },
       "targets": [
         {
-          "expr": "100 * ((sum(increase(ariadne_task_runs_total{status=\"ok\"}[30d])) or on() vector(0)) + (sum(increase(metis_builds_total{status=\"ok\"}[30d])) or on() vector(0)) + (sum(increase(metis_flashes_total{status=\"ok\"}[30d])) or on() vector(0)) + (sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\",status=\"ok\"}[30d])) or on() vector(0))) / clamp_min(((sum(increase(ariadne_task_runs_total[30d])) or on() vector(0)) + (sum(increase(metis_builds_total[30d])) or on() vector(0)) + (sum(increase(metis_flashes_total[30d])) or on() vector(0)) + (sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[30d])) or on() vector(0))), 1)",
+          "expr": "100 * ((sum(increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\",status=~\"ok|passed|success\"}[30d])) or on() vector(0))) / clamp_min(((sum(increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\"}[30d])) or on() vector(0))), 1)",
           "refId": "A",
           "instant": true
         }
@@ -1201,7 +1201,7 @@
       },
       "targets": [
         {
-          "expr": "label_replace(sum by (status) (increase(ariadne_task_runs_total[30d])), \"source\", \"ariadne\", \"__name__\", \".*\") or label_replace(sum by (status) (increase(metis_builds_total[30d])), \"source\", \"metis-build\", \"__name__\", \".*\") or label_replace(sum by (status) (increase(metis_flashes_total[30d])), \"source\", \"metis-flash\", \"__name__\", \".*\") or label_replace(sum by (status) (increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[30d])), \"source\", \"ananke-quality\", \"__name__\", \".*\")",
+          "expr": "sum by (suite, status) (increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\"}[30d]))",
           "refId": "A",
           "instant": true
         }
@@ -1253,17 +1253,17 @@
       "targets": [
         {
           "refId": "A",
-          "expr": "(100 * (sum(increase(ariadne_task_runs_total{status=\"ok\"}[1h]))) / clamp_min((sum(increase(ariadne_task_runs_total[1h]))), 1)) and on() ((sum(increase(ariadne_task_runs_total[1h]))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\"}[1h]))) > 0)",
           "legendFormat": "ariadne"
         },
         {
           "refId": "B",
-          "expr": "(100 * ((sum(increase(metis_builds_total{status=\"ok\"}[1h])) + sum(increase(metis_flashes_total{status=\"ok\"}[1h])))) / clamp_min(((sum(increase(metis_builds_total[1h])) + sum(increase(metis_flashes_total[1h])))), 1)) and on() (((sum(increase(metis_builds_total[1h])) + sum(increase(metis_flashes_total[1h])))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"metis\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"metis\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"metis\"}[1h]))) > 0)",
           "legendFormat": "metis"
         },
         {
           "refId": "C",
-          "expr": "(100 * (sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\",status=\"ok\"}[1h]))) / clamp_min((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[1h]))), 1)) and on() ((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[1h]))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"ananke\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"ananke\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"ananke\"}[1h]))) > 0)",
           "legendFormat": "ananke"
         },
         {
@@ -1300,6 +1300,11 @@
           "refId": "J",
           "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\"}[1h]))) > 0)",
           "legendFormat": "arcanagon"
+        },
+        {
+          "refId": "K",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\"}[1h]))) > 0)",
+          "legendFormat": "data-prepper"
         }
       ],
       "fieldConfig": {

services/monitoring/dashboards/atlas-nodes.json

@@ -584,6 +584,44 @@
         }
       },
       "timeFrom": "30d"
+    },
+    {
+      "id": 9,
+      "type": "timeseries",
+      "title": "Astraios Usage",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "atlas-vm"
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 24,
+        "x": 0,
+        "y": 44
+      },
+      "targets": [
+        {
+          "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+          "refId": "A",
+          "legendFormat": "{{node}}"
+        }
+      ],
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "displayMode": "table",
+          "placement": "right"
+        },
+        "tooltip": {
+          "mode": "multi"
+        }
+      },
+      "timeFrom": "30d"
     }
   ],
   "time": {

services/monitoring/dashboards/atlas-overview.json

@@ -1861,53 +1861,58 @@
       "targets": [
         {
           "refId": "A",
-          "expr": "(100 * (sum(increase(ariadne_task_runs_total{status=\"ok\"}[1h]))) / clamp_min((sum(increase(ariadne_task_runs_total[1h]))), 1)) and on() ((sum(increase(ariadne_task_runs_total[1h]))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\"}[1h]))) > 0)",
           "legendFormat": "ariadne"
         },
         {
           "refId": "B",
-          "expr": "(100 * ((sum(increase(metis_builds_total{status=\"ok\"}[1h])) + sum(increase(metis_flashes_total{status=\"ok\"}[1h])))) / clamp_min(((sum(increase(metis_builds_total[1h])) + sum(increase(metis_flashes_total[1h])))), 1)) and on() (((sum(increase(metis_builds_total[1h])) + sum(increase(metis_flashes_total[1h])))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"metis\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"metis\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"metis\"}[1h]))) > 0)",
           "legendFormat": "metis"
         },
         {
           "refId": "C",
-          "expr": "(100 * (sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\",status=\"ok\"}[1h]))) / clamp_min((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[1h]))), 1)) and on() ((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[1h]))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"ananke\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"ananke\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"ananke\"}[1h]))) > 0)",
           "legendFormat": "ananke"
         },
         {
           "refId": "D",
-          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"atlasbot\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"atlasbot\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"atlasbot\"}[1h]))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"atlasbot\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"atlasbot\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"atlasbot\"}[1h]))) > 0)",
           "legendFormat": "atlasbot"
         },
         {
           "refId": "E",
-          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"lesavka\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"lesavka\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"lesavka\"}[1h]))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"lesavka\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"lesavka\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"lesavka\"}[1h]))) > 0)",
           "legendFormat": "lesavka"
         },
         {
           "refId": "F",
-          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"pegasus\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"pegasus\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"pegasus\"}[1h]))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"pegasus\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"pegasus\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"pegasus\"}[1h]))) > 0)",
           "legendFormat": "pegasus"
         },
         {
           "refId": "G",
-          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"soteria\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"soteria\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"soteria\"}[1h]))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"soteria\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"soteria\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"soteria\"}[1h]))) > 0)",
           "legendFormat": "soteria"
         },
         {
           "refId": "H",
-          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"titan-iac\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"titan-iac\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"titan-iac\"}[1h]))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"titan-iac\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"titan-iac\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"titan-iac\"}[1h]))) > 0)",
           "legendFormat": "titan-iac"
         },
         {
           "refId": "I",
-          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"bstein-home\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"bstein-home\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"bstein-home\"}[1h]))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"bstein-home\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"bstein-home\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"bstein-home\"}[1h]))) > 0)",
           "legendFormat": "bstein-home"
         },
         {
           "refId": "J",
-          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\"}[1h]))) > 0)",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"arcanagon\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"arcanagon\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"arcanagon\"}[1h]))) > 0)",
           "legendFormat": "arcanagon"
+        },
+        {
+          "refId": "K",
+          "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\"}[1h]))) > 0)",
+          "legendFormat": "data-prepper"
         }
       ],
       "fieldConfig": {
@@ -1952,7 +1957,7 @@
     {
       "id": 47,
       "type": "bargauge",
-      "title": "Platform Suite Pass Rate (24h)",
+      "title": "PVC Backup Health / Age",
       "datasource": {
         "type": "prometheus",
         "uid": "atlas-vm"
@@ -1965,31 +1970,35 @@
       },
       "targets": [
         {
-          "expr": "sort_desc((label_replace((100 * (sum(increase(ariadne_task_runs_total{status=\"ok\"}[24h]))) / clamp_min((sum(increase(ariadne_task_runs_total[24h]))), 1)) and on() ((sum(increase(ariadne_task_runs_total[24h]))) > 0), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace((100 * ((sum(increase(metis_builds_total{status=\"ok\"}[24h])) + sum(increase(metis_flashes_total{status=\"ok\"}[24h])))) / clamp_min(((sum(increase(metis_builds_total[24h])) + sum(increase(metis_flashes_total[24h])))), 1)) and on() (((sum(increase(metis_builds_total[24h])) + sum(increase(metis_flashes_total[24h])))) > 0), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace((100 * (sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\",status=\"ok\"}[24h]))) / clamp_min((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[24h]))), 1)) and on() ((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[24h]))) > 0), \"suite\", \"ananke\", \"__name__\", \".*\")) or ((100 * (sum by (suite) (increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[24h]))) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\"}[24h]))), 1)) and on(suite) ((sum by (suite) (increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\"}[24h]))) > 0)))",
+          "expr": "sort_desc(max by (namespace, pvc) (pvc_backup_age_hours))",
           "refId": "A",
-          "legendFormat": "{{suite}}",
+          "legendFormat": "{{namespace}}/{{pvc}}",
           "instant": true
         }
       ],
       "fieldConfig": {
         "defaults": {
-          "unit": "percent",
+          "unit": "h",
           "min": 0,
-          "max": 100,
+          "max": null,
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "red",
+                "color": "green",
                 "value": null
               },
               {
                 "color": "yellow",
-                "value": 80
+                "value": 6
               },
               {
-                "color": "green",
+                "color": "orange",
-                "value": 95
+                "value": 12
+              },
+              {
+                "color": "red",
+                "value": 24
               }
             ]
           }
@@ -2020,12 +2029,12 @@
       ],
       "links": [
         {
-          "title": "Open atlas-jobs dashboard",
+          "title": "Open atlas-storage dashboard",
-          "url": "/d/atlas-jobs",
+          "url": "/d/atlas-storage",
           "targetBlank": true
         }
       ],
-      "description": "24-hour per-suite pass-rate snapshot. This complements the 7-day trend by showing each suite's current quality posture."
+      "description": "Oldest backup age in hours by PVC. This panel is reserved for the upcoming PVC backup health feed and will show no data until those metrics are published."
     },
     {
       "id": 30,
@@ -3166,8 +3175,8 @@
     },
     {
       "id": 22,
-      "type": "bargauge",
+      "type": "timeseries",
-      "title": "Nodes Closest to Full Root Disks",
+      "title": "Nodes Closest to Full Astraios Disks",
       "datasource": {
         "type": "prometheus",
         "uid": "atlas-vm"
@@ -3180,68 +3189,36 @@
       },
       "targets": [
         {
-          "expr": "sort_desc(topk(12, avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))))",
+          "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
           "refId": "A",
           "legendFormat": "{{node}}"
         }
       ],
       "fieldConfig": {
         "defaults": {
-          "unit": "percent",
+          "unit": "percent"
-          "min": 0,
-          "max": 100,
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              },
-              {
-                "color": "yellow",
-                "value": 50
-              },
-              {
-                "color": "orange",
-                "value": 75
-              },
-              {
-                "color": "red",
-                "value": 91.5
-              }
-            ]
-          }
         },
         "overrides": []
       },
       "options": {
-        "displayMode": "gradient",
+        "legend": {
-        "orientation": "horizontal",
+          "displayMode": "table",
-        "reduceOptions": {
+          "placement": "right",
           "calcs": [
-            "lastNotNull"
+            "last"
-          ],
+          ]
-          "fields": "",
+        },
-          "values": false
+        "tooltip": {
+          "mode": "multi"
         }
       },
+      "timeFrom": "1w",
       "links": [
         {
           "title": "Open atlas-storage dashboard",
           "url": "/d/atlas-storage",
           "targetBlank": true
         }
-      ],
-      "transformations": [
-        {
-          "id": "sortBy",
-          "options": {
-            "fields": [
-              "Value"
-            ],
-            "order": "desc"
-          }
-        }
       ]
     }
   ],

services/monitoring/grafana-dashboard-jobs.yaml

@@ -1147,7 +1147,7 @@ data:
           },
           "targets": [
             {
-              "expr": "100 * ((sum(increase(ariadne_task_runs_total{status=\"ok\"}[30d])) or on() vector(0)) + (sum(increase(metis_builds_total{status=\"ok\"}[30d])) or on() vector(0)) + (sum(increase(metis_flashes_total{status=\"ok\"}[30d])) or on() vector(0)) + (sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\",status=\"ok\"}[30d])) or on() vector(0))) / clamp_min(((sum(increase(ariadne_task_runs_total[30d])) or on() vector(0)) + (sum(increase(metis_builds_total[30d])) or on() vector(0)) + (sum(increase(metis_flashes_total[30d])) or on() vector(0)) + (sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[30d])) or on() vector(0))), 1)",
+              "expr": "100 * ((sum(increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\",status=~\"ok|passed|success\"}[30d])) or on() vector(0))) / clamp_min(((sum(increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\"}[30d])) or on() vector(0))), 1)",
               "refId": "A",
               "instant": true
             }
@@ -1210,7 +1210,7 @@ data:
           },
           "targets": [
             {
-              "expr": "label_replace(sum by (status) (increase(ariadne_task_runs_total[30d])), \"source\", \"ariadne\", \"__name__\", \".*\") or label_replace(sum by (status) (increase(metis_builds_total[30d])), \"source\", \"metis-build\", \"__name__\", \".*\") or label_replace(sum by (status) (increase(metis_flashes_total[30d])), \"source\", \"metis-flash\", \"__name__\", \".*\") or label_replace(sum by (status) (increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[30d])), \"source\", \"ananke-quality\", \"__name__\", \".*\")",
+              "expr": "sum by (suite, status) (increase(platform_quality_gate_runs_total{suite=~\"ariadne|metis|ananke|atlasbot|lesavka|pegasus|soteria|titan-iac|bstein-home|arcanagon|data-prepper\"}[30d]))",
               "refId": "A",
               "instant": true
             }
@@ -1262,17 +1262,17 @@ data:
           "targets": [
             {
               "refId": "A",
-              "expr": "(100 * (sum(increase(ariadne_task_runs_total{status=\"ok\"}[1h]))) / clamp_min((sum(increase(ariadne_task_runs_total[1h]))), 1)) and on() ((sum(increase(ariadne_task_runs_total[1h]))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\"}[1h]))) > 0)",
               "legendFormat": "ariadne"
             },
             {
               "refId": "B",
-              "expr": "(100 * ((sum(increase(metis_builds_total{status=\"ok\"}[1h])) + sum(increase(metis_flashes_total{status=\"ok\"}[1h])))) / clamp_min(((sum(increase(metis_builds_total[1h])) + sum(increase(metis_flashes_total[1h])))), 1)) and on() (((sum(increase(metis_builds_total[1h])) + sum(increase(metis_flashes_total[1h])))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"metis\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"metis\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"metis\"}[1h]))) > 0)",
               "legendFormat": "metis"
             },
             {
               "refId": "C",
-              "expr": "(100 * (sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\",status=\"ok\"}[1h]))) / clamp_min((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[1h]))), 1)) and on() ((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[1h]))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"ananke\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"ananke\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"ananke\"}[1h]))) > 0)",
               "legendFormat": "ananke"
             },
             {
@@ -1309,6 +1309,11 @@ data:
               "refId": "J",
               "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\"}[1h]))) > 0)",
               "legendFormat": "arcanagon"
+            },
+            {
+              "refId": "K",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\"}[1h]))) > 0)",
+              "legendFormat": "data-prepper"
             }
           ],
           "fieldConfig": {

services/monitoring/grafana-dashboard-nodes.yaml

@@ -593,6 +593,44 @@ data:
             }
           },
           "timeFrom": "30d"
+        },
+        {
+          "id": 9,
+          "type": "timeseries",
+          "title": "Astraios Usage",
+          "datasource": {
+            "type": "prometheus",
+            "uid": "atlas-vm"
+          },
+          "gridPos": {
+            "h": 9,
+            "w": 24,
+            "x": 0,
+            "y": 44
+          },
+          "targets": [
+            {
+              "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
+              "refId": "A",
+              "legendFormat": "{{node}}"
+            }
+          ],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "percent"
+            },
+            "overrides": []
+          },
+          "options": {
+            "legend": {
+              "displayMode": "table",
+              "placement": "right"
+            },
+            "tooltip": {
+              "mode": "multi"
+            }
+          },
+          "timeFrom": "30d"
         }
       ],
       "time": {

services/monitoring/grafana-dashboard-overview.yaml

@@ -1870,53 +1870,58 @@ data:
           "targets": [
             {
               "refId": "A",
-              "expr": "(100 * (sum(increase(ariadne_task_runs_total{status=\"ok\"}[1h]))) / clamp_min((sum(increase(ariadne_task_runs_total[1h]))), 1)) and on() ((sum(increase(ariadne_task_runs_total[1h]))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"ariadne\"}[1h]))) > 0)",
               "legendFormat": "ariadne"
             },
             {
               "refId": "B",
-              "expr": "(100 * ((sum(increase(metis_builds_total{status=\"ok\"}[1h])) + sum(increase(metis_flashes_total{status=\"ok\"}[1h])))) / clamp_min(((sum(increase(metis_builds_total[1h])) + sum(increase(metis_flashes_total[1h])))), 1)) and on() (((sum(increase(metis_builds_total[1h])) + sum(increase(metis_flashes_total[1h])))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"metis\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"metis\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"metis\"}[1h]))) > 0)",
               "legendFormat": "metis"
             },
             {
               "refId": "C",
-              "expr": "(100 * (sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\",status=\"ok\"}[1h]))) / clamp_min((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[1h]))), 1)) and on() ((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[1h]))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"ananke\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"ananke\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"ananke\"}[1h]))) > 0)",
               "legendFormat": "ananke"
             },
             {
               "refId": "D",
-              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"atlasbot\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"atlasbot\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"atlasbot\"}[1h]))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"atlasbot\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"atlasbot\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"atlasbot\"}[1h]))) > 0)",
               "legendFormat": "atlasbot"
             },
             {
               "refId": "E",
-              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"lesavka\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"lesavka\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"lesavka\"}[1h]))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"lesavka\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"lesavka\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"lesavka\"}[1h]))) > 0)",
               "legendFormat": "lesavka"
             },
             {
               "refId": "F",
-              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"pegasus\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"pegasus\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"pegasus\"}[1h]))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"pegasus\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"pegasus\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"pegasus\"}[1h]))) > 0)",
               "legendFormat": "pegasus"
             },
             {
               "refId": "G",
-              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"soteria\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"soteria\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"soteria\"}[1h]))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"soteria\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"soteria\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"soteria\"}[1h]))) > 0)",
               "legendFormat": "soteria"
             },
             {
               "refId": "H",
-              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"titan-iac\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"titan-iac\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"titan-iac\"}[1h]))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"titan-iac\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"titan-iac\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"titan-iac\"}[1h]))) > 0)",
               "legendFormat": "titan-iac"
             },
             {
               "refId": "I",
-              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"bstein-home\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"bstein-home\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"bstein-home\"}[1h]))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"bstein-home\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"bstein-home\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"bstein-home\"}[1h]))) > 0)",
               "legendFormat": "bstein-home"
             },
             {
               "refId": "J",
-              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",suite=\"arcanagon\"}[1h]))) > 0)",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"arcanagon\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"arcanagon\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"arcanagon\"}[1h]))) > 0)",
               "legendFormat": "arcanagon"
+            },
+            {
+              "refId": "K",
+              "expr": "(100 * (sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\",status=~\"ok|passed|success\"}[1h]))) / clamp_min((sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\"}[1h]))), 1)) and on() ((sum(increase(platform_quality_gate_runs_total{suite=\"data-prepper\"}[1h]))) > 0)",
+              "legendFormat": "data-prepper"
             }
           ],
           "fieldConfig": {
@@ -1961,7 +1966,7 @@ data:
         {
           "id": 47,
           "type": "bargauge",
-          "title": "Platform Suite Pass Rate (24h)",
+          "title": "PVC Backup Health / Age",
           "datasource": {
             "type": "prometheus",
             "uid": "atlas-vm"
@@ -1974,31 +1979,35 @@ data:
           },
           "targets": [
             {
-              "expr": "sort_desc((label_replace((100 * (sum(increase(ariadne_task_runs_total{status=\"ok\"}[24h]))) / clamp_min((sum(increase(ariadne_task_runs_total[24h]))), 1)) and on() ((sum(increase(ariadne_task_runs_total[24h]))) > 0), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace((100 * ((sum(increase(metis_builds_total{status=\"ok\"}[24h])) + sum(increase(metis_flashes_total{status=\"ok\"}[24h])))) / clamp_min(((sum(increase(metis_builds_total[24h])) + sum(increase(metis_flashes_total[24h])))), 1)) and on() (((sum(increase(metis_builds_total[24h])) + sum(increase(metis_flashes_total[24h])))) > 0), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace((100 * (sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\",status=\"ok\"}[24h]))) / clamp_min((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[24h]))), 1)) and on() ((sum(increase(ananke_quality_gate_runs_total{suite=\"ananke\"}[24h]))) > 0), \"suite\", \"ananke\", \"__name__\", \".*\")) or ((100 * (sum by (suite) (increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}[24h]))) / clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\"}[24h]))), 1)) and on(suite) ((sum by (suite) (increase(platform_quality_gate_runs_total{exported_job=\"platform-quality-ci\"}[24h]))) > 0)))",
+              "expr": "sort_desc(max by (namespace, pvc) (pvc_backup_age_hours))",
               "refId": "A",
-              "legendFormat": "{{suite}}",
+              "legendFormat": "{{namespace}}/{{pvc}}",
               "instant": true
             }
           ],
           "fieldConfig": {
             "defaults": {
-              "unit": "percent",
+              "unit": "h",
               "min": 0,
-              "max": 100,
+              "max": null,
               "thresholds": {
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "red",
+                    "color": "green",
                     "value": null
                   },
                   {
                     "color": "yellow",
-                    "value": 80
+                    "value": 6
                   },
                   {
-                    "color": "green",
+                    "color": "orange",
-                    "value": 95
+                    "value": 12
+                  },
+                  {
+                    "color": "red",
+                    "value": 24
                   }
                 ]
               }
@@ -2029,12 +2038,12 @@ data:
           ],
           "links": [
             {
-              "title": "Open atlas-jobs dashboard",
+              "title": "Open atlas-storage dashboard",
-              "url": "/d/atlas-jobs",
+              "url": "/d/atlas-storage",
               "targetBlank": true
             }
           ],
-          "description": "24-hour per-suite pass-rate snapshot. This complements the 7-day trend by showing each suite's current quality posture."
+          "description": "Oldest backup age in hours by PVC. This panel is reserved for the upcoming PVC backup health feed and will show no data until those metrics are published."
         },
         {
           "id": 30,
@@ -3175,8 +3184,8 @@ data:
         },
         {
           "id": 22,
-          "type": "bargauge",
+          "type": "timeseries",
-          "title": "Nodes Closest to Full Root Disks",
+          "title": "Nodes Closest to Full Astraios Disks",
           "datasource": {
             "type": "prometheus",
             "uid": "atlas-vm"
@@ -3189,68 +3198,36 @@ data:
           },
           "targets": [
             {
-              "expr": "sort_desc(topk(12, avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))))",
+              "expr": "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))",
               "refId": "A",
               "legendFormat": "{{node}}"
             }
           ],
           "fieldConfig": {
             "defaults": {
-              "unit": "percent",
+              "unit": "percent"
-              "min": 0,
-              "max": 100,
-              "thresholds": {
-                "mode": "absolute",
-                "steps": [
-                  {
-                    "color": "green",
-                    "value": null
-                  },
-                  {
-                    "color": "yellow",
-                    "value": 50
-                  },
-                  {
-                    "color": "orange",
-                    "value": 75
-                  },
-                  {
-                    "color": "red",
-                    "value": 91.5
-                  }
-                ]
-              }
             },
             "overrides": []
           },
           "options": {
-            "displayMode": "gradient",
+            "legend": {
-            "orientation": "horizontal",
+              "displayMode": "table",
-            "reduceOptions": {
+              "placement": "right",
               "calcs": [
-                "lastNotNull"
+                "last"
-              ],
+              ]
-              "fields": "",
+            },
-              "values": false
+            "tooltip": {
+              "mode": "multi"
             }
           },
+          "timeFrom": "1w",
           "links": [
             {
               "title": "Open atlas-storage dashboard",
               "url": "/d/atlas-storage",
               "targetBlank": true
             }
-          ],
-          "transformations": [
-            {
-              "id": "sortBy",
-              "options": {
-                "fields": [
-                  "Value"
-                ],
-                "order": "desc"
-              }
-            }
           ]
         }
       ],

services/nextcloud/collabora.yaml

@@ -18,6 +18,7 @@ spec:
     spec:
       nodeSelector:
         hardware: rpi5
+        node-role.kubernetes.io/worker: "true"
       containers:
         - name: collabora
           image: collabora/code@sha256:3c58d0e9bae75e4647467d0c7d91cb66f261d3e814709aed590b5c334a04db26

services/sui-metrics/overlays/atlas/patch-node-selector.yaml

@@ -9,3 +9,4 @@ spec:
     spec:
       nodeSelector:
         hardware: rpi5
+        node-role.kubernetes.io/worker: "true"

testing/__init__.py

@@ -0,0 +1 @@
+"""Top-level testing contract and quality-gate tooling for titan-iac."""

testing/quality_contract.json

@@ -0,0 +1,161 @@
+{
+  "scope_note": "Quality-gate LOC/naming/coverage checks apply to managed automation and testing modules only, not broad Flux/Kubernetes manifest trees.",
+  "required_docs": [
+    {
+      "path": "README.md",
+      "description": "Top-level repository handbook."
+    },
+    {
+      "path": "Jenkinsfile",
+      "description": "Top-level Jenkins mirror for multibranch discovery."
+    },
+    {
+      "path": "ci/Jenkinsfile.titan-iac",
+      "description": "Canonical titan-iac Jenkins pipeline definition."
+    }
+  ],
+  "managed_modules": [
+    "ci/scripts/publish_test_metrics.py",
+    "services/mailu/scripts/mailu_sync.py",
+    "testing/__init__.py",
+    "testing/quality_contract.py",
+    "testing/quality_docs.py",
+    "testing/quality_hygiene.py",
+    "testing/quality_coverage.py",
+    "testing/quality_gate.py"
+  ],
+  "lint_paths": [
+    "ci/scripts/publish_test_metrics.py",
+    "ci/tests/glue",
+    "scripts/tests",
+    "services/comms/scripts/tests",
+    "services/mailu/scripts/mailu_sync.py",
+    "testing"
+  ],
+  "pytest_suites": {
+    "unit": {
+      "description": "Fast unit and contract tests for repo automation.",
+      "paths": [
+        "scripts/tests",
+        "services/comms/scripts/tests",
+        "testing/tests"
+      ],
+      "junit": "build/junit-unit.xml",
+      "coverage_sources": [
+        "ci/scripts",
+        "services/mailu/scripts",
+        "testing"
+      ],
+      "coverage_xml": "build/coverage-unit.xml"
+    },
+    "glue": {
+      "description": "Cluster-live glue checks that validate CronJobs and exported metrics.",
+      "paths": [
+        "ci/tests/glue"
+      ],
+      "junit": "build/junit-glue.xml"
+    }
+  },
+  "profiles": {
+    "local": [
+      "docs",
+      "smell",
+      "hygiene",
+      "unit",
+      "coverage"
+    ],
+    "jenkins": [
+      "docs",
+      "smell",
+      "hygiene",
+      "unit",
+      "coverage",
+      "glue"
+    ]
+  },
+  "manual_scripts": [
+    {
+      "path": "scripts/test_atlas_user_cleanup.py",
+      "description": "Manual cleanup validation for Atlas user lifecycle automation."
+    },
+    {
+      "path": "scripts/test_user_cleanup.py",
+      "description": "Manual cleanup validation for shared user lifecycle automation."
+    },
+    {
+      "path": "scripts/test_vaultwarden_user_cleanup.py",
+      "description": "Manual cleanup validation for Vaultwarden user lifecycle automation."
+    },
+    {
+      "path": "services/bstein-dev-home/scripts/test_portal_onboarding_flow.py",
+      "description": "Portal onboarding end-to-end flow validation with mail delivery checks."
+    },
+    {
+      "path": "services/keycloak/scripts/tests/test_keycloak_execute_actions_email.py",
+      "description": "Standalone Keycloak SMTP execute-actions-email validation script."
+    },
+    {
+      "path": "services/keycloak/scripts/tests/test_portal_token_exchange.py",
+      "description": "Standalone Keycloak token-exchange validation script."
+    }
+  ],
+  "hygiene": {
+    "max_lines": 500,
+    "line_limit_globs": [
+      "testing/**/*.py",
+      "ci/scripts/*.py",
+      "ci/tests/**/*.py",
+      "scripts/tests/**/*.py",
+      "services/*/scripts/tests/**/*.py",
+      "services/mailu/scripts/mailu_sync.py"
+    ],
+    "naming_rules": [
+      {
+        "glob": "testing/*.py",
+        "pattern": "^(?:__init__|quality_[a-z0-9_]+)\\.py$",
+        "description": "Top-level testing helpers use quality_* module names."
+      },
+      {
+        "glob": "testing/tests/*.py",
+        "pattern": "^test_[a-z0-9_]+\\.py$",
+        "description": "Top-level pytest files use test_*.py names."
+      },
+      {
+        "glob": "ci/tests/**/*.py",
+        "pattern": "^test_[a-z0-9_]+\\.py$",
+        "description": "CI pytest files use test_*.py names."
+      },
+      {
+        "glob": "scripts/tests/**/*.py",
+        "pattern": "^test_[a-z0-9_]+\\.py$",
+        "description": "Script pytest files use test_*.py names."
+      },
+      {
+        "glob": "scripts/test_*.py",
+        "pattern": "^test_[a-z0-9_]+\\.py$",
+        "description": "Standalone script tests use test_*.py names."
+      },
+      {
+        "glob": "services/*/scripts/tests/**/*.py",
+        "pattern": "^test_[a-z0-9_]+\\.py$",
+        "description": "Service pytest files use test_*.py names."
+      },
+      {
+        "glob": "services/*/scripts/test_*.py",
+        "pattern": "^test_[a-z0-9_]+\\.py$",
+        "description": "Standalone service test scripts use test_*.py names."
+      }
+    ]
+  },
+  "coverage": {
+    "minimum_percent": 95.0,
+    "tracked_files": [
+      "ci/scripts/publish_test_metrics.py",
+      "testing/quality_contract.py",
+      "testing/quality_docs.py",
+      "testing/quality_hygiene.py",
+      "testing/quality_coverage.py",
+      "testing/quality_gate.py"
+    ]
+  }
+}

testing/quality_contract.py

@@ -0,0 +1,17 @@
+"""Helpers for loading the repository testing contract."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+
+CONTRACT_PATH = Path(__file__).with_name("quality_contract.json")
+
+
+def load_contract(contract_path: Path | None = None) -> dict[str, Any]:
+    """Return the parsed testing contract."""
+    path = contract_path or CONTRACT_PATH
+    with path.open("r", encoding="utf-8") as handle:
+        return json.load(handle)

testing/quality_coverage.py

@@ -0,0 +1,58 @@
+"""Per-file coverage threshold validation for quality-managed modules."""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from typing import Any
+
+
+def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]:
+    tree = ET.parse(xml_path)
+    xml_root = tree.getroot()
+    source_roots = [
+        Path(node.text)
+        for node in xml_root.findall("./sources/source")
+        if node.text
+    ]
+    percentages: dict[str, float] = {}
+    for class_node in xml_root.findall(".//class"):
+        filename = class_node.attrib.get("filename")
+        line_rate = class_node.attrib.get("line-rate")
+        if not filename or line_rate is None:
+            continue
+        normalized = filename.replace("\\", "/")
+        if normalized.startswith("/"):
+            key = Path(normalized).relative_to(root).as_posix()
+        else:
+            key = normalized
+            for source_root in source_roots:
+                candidate = source_root / filename
+                if candidate.exists():
+                    key = candidate.relative_to(root).as_posix()
+                    break
+        percentages[key] = float(line_rate) * 100.0
+    return percentages
+
+
+def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str]:
+    """Return human-readable issues for tracked files below the coverage floor."""
+    if not xml_path.exists():
+        return [f"coverage xml missing: {xml_path.relative_to(root)}"]
+
+    percentages = _load_percentages(xml_path, root)
+    minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0))
+    issues: list[str] = []
+
+    for relative_path in contract.get("coverage", {}).get("tracked_files", []):
+        normalized = relative_path.replace("\\", "/")
+        percent = percentages.get(normalized)
+        if percent is None:
+            issues.append(f"coverage missing for tracked file: {relative_path}")
+            continue
+        if percent + 1e-9 < minimum:
+            issues.append(
+                f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)"
+            )
+
+    return issues

testing/quality_docs.py

@@ -0,0 +1,59 @@
+"""Documentation-oriented validation for the testing contract."""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+from typing import Any
+
+
+def _module_has_docstring(path: Path) -> bool:
+    source = path.read_text(encoding="utf-8")
+    return ast.get_docstring(ast.parse(source)) is not None
+
+
+def _iter_contract_paths(contract: dict[str, Any]) -> list[str]:
+    paths: list[str] = []
+    for item in contract.get("required_docs", []):
+        paths.append(item["path"])
+    paths.extend(contract.get("managed_modules", []))
+    paths.extend(contract.get("lint_paths", []))
+    for suite in contract.get("pytest_suites", {}).values():
+        paths.extend(suite.get("paths", []))
+    for item in contract.get("manual_scripts", []):
+        paths.append(item["path"])
+    return paths
+
+
+def run_check(contract: dict[str, Any], root: Path) -> list[str]:
+    """Return human-readable issues for contract/documentation violations."""
+    issues: list[str] = []
+
+    for item in contract.get("required_docs", []):
+        path = root / item["path"]
+        if not path.exists():
+            issues.append(f"required doc missing: {item['path']}")
+            continue
+        if path.is_file() and not path.read_text(encoding="utf-8").strip():
+            issues.append(f"required doc empty: {item['path']}")
+        if not item.get("description", "").strip():
+            issues.append(f"required doc missing description: {item['path']}")
+
+    for relative_path in sorted(set(_iter_contract_paths(contract))):
+        if not (root / relative_path).exists():
+            issues.append(f"contract path missing: {relative_path}")
+
+    for suite_name, suite in contract.get("pytest_suites", {}).items():
+        if not suite.get("description", "").strip():
+            issues.append(f"pytest suite missing description: {suite_name}")
+
+    for item in contract.get("manual_scripts", []):
+        if not item.get("description", "").strip():
+            issues.append(f"manual script missing description: {item['path']}")
+
+    for relative_path in contract.get("managed_modules", []):
+        path = root / relative_path
+        if path.exists() and path.suffix == ".py" and not _module_has_docstring(path):
+            issues.append(f"module docstring missing: {relative_path}")
+
+    return issues

testing/quality_gate.py

@@ -0,0 +1,175 @@
+"""Source-of-truth quality-gate runner for titan-iac."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+from testing.quality_contract import load_contract
+from testing.quality_coverage import run_check as run_coverage_check
+from testing.quality_docs import run_check as run_docs_check
+from testing.quality_hygiene import run_check as run_hygiene_check
+
+
+RUFF_SELECT = ["F", "B", "SIM", "C4", "UP"]
+RUFF_IGNORE = ["B017", "UP015", "UP035"]
+
+
+def _status_from_issues(issues: list[str]) -> str:
+    return "ok" if not issues else "failed"
+
+
+def _result(name: str, description: str, status: str, **extra: Any) -> dict[str, Any]:
+    return {"name": name, "description": description, "status": status, **extra}
+
+
+def _run_ruff(contract: dict[str, Any], root: Path) -> dict[str, Any]:
+    command = [
+        sys.executable,
+        "-m",
+        "ruff",
+        "check",
+        "--select",
+        ",".join(RUFF_SELECT),
+        "--ignore",
+        ",".join(RUFF_IGNORE),
+        *contract.get("lint_paths", []),
+    ]
+    started_at = time.monotonic()
+    completed = subprocess.run(command, cwd=root, check=False)
+    return _result(
+        "smell",
+        "Code-smell lint for managed Python automation.",
+        "ok" if completed.returncode == 0 else "failed",
+        returncode=completed.returncode,
+        command=command,
+        duration_seconds=round(time.monotonic() - started_at, 3),
+    )
+
+
+def _run_pytest_suite(root: Path, suite_name: str, suite: dict[str, Any]) -> dict[str, Any]:
+    junit_path = root / suite["junit"]
+    junit_path.parent.mkdir(parents=True, exist_ok=True)
+    command = [
+        sys.executable,
+        "-m",
+        "pytest",
+        "-q",
+        *suite.get("paths", []),
+        f"--junitxml={junit_path}",
+    ]
+    coverage_xml = suite.get("coverage_xml")
+    if coverage_xml:
+        for source in suite.get("coverage_sources", []):
+            command.append(f"--cov={source}")
+        command.extend(
+            [
+                "--cov-branch",
+                f"--cov-report=xml:{root / coverage_xml}",
+            ]
+        )
+    started_at = time.monotonic()
+    completed = subprocess.run(command, cwd=root, check=False)
+    return _result(
+        suite_name,
+        suite["description"],
+        "ok" if completed.returncode == 0 else "failed",
+        returncode=completed.returncode,
+        command=command,
+        junit=str(junit_path.relative_to(root)),
+        coverage_xml=coverage_xml,
+        duration_seconds=round(time.monotonic() - started_at, 3),
+    )
+
+
+def run_profile(
+    contract: dict[str, Any],
+    root: Path,
+    profile_name: str,
+    build_dir: Path,
+) -> dict[str, Any]:
+    """Execute the configured profile and return a JSON-serializable summary."""
+    build_dir.mkdir(parents=True, exist_ok=True)
+    results: list[dict[str, Any]] = []
+    profiles = contract.get("profiles", {})
+    if profile_name not in profiles:
+        raise SystemExit(f"unknown profile: {profile_name}")
+
+    for check_name in profiles[profile_name]:
+        if check_name == "docs":
+            issues = run_docs_check(contract, root)
+            results.append(
+                _result(
+                    "docs",
+                    "Required docs, contract descriptions, and module docstrings.",
+                    _status_from_issues(issues),
+                    issues=issues,
+                )
+            )
+            continue
+        if check_name == "smell":
+            results.append(_run_ruff(contract, root))
+            continue
+        if check_name == "hygiene":
+            issues = run_hygiene_check(contract, root)
+            results.append(
+                _result(
+                    "hygiene",
+                    "500 LOC hygiene and naming rules for managed test automation.",
+                    _status_from_issues(issues),
+                    issues=issues,
+                )
+            )
+            continue
+        if check_name == "coverage":
+            unit_suite = contract.get("pytest_suites", {}).get("unit", {})
+            coverage_xml = root / unit_suite.get("coverage_xml", "build/coverage-unit.xml")
+            issues = run_coverage_check(contract, root, coverage_xml)
+            results.append(
+                _result(
+                    "coverage",
+                    "Per-file 95% coverage floor for tracked quality-managed modules.",
+                    _status_from_issues(issues),
+                    issues=issues,
+                    coverage_xml=str(coverage_xml.relative_to(root)),
+                )
+            )
+            continue
+        suite = contract.get("pytest_suites", {}).get(check_name)
+        if suite is None:
+            raise SystemExit(f"profile {profile_name} references unknown check: {check_name}")
+        results.append(_run_pytest_suite(root, check_name, suite))
+
+    status = "ok" if all(item["status"] == "ok" for item in results) else "failed"
+    return {
+        "profile": profile_name,
+        "status": status,
+        "results": results,
+        "manual_scripts": contract.get("manual_scripts", []),
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    """CLI entrypoint for the quality gate."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--profile", default="local")
+    parser.add_argument("--build-dir", default="build")
+    args = parser.parse_args(argv)
+
+    root = Path.cwd()
+    build_dir = root / args.build_dir
+    build_dir.mkdir(parents=True, exist_ok=True)
+    contract = load_contract()
+    summary = run_profile(contract, root, args.profile, build_dir)
+    summary_path = build_dir / "quality-gate-summary.json"
+    summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return 0 if summary["status"] == "ok" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

testing/quality_hygiene.py

@@ -0,0 +1,37 @@
+"""File-size and naming validation for the managed testing surface."""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+
+def _expand_globs(root: Path, patterns: Iterable[str]) -> list[Path]:
+    matched: set[Path] = set()
+    for pattern in patterns:
+        matched.update(path for path in root.glob(pattern) if path.is_file())
+    return sorted(matched)
+
+
+def run_check(contract: dict[str, Any], root: Path) -> list[str]:
+    """Return human-readable issues for naming and file-size rules."""
+    config = contract.get("hygiene", {})
+    max_lines = int(config.get("max_lines", 500))
+    issues: list[str] = []
+
+    for path in _expand_globs(root, config.get("line_limit_globs", [])):
+        line_count = sum(1 for _ in path.open("r", encoding="utf-8"))
+        if line_count > max_lines:
+            issues.append(f"file exceeds {max_lines} LOC: {path.relative_to(root)} ({line_count})")
+
+    for rule in config.get("naming_rules", []):
+        pattern = re.compile(rule["pattern"])
+        for path in _expand_globs(root, [rule["glob"]]):
+            if not pattern.match(path.name):
+                issues.append(
+                    f"naming rule failed ({rule['description']}): {path.relative_to(root)}"
+                )
+
+    return issues

testing/tests/test_publish_test_metrics.py

@@ -0,0 +1,264 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from ci.scripts import publish_test_metrics
+
+
+def test_parse_junit_supports_testsuite_and_missing_file(tmp_path: Path):
+    junit_path = tmp_path / "suite.xml"
+    junit_path.write_text(
+        '<testsuite tests="3" failures="1" errors="0" skipped="1" />',
+        encoding="utf-8",
+    )
+
+    assert publish_test_metrics._parse_junit(str(junit_path)) == {
+        "tests": 3,
+        "failures": 1,
+        "errors": 0,
+        "skipped": 1,
+    }
+    assert publish_test_metrics._parse_junit(str(tmp_path / "missing.xml")) == {
+        "tests": 0,
+        "failures": 0,
+        "errors": 0,
+        "skipped": 0,
+    }
+
+
+def test_collect_junit_totals_sums_multiple_files(tmp_path: Path):
+    first = tmp_path / "junit-a.xml"
+    second = tmp_path / "junit-b.xml"
+    first.write_text('<testsuite tests="2" failures="1" errors="0" skipped="0" />', encoding="utf-8")
+    second.write_text('<testsuite tests="3" failures="0" errors="1" skipped="1" />', encoding="utf-8")
+
+    totals = publish_test_metrics._collect_junit_totals(str(tmp_path / "junit-*.xml"))
+
+    assert totals == {"tests": 5, "failures": 1, "errors": 1, "skipped": 1}
+
+
+def test_parse_junit_handles_testsuites_and_invalid_counts(tmp_path: Path):
+    junit_path = tmp_path / "suite.xml"
+    junit_path.write_text(
+        (
+            "<testsuites>"
+            '<testsuite tests="2" failures="1" errors="0" skipped="0" />'
+            '<testsuite tests="bad" failures="0" errors="0" skipped="0" />'
+            "</testsuites>"
+        ),
+        encoding="utf-8",
+    )
+
+    assert publish_test_metrics._parse_junit(str(junit_path)) == {
+        "tests": 2,
+        "failures": 1,
+        "errors": 0,
+        "skipped": 0,
+    }
+
+
+def test_read_exit_code_and_summary_fallbacks(tmp_path: Path):
+    rc_path = tmp_path / "rc.txt"
+    rc_path.write_text("0\n", encoding="utf-8")
+    summary_path = tmp_path / "summary.json"
+    summary_path.write_text("{bad json", encoding="utf-8")
+
+    assert publish_test_metrics._read_exit_code(str(rc_path)) == 0
+    assert publish_test_metrics._read_exit_code(str(tmp_path / "missing.rc")) == 1
+    assert publish_test_metrics._load_summary(str(summary_path)) == {}
+    assert publish_test_metrics._load_summary(str(tmp_path / "missing.json")) == {}
+
+
+def test_read_text_post_text_and_fetch_existing_counter(monkeypatch):
+    class _FakeResponse:
+        def __init__(self, payload: str, status: int = 200):
+            self.payload = payload
+            self.status = status
+
+        def read(self):
+            return self.payload.encode("utf-8")
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc, tb):
+            return None
+
+    responses = iter(
+        [
+            _FakeResponse("alpha"),
+            _FakeResponse("", status=202),
+            _FakeResponse(
+                "\n".join(
+                    [
+                        'platform_quality_gate_runs_total{job="platform-quality-ci",suite="titan-iac",status="ok"} 7',
+                        'platform_quality_gate_runs_total{job="other",suite="titan-iac",status="ok"} 1',
+                    ]
+                )
+            ),
+        ]
+    )
+    monkeypatch.setattr(
+        publish_test_metrics.urllib.request,
+        "urlopen",
+        lambda *args, **kwargs: next(responses),
+    )
+
+    assert publish_test_metrics._read_text("http://example.invalid") == "alpha"
+    publish_test_metrics._post_text("http://example.invalid", "payload")
+    assert (
+        publish_test_metrics._fetch_existing_counter(
+            "http://push.invalid",
+            "platform_quality_gate_runs_total",
+            {"job": "platform-quality-ci", "suite": "titan-iac", "status": "ok"},
+        )
+        == 7.0
+    )
+
+
+def test_post_text_raises_and_counter_handles_bad_metric_lines(monkeypatch):
+    class _FakeResponse:
+        def __init__(self, payload: str, status: int = 200):
+            self.payload = payload
+            self.status = status
+
+        def read(self):
+            return self.payload.encode("utf-8")
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc, tb):
+            return None
+
+    monkeypatch.setattr(
+        publish_test_metrics.urllib.request,
+        "urlopen",
+        lambda *args, **kwargs: _FakeResponse("", status=500),
+    )
+    try:
+        publish_test_metrics._post_text("http://example.invalid", "payload")
+    except RuntimeError as exc:
+        assert "push failed" in str(exc)
+    else:
+        raise AssertionError("expected RuntimeError for failing push")
+
+    monkeypatch.setattr(
+        publish_test_metrics,
+        "_read_text",
+        lambda url: "\n".join(
+            [
+                'platform_quality_gate_runs_total{job="platform-quality-ci",suite="titan-iac",status="ok"}',
+                'platform_quality_gate_runs_total{job="platform-quality-ci",suite="titan-iac",status="ok"} nope',
+            ]
+        ),
+    )
+    assert (
+        publish_test_metrics._fetch_existing_counter(
+            "http://push.invalid",
+            "platform_quality_gate_runs_total",
+            {"job": "platform-quality-ci", "suite": "titan-iac", "status": "ok"},
+        )
+        == 0.0
+    )
+
+
+def test_build_payload_includes_summary_metrics():
+    payload = publish_test_metrics._build_payload(
+        suite="titan-iac",
+        status="ok",
+        tests={"tests": 4, "failures": 1, "errors": 0, "skipped": 1},
+        ok_count=7,
+        failed_count=2,
+        branch="main",
+        build_number="42",
+        summary={
+            "results": [
+                {"name": "docs", "status": "ok"},
+                {"name": "unit", "status": "failed"},
+            ]
+        },
+    )
+
+    assert 'platform_quality_gate_runs_total{suite="titan-iac",status="ok"} 7' in payload
+    assert 'titan_iac_quality_gate_checks_total{suite="titan-iac",check="docs",result="ok"} 1' in payload
+    assert 'titan_iac_quality_gate_checks_total{suite="titan-iac",check="unit",result="failed"} 1' in payload
+
+
+def test_build_payload_skips_incomplete_results():
+    payload = publish_test_metrics._build_payload(
+        suite="titan-iac",
+        status="failed",
+        tests={"tests": 0, "failures": 0, "errors": 0, "skipped": 0},
+        ok_count=1,
+        failed_count=2,
+        branch="",
+        build_number="",
+        summary={"results": [{"name": "docs"}, {"status": "ok"}]},
+    )
+
+    assert "titan_iac_quality_gate_checks_total" in payload
+    assert 'check="docs"' not in payload
+
+
+def test_main_uses_quality_gate_summary_and_junit_glob(tmp_path: Path, monkeypatch):
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+    (build_dir / "junit-unit.xml").write_text(
+        '<testsuite tests="2" failures="0" errors="0" skipped="0" />',
+        encoding="utf-8",
+    )
+    (build_dir / "junit-glue.xml").write_text(
+        '<testsuite tests="3" failures="1" errors="0" skipped="0" />',
+        encoding="utf-8",
+    )
+    (build_dir / "quality-gate.rc").write_text("1\n", encoding="utf-8")
+    (build_dir / "quality-gate-summary.json").write_text(
+        json.dumps({"results": [{"name": "docs", "status": "ok"}, {"name": "glue", "status": "failed"}]}),
+        encoding="utf-8",
+    )
+
+    posted = {}
+
+    monkeypatch.setenv("SUITE_NAME", "titan-iac")
+    monkeypatch.setenv("PUSHGATEWAY_URL", "http://pushgateway.invalid")
+    monkeypatch.setenv("QUALITY_GATE_JOB_NAME", "platform-quality-ci")
+    monkeypatch.setenv("JUNIT_GLOB", str(build_dir / "junit-*.xml"))
+    monkeypatch.setenv("QUALITY_GATE_EXIT_CODE_PATH", str(build_dir / "quality-gate.rc"))
+    monkeypatch.setenv("QUALITY_GATE_SUMMARY_PATH", str(build_dir / "quality-gate-summary.json"))
+    monkeypatch.setenv("BRANCH_NAME", "main")
+    monkeypatch.setenv("BUILD_NUMBER", "88")
+
+    monkeypatch.setattr(publish_test_metrics, "_fetch_existing_counter", lambda *args, **kwargs: 5)
+    monkeypatch.setattr(publish_test_metrics, "_post_text", lambda url, payload: posted.update({"url": url, "payload": payload}))
+
+    rc = publish_test_metrics.main()
+
+    assert rc == 0
+    assert posted["url"].endswith("/metrics/job/platform-quality-ci/suite/titan-iac")
+    assert 'titan_iac_quality_gate_tests_total{suite="titan-iac",result="failed"} 1' in posted["payload"]
+    assert 'titan_iac_quality_gate_checks_total{suite="titan-iac",check="glue",result="failed"} 1' in posted["payload"]
+
+
+def test_main_marks_successful_run(tmp_path: Path, monkeypatch, capsys):
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+    (build_dir / "junit.xml").write_text(
+        '<testsuite tests="1" failures="0" errors="0" skipped="0" />',
+        encoding="utf-8",
+    )
+    (build_dir / "quality-gate.rc").write_text("0\n", encoding="utf-8")
+
+    monkeypatch.setenv("JUNIT_GLOB", str(build_dir / "*.xml"))
+    monkeypatch.setenv("QUALITY_GATE_EXIT_CODE_PATH", str(build_dir / "quality-gate.rc"))
+    monkeypatch.setenv("QUALITY_GATE_SUMMARY_PATH", str(build_dir / "missing-summary.json"))
+    monkeypatch.setattr(publish_test_metrics, "_fetch_existing_counter", lambda *args, **kwargs: 0)
+    monkeypatch.setattr(publish_test_metrics, "_post_text", lambda *args, **kwargs: None)
+
+    rc = publish_test_metrics.main()
+
+    summary = json.loads(capsys.readouterr().out)
+    assert rc == 0
+    assert summary["status"] == "ok"
+    assert summary["checks_recorded"] == 0

testing/tests/test_quality_contract.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+from pathlib import Path
+import textwrap
+
+from testing.quality_contract import load_contract
+from testing.quality_coverage import run_check as run_coverage_check
+from testing.quality_docs import run_check as run_docs_check
+from testing.quality_hygiene import run_check as run_hygiene_check
+
+
+def test_bundled_contract_exposes_local_and_jenkins_profiles():
+    contract = load_contract()
+    assert "local" in contract["profiles"]
+    assert "jenkins" in contract["profiles"]
+    assert contract["pytest_suites"]["unit"]["paths"]
+
+
+def test_bundled_contract_keeps_monorepo_manifest_trees_out_of_hygiene_scope():
+    contract = load_contract()
+    required_doc_paths = {item["path"] for item in contract.get("required_docs", [])}
+    assert "AGENTS.md" not in required_doc_paths
+
+    globs = contract.get("hygiene", {}).get("line_limit_globs", [])
+    assert globs
+    for entry in globs:
+        assert entry.startswith(("testing/", "ci/", "scripts/tests/", "services/"))
+        assert "/scripts/" in entry or not entry.startswith("services/")
+
+
+def test_docs_check_reports_missing_docstring_and_missing_path(tmp_path: Path):
+    module_path = tmp_path / "managed.py"
+    module_path.write_text("value = 1\n", encoding="utf-8")
+    (tmp_path / "README.md").write_text("repo docs\n", encoding="utf-8")
+
+    contract = {
+        "required_docs": [{"path": "README.md", "description": "Docs"}],
+        "managed_modules": ["managed.py"],
+        "lint_paths": ["missing-dir"],
+        "pytest_suites": {"unit": {"description": "Unit", "paths": ["missing-tests"]}},
+        "manual_scripts": [{"path": "missing-script.py", "description": "Manual"}],
+    }
+
+    issues = run_docs_check(contract, tmp_path)
+
+    assert "module docstring missing: managed.py" in issues
+    assert "contract path missing: missing-dir" in issues
+    assert "contract path missing: missing-tests" in issues
+    assert "contract path missing: missing-script.py" in issues
+
+
+def test_docs_check_reports_missing_required_doc_metadata(tmp_path: Path):
+    (tmp_path / "README.md").write_text("", encoding="utf-8")
+
+    contract = {
+        "required_docs": [{"path": "README.md", "description": ""}, {"path": "missing.md", "description": "Missing"}],
+        "managed_modules": [],
+        "lint_paths": [],
+        "pytest_suites": {"unit": {"description": "", "paths": []}},
+        "manual_scripts": [{"path": "manual.py", "description": ""}],
+    }
+
+    issues = run_docs_check(contract, tmp_path)
+
+    assert "required doc empty: README.md" in issues
+    assert "required doc missing description: README.md" in issues
+    assert "required doc missing: missing.md" in issues
+    assert "pytest suite missing description: unit" in issues
+    assert "manual script missing description: manual.py" in issues
+
+
+def test_hygiene_check_enforces_line_limit_and_name_rules(tmp_path: Path):
+    tests_dir = tmp_path / "tests"
+    tests_dir.mkdir()
+    bad_name = tests_dir / "bad-name.py"
+    bad_name.write_text("x = 1\n", encoding="utf-8")
+    long_file = tests_dir / "test_too_long.py"
+    long_file.write_text("line\n" * 4, encoding="utf-8")
+
+    contract = {
+        "hygiene": {
+            "max_lines": 3,
+            "line_limit_globs": ["tests/*.py"],
+            "naming_rules": [
+                {
+                    "glob": "tests/*.py",
+                    "pattern": r"^test_[a-z0-9_]+\.py$",
+                    "description": "pytest files use test_*.py names.",
+                }
+            ],
+        }
+    }
+
+    issues = run_hygiene_check(contract, tmp_path)
+
+    assert any("file exceeds 3 LOC" in issue for issue in issues)
+    assert any("naming rule failed" in issue and "bad-name.py" in issue for issue in issues)
+
+
+def test_coverage_check_enforces_per_file_floor(tmp_path: Path):
+    build_dir = tmp_path / "build"
+    build_dir.mkdir()
+    coverage_xml = build_dir / "coverage.xml"
+    coverage_xml.write_text(
+        textwrap.dedent(
+            """\
+            <coverage>
+              <packages>
+                <package>
+                  <classes>
+                    <class filename="ok.py" line-rate="1.0" />
+                    <class filename="low.py" line-rate="0.90" />
+                  </classes>
+                </package>
+              </packages>
+            </coverage>
+            """
+        ),
+        encoding="utf-8",
+    )
+
+    contract = {
+        "coverage": {
+            "minimum_percent": 95.0,
+            "tracked_files": ["ok.py", "low.py", "missing.py"],
+        }
+    }
+
+    issues = run_coverage_check(contract, tmp_path, coverage_xml)
+
+    assert "coverage below 95.0%: low.py (90.0%)" in issues
+    assert "coverage missing for tracked file: missing.py" in issues
+
+
+def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Path):
+    missing_xml = tmp_path / "missing.xml"
+    assert run_coverage_check({"coverage": {"tracked_files": []}}, tmp_path, missing_xml) == [
+        "coverage xml missing: missing.xml"
+    ]
+
+    source_dir = tmp_path / "pkg"
+    source_dir.mkdir()
+    (source_dir / "mapped.py").write_text("value = 1\n", encoding="utf-8")
+    coverage_xml = tmp_path / "coverage.xml"
+    coverage_xml.write_text(
+        textwrap.dedent(
+            f"""\
+            <coverage>
+              <sources>
+                <source>{source_dir}</source>
+              </sources>
+              <packages>
+                <package>
+                  <classes>
+                    <class filename="mapped.py" line-rate="1.0" />
+                    <class filename="{(tmp_path / 'absolute.py').as_posix()}" line-rate="1.0" />
+                    <class filename="skip.py" />
+                  </classes>
+                </package>
+              </packages>
+            </coverage>
+            """
+        ),
+        encoding="utf-8",
+    )
+    (tmp_path / "absolute.py").write_text("value = 2\n", encoding="utf-8")
+
+    issues = run_coverage_check(
+        {
+            "coverage": {
+                "minimum_percent": 95.0,
+                "tracked_files": ["pkg/mapped.py", "absolute.py"],
+            }
+        },
+        tmp_path,
+        coverage_xml,
+    )
+
+    assert issues == []

testing/tests/test_quality_gate.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from testing import quality_gate
+
+
+def test_run_profile_aggregates_internal_and_pytest_results(tmp_path: Path, monkeypatch):
+    build_dir = tmp_path / "build"
+    unit_test = tmp_path / "test_sample.py"
+    unit_test.write_text("def test_ok():\n    assert True\n", encoding="utf-8")
+
+    contract = {
+        "profiles": {"local": ["docs", "smell", "hygiene", "unit", "coverage"]},
+        "pytest_suites": {
+            "unit": {
+                "description": "Unit suite",
+                "paths": [str(unit_test.relative_to(tmp_path))],
+                "junit": "build/junit-unit.xml",
+                "coverage_xml": "build/coverage-unit.xml",
+                "coverage_sources": [],
+            }
+        },
+        "manual_scripts": [{"path": "manual.py", "description": "Manual"}],
+    }
+
+    monkeypatch.setattr(quality_gate, "run_docs_check", lambda *_: [])
+    monkeypatch.setattr(quality_gate, "run_hygiene_check", lambda *_: [])
+    monkeypatch.setattr(quality_gate, "run_coverage_check", lambda *_: [])
+
+    calls = []
+
+    def fake_run(command, cwd, check):
+        calls.append((command, cwd, check))
+        if "--junitxml=" in " ".join(command):
+            (build_dir / "junit-unit.xml").write_text(
+                '<testsuite tests="1" failures="0" errors="0" skipped="0" />',
+                encoding="utf-8",
+            )
+            (build_dir / "coverage-unit.xml").write_text("<coverage />", encoding="utf-8")
+        return type("Completed", (), {"returncode": 0})()
+
+    monkeypatch.setattr(quality_gate.subprocess, "run", fake_run)
+
+    summary = quality_gate.run_profile(contract, tmp_path, "local", build_dir)
+
+    assert summary["status"] == "ok"
+    assert [result["name"] for result in summary["results"]] == [
+        "docs",
+        "smell",
+        "hygiene",
+        "unit",
+        "coverage",
+    ]
+    assert calls[0][0][:3] == [quality_gate.sys.executable, "-m", "ruff"]
+    assert any(result.get("junit") == "build/junit-unit.xml" for result in summary["results"])
+
+
+def test_main_writes_summary_file(tmp_path: Path, monkeypatch):
+    summary = {"status": "ok", "profile": "local", "results": [], "manual_scripts": []}
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(quality_gate, "load_contract", lambda: {"profiles": {"local": []}, "pytest_suites": {}})
+    monkeypatch.setattr(quality_gate, "run_profile", lambda *args, **kwargs: summary)
+
+    rc = quality_gate.main(["--profile", "local", "--build-dir", "build"])
+
+    assert rc == 0
+    assert (tmp_path / "build" / "quality-gate-summary.json").exists()