ci: add glue tests and deploy gate

ci/Jenkinsfile.titan-iac

@@ -0,0 +1,53 @@
+pipeline {
+  agent {
+    kubernetes {
+      defaultContainer 'python'
+      yaml """
+apiVersion: v1
+kind: Pod
+spec:
+  containers:
+    - name: python
+      image: python:3.12-slim
+      command:
+        - cat
+      tty: true
+"""
+    }
+  }
+  environment {
+    PIP_DISABLE_PIP_VERSION_CHECK = '1'
+    PYTHONUNBUFFERED = '1'
+    DEPLOY_BRANCH = 'deploy'
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        checkout scm
+      }
+    }
+    stage('Install deps') {
+      steps {
+        sh 'pip install --no-cache-dir -r ci/requirements.txt'
+      }
+    }
+    stage('Glue tests') {
+      steps {
+        sh 'pytest -q ci/tests/glue'
+      }
+    }
+    stage('Promote') {
+      steps {
+        withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
+          sh '''
+            set +x
+            git config user.email "jenkins@bstein.dev"
+            git config user.name "jenkins"
+            git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
+            git push origin HEAD:${DEPLOY_BRANCH}
+          '''
+        }
+      }
+    }
+  }
+}

ci/requirements.txt

@@ -0,0 +1,4 @@
+pytest==8.3.4
+kubernetes==30.1.0
+PyYAML==6.0.2
+requests==2.32.3

ci/tests/glue/config.yaml

@@ -0,0 +1,7 @@
+max_success_age_hours: 48
+allow_suspended:
+  - comms/othrys-room-reset
+  - comms/pin-othrys-invite
+  - comms/seed-othrys-room
+  - finance/firefly-user-sync
+  - health/wger-user-sync

ci/tests/glue/test_glue_cronjobs.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+import yaml
+from kubernetes import client, config
+
+
+CONFIG_PATH = Path(__file__).with_name("config.yaml")
+
+
+def _load_config() -> dict:
+    with CONFIG_PATH.open("r", encoding="utf-8") as handle:
+        return yaml.safe_load(handle) or {}
+
+
+def _load_kube():
+    try:
+        config.load_incluster_config()
+    except config.ConfigException:
+        config.load_kube_config()
+
+
+def test_glue_cronjobs_recent_success():
+    cfg = _load_config()
+    max_age_hours = int(cfg.get("max_success_age_hours", 48))
+    allow_suspended = set(cfg.get("allow_suspended", []))
+
+    _load_kube()
+    batch = client.BatchV1Api()
+    cronjobs = batch.list_cron_job_for_all_namespaces(label_selector="atlas.bstein.dev/glue=true").items
+
+    assert cronjobs, "No glue cronjobs found with atlas.bstein.dev/glue=true"
+
+    now = datetime.now(timezone.utc)
+    for cronjob in cronjobs:
+        name = f"{cronjob.metadata.namespace}/{cronjob.metadata.name}"
+        if cronjob.spec.suspend:
+            assert name in allow_suspended, f"{name} is suspended but not in allow_suspended"
+            continue
+
+        last_success = cronjob.status.last_successful_time
+        assert last_success is not None, f"{name} has no lastSuccessfulTime"
+        age_hours = (now - last_success).total_seconds() / 3600
+        assert age_hours <= max_age_hours, f"{name} last success {age_hours:.1f}h ago"

ci/tests/glue/test_glue_metrics.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import os
+
+import requests
+
+
+VM_URL = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
+
+
+def _query(promql: str) -> list[dict]:
+    response = requests.get(f"{VM_URL}/api/v1/query", params={"query": promql}, timeout=10)
+    response.raise_for_status()
+    payload = response.json()
+    return payload.get("data", {}).get("result", [])
+
+
+def test_glue_metrics_present():
+    series = _query('kube_cronjob_labels{label_atlas_bstein_dev_glue="true"}')
+    assert series, "No glue cronjob label series found"
+
+
+def test_glue_metrics_success_join():
+    query = (
+        "kube_cronjob_status_last_successful_time "
+        'and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue="true"}'
+    )
+    series = _query(query)
+    assert series, "No glue cronjob last success series found"

clusters/atlas/flux-system/gotk-sync.yaml

@@ -9,7 +9,7 @@ metadata:
 spec:
   interval: 1m0s
   ref:
-    branch: feature/vault-consumption
+    branch: deploy
   secretRef:
     name: flux-system-gitea
   url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git

services/jenkins/configmap-jcasc.yaml

@@ -139,6 +139,25 @@ data:
               }
             }
           }
+          pipelineJob('titan-iac-quality-gate') {
+            triggers {
+              scm('H/5 * * * *')
+            }
+            definition {
+              cpsScm {
+                scm {
+                  git {
+                    remote {
+                      url('https://scm.bstein.dev/bstein/titan-iac.git')
+                      credentials('gitea-pat')
+                    }
+                    branches('*/feature/vault-consumption')
+                  }
+                }
+                scriptPath('ci/Jenkinsfile.titan-iac')
+              }
+            }
+          }
   base.yaml: |
     jenkins:
       disableRememberMe: false