Roll Veles backend launch-scope hotfix

This reverts commit 19d6ffcf2a4268fd414cbe5109aafd043d7bb514.

Jenkinsfile

@@ -7,14 +7,52 @@ pipeline {
 apiVersion: v1
 kind: Pod
 spec:
-  serviceAccountName: "jenkins"
   nodeSelector:
-    hardware: rpi5
     kubernetes.io/arch: arm64
     node-role.kubernetes.io/worker: "true"
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: NotIn
+                values:
+                  - titan-04
+                  - titan-06
+                  - titan-11
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 100
+          preference:
+            matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: NotIn
+                values:
+                  - titan-13
+                  - titan-15
+                  - titan-17
+                  - titan-19
+  topologySpreadConstraints:
+    - maxSkew: 1
+      topologyKey: kubernetes.io/hostname
+      whenUnsatisfiable: ScheduleAnyway
+      labelSelector:
+        matchLabels:
+          jenkins/jenkins-jenkins-agent: "true"
   containers:
+    - name: jnlp
+      image: jenkins/inbound-agent:3355.v388858a_47b_33-2-jdk21
+      resources:
+        requests:
+          cpu: "25m"
+          memory: "256Mi"
     - name: python
-      image: python:3.12-slim
+      image: registry.bstein.dev/bstein/python:3.12-slim
+      command:
+        - cat
+      tty: true
+    - name: quality-tools
+      image: registry.bstein.dev/bstein/quality-tools:sonar8.0.1-trivy0.70.0-db20260422-arm64
       command:
         - cat
       tty: true
@@ -24,9 +62,21 @@ spec:
   environment {
     PIP_DISABLE_PIP_VERSION_CHECK = '1'
     PYTHONUNBUFFERED = '1'
-    SUITE_NAME = 'titan-iac'
+    SUITE_NAME = 'titan_iac'
     PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
+    SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
+    SONARQUBE_PROJECT_KEY = 'titan_iac'
+    SONARQUBE_TOKEN = credentials('sonarqube-token')
     VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
+    QUALITY_GATE_SONARQUBE_ENFORCE = '0'
+    QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
+    QUALITY_GATE_IRONBANK_ENFORCE = '1'
+    QUALITY_GATE_IRONBANK_REQUIRED = '0'
+    QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
+  }
+  options {
+    disableConcurrentBuilds()
+    buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
   }
   stages {
     stage('Checkout') {
@@ -36,7 +86,175 @@ spec:
     }
     stage('Install deps') {
       steps {
-        sh 'pip install --no-cache-dir -r ci/requirements.txt'
+        sh '''
+          set -eu
+          if ! command -v git >/dev/null 2>&1; then
+            apt-get update
+            apt-get install -y --no-install-recommends git ca-certificates
+            rm -rf /var/lib/apt/lists/*
+          fi
+          pip install --no-cache-dir -r ci/requirements.txt
+        '''
+      }
+    }
+    stage('Prepare local quality evidence') {
+      steps {
+        sh '''
+          set -eu
+          mkdir -p build
+          set +e
+          python3 -m testing.quality_gate --profile local --build-dir build
+          local_quality_rc=$?
+          set -e
+          printf '%s\n' "${local_quality_rc}" > build/local-quality-gate.rc
+        '''
+      }
+    }
+    stage('Collect SonarQube evidence') {
+      steps {
+        container('quality-tools') {
+          sh '''#!/usr/bin/env bash
+            set -euo pipefail
+            mkdir -p build
+            args=(
+              "-Dsonar.host.url=${SONARQUBE_HOST_URL}"
+              "-Dsonar.login=${SONARQUBE_TOKEN}"
+              "-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}"
+              "-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}"
+              "-Dsonar.sources=."
+              "-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**,services/monitoring/dashboards/**,services/monitoring/grafana-dashboard-*.yaml,services/game-stream/**"
+              "-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
+            )
+            [ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
+            set +e
+            sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
+            rc=${PIPESTATUS[0]}
+            set -e
+            printf '%s\n' "${rc}" > build/sonarqube-analysis.rc
+          '''
+        }
+        sh '''
+          set -eu
+          mkdir -p build
+          python3 - <<'PY'
+import base64
+import json
+import os
+import time
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/')
+project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip()
+token = os.getenv('SONARQUBE_TOKEN', '').strip()
+report_path = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json')
+
+payload = {
+    "status": "ERROR",
+    "note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY",
+}
+if host and project_key:
+    task_file = Path('.scannerwork/report-task.txt')
+    task_id = ''
+    if task_file.exists():
+        for line in task_file.read_text(encoding='utf-8').splitlines():
+            key, _, value = line.partition('=')
+            if key == 'ceTaskId':
+                task_id = value.strip()
+                break
+    if task_id:
+        ce_query = urllib.parse.urlencode({"id": task_id})
+        deadline = time.monotonic() + 180
+        while time.monotonic() < deadline:
+            ce_request = urllib.request.Request(f"{host}/api/ce/task?{ce_query}", method="GET")
+            if token:
+                encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
+                ce_request.add_header("Authorization", f"Basic {encoded}")
+            try:
+                with urllib.request.urlopen(ce_request, timeout=12) as response:
+                    ce_payload = json.loads(response.read().decode("utf-8"))
+            except Exception:
+                time.sleep(3)
+                continue
+            status = str(ce_payload.get("task", {}).get("status", "")).upper()
+            if status in {"SUCCESS", "FAILED", "CANCELED"}:
+                break
+            time.sleep(3)
+
+    query = urllib.parse.urlencode({"projectKey": project_key})
+    request = urllib.request.Request(
+        f"{host}/api/qualitygates/project_status?{query}",
+        method="GET",
+    )
+    if token:
+        encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
+        request.add_header("Authorization", f"Basic {encoded}")
+    try:
+        with urllib.request.urlopen(request, timeout=12) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+    except Exception as exc:  # noqa: BLE001
+        payload = {"status": "ERROR", "error": str(exc)}
+
+with open(report_path, "w", encoding="utf-8") as handle:
+    json.dump(payload, handle, indent=2, sort_keys=True)
+    handle.write("\\n")
+PY
+        '''
+      }
+    }
+    stage('Collect IronBank evidence') {
+      steps {
+        container('quality-tools') {
+          sh '''#!/usr/bin/env bash
+            set -euo pipefail
+            mkdir -p build
+            set +e
+            trivy fs --cache-dir "${TRIVY_CACHE_DIR}" --skip-db-update --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output build/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
+            trivy_rc=$?
+            set -e
+            if [ ! -s build/trivy-fs.json ]; then
+              cat > build/ironbank-compliance.json <<EOF
+{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy did not produce JSON output","trivy_rc":${trivy_rc}}
+EOF
+              exit 0
+            fi
+          '''
+        }
+        sh '''
+          set -eu
+          mkdir -p build
+          if [ -s build/trivy-fs.json ]; then
+            python3 ci/scripts/supply_chain_report.py --trivy-json build/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output build/ironbank-compliance.json
+            exit 0
+          fi
+          python3 - <<'PY'
+import json
+import os
+from pathlib import Path
+
+report_path = Path(os.getenv('QUALITY_GATE_IRONBANK_REPORT', 'build/ironbank-compliance.json'))
+if report_path.exists():
+    raise SystemExit(0)
+
+status = os.getenv('IRONBANK_COMPLIANCE_STATUS', '').strip()
+compliant = os.getenv('IRONBANK_COMPLIANT', '').strip().lower()
+payload = {
+    "status": status or "unknown",
+    "compliant": compliant in {"1", "true", "yes", "on"} if compliant else None,
+}
+payload = {k: v for k, v in payload.items() if v is not None}
+if "status" not in payload:
+    payload["status"] = "unknown"
+payload["note"] = (
+    "Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT "
+    "or write build/ironbank-compliance.json in image-building repos."
+)
+
+report_path.parent.mkdir(parents=True, exist_ok=True)
+report_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
+PY
+        '''
       }
     }
     stage('Run quality gate') {
@@ -66,8 +284,96 @@ spec:
     stage('Enforce quality gate') {
       steps {
         sh '''
-          set -eu
+          set -euo pipefail
-          test "$(cat build/quality-gate.rc 2>/dev/null || echo 1)" -eq 0
+          gate_rc="$(cat build/quality-gate.rc 2>/dev/null || echo 1)"
+          fail=0
+          if [ "${gate_rc}" -ne 0 ]; then
+            echo "quality gate failed with rc=${gate_rc}" >&2
+            fail=1
+          fi
+
+          enabled() {
+            case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in
+              1|true|yes|on) return 0 ;;
+              *) return 1 ;;
+            esac
+          }
+
+          if enabled "${QUALITY_GATE_SONARQUBE_ENFORCE:-1}"; then
+            sonar_status="$(python3 - <<'PY'
+import json
+from pathlib import Path
+
+path = Path("build/sonarqube-quality-gate.json")
+if not path.exists():
+    print("missing")
+    raise SystemExit(0)
+try:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+except Exception:  # noqa: BLE001
+    print("error")
+    raise SystemExit(0)
+status = (payload.get("status") or payload.get("projectStatus", {}).get("status") or payload.get("qualityGate", {}).get("status") or "").strip().lower()
+print(status or "missing")
+PY
+)"
+            case "${sonar_status}" in
+              ok|pass|passed|success) ;;
+              *)
+                echo "sonarqube gate failed: ${sonar_status}" >&2
+                fail=1
+                ;;
+            esac
+          fi
+
+          ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
+          if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
+            ironbank_required=1
+          fi
+          if enabled "${QUALITY_GATE_IRONBANK_ENFORCE:-1}"; then
+            supply_status="$(python3 - <<'PY'
+import json
+from pathlib import Path
+
+path = Path("build/ironbank-compliance.json")
+if not path.exists():
+    print("missing")
+    raise SystemExit(0)
+try:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+except Exception:  # noqa: BLE001
+    print("error")
+    raise SystemExit(0)
+compliant = payload.get("compliant")
+if compliant is True:
+    print("ok")
+elif compliant is False:
+    print("failed")
+else:
+    status = str(payload.get("status") or payload.get("result") or payload.get("compliance") or "").strip().lower()
+    print(status or "missing")
+PY
+)"
+            case "${supply_status}" in
+              ok|pass|passed|success|compliant) ;;
+              not_applicable|na|n/a)
+                if enabled "${ironbank_required}"; then
+                  echo "supply chain gate required but status=${supply_status}" >&2
+                  fail=1
+                fi
+                ;;
+              *)
+                if enabled "${ironbank_required}"; then
+                  echo "supply chain gate failed: ${supply_status}" >&2
+                  fail=1
+                else
+                  echo "supply chain gate not passing (${supply_status}) but not required for this run" >&2
+                fi
+                ;;
+            esac
+          fi
+
+          exit "${fail}"
         '''
       }
     }
@@ -76,7 +382,7 @@ spec:
         script {
           env.FLUX_BRANCH = sh(
             returnStdout: true,
-            script: '''awk '/branch:/{print $2; exit}' clusters/atlas/flux-system/gotk-sync.yaml'''
+            script: "grep -m1 '^\\s*branch:' clusters/atlas/flux-system/gotk-sync.yaml | sed 's/^\\s*branch:\\s*//'"
           ).trim()
           if (!env.FLUX_BRANCH) {
             error('Flux branch not found in gotk-sync.yaml')
@@ -93,16 +399,28 @@ spec:
         }
       }
       steps {
-        container('jnlp') {
+        withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
-          withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
+          sh '''
-            sh '''
+            set -euo pipefail
-              set +x
+            if ! command -v git >/dev/null 2>&1; then
-              git config user.email "jenkins@bstein.dev"
+              if command -v apk >/dev/null 2>&1; then
-              git config user.name "jenkins"
+                apk add --no-cache git >/dev/null
-              git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
+              elif command -v apt-get >/dev/null 2>&1; then
-              git push origin HEAD:${FLUX_BRANCH}
+                apt-get update >/dev/null
-            '''
+                apt-get install -y git >/dev/null
-          }
+              fi
+            fi
+            cd "${WORKSPACE:-$PWD}"
+            if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+              echo "workspace is not a git checkout; skipping promote"
+              exit 0
+            fi
+            set +x
+            git config user.email "jenkins@bstein.dev"
+            git config user.name "jenkins"
+            git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
+            git push origin HEAD:${FLUX_BRANCH}
+          '''
         }
       }
     }
@@ -110,15 +428,23 @@ spec:
   post {
     always {
       script {
-        if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
+        try {
-          try {
+          if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
-            junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
+            try {
-          } catch (Throwable err) {
+              junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
-            echo "junit step unavailable: ${err.class.simpleName}"
+            } catch (Throwable err) {
+              echo "junit step unavailable: ${err.class.simpleName}"
+            }
+          }
+          archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
+        } catch (Throwable err) {
+          if (err.class.simpleName == 'MissingContextVariableException') {
+            echo 'workspace unavailable; skipping post-build artifact collection'
+          } else {
+            throw err
           }
         }
       }
-      archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
     }
   }
 }

ci/Jenkinsfile.titan-iac

@@ -6,14 +6,52 @@ pipeline {
 apiVersion: v1
 kind: Pod
 spec:
-  serviceAccountName: "jenkins"
   nodeSelector:
-    hardware: rpi5
     kubernetes.io/arch: arm64
     node-role.kubernetes.io/worker: "true"
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: NotIn
+                values:
+                  - titan-04
+                  - titan-06
+                  - titan-11
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 100
+          preference:
+            matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: NotIn
+                values:
+                  - titan-13
+                  - titan-15
+                  - titan-17
+                  - titan-19
+  topologySpreadConstraints:
+    - maxSkew: 1
+      topologyKey: kubernetes.io/hostname
+      whenUnsatisfiable: ScheduleAnyway
+      labelSelector:
+        matchLabels:
+          jenkins/jenkins-jenkins-agent: "true"
   containers:
+    - name: jnlp
+      image: jenkins/inbound-agent:3355.v388858a_47b_33-2-jdk21
+      resources:
+        requests:
+          cpu: "25m"
+          memory: "256Mi"
     - name: python
-      image: python:3.12-slim
+      image: registry.bstein.dev/bstein/python:3.12-slim
+      command:
+        - cat
+      tty: true
+    - name: quality-tools
+      image: registry.bstein.dev/bstein/quality-tools:sonar8.0.1-trivy0.70.0-db20260422-arm64
       command:
         - cat
       tty: true
@@ -23,9 +61,21 @@ spec:
   environment {
     PIP_DISABLE_PIP_VERSION_CHECK = '1'
     PYTHONUNBUFFERED = '1'
-    SUITE_NAME = 'titan-iac'
+    SUITE_NAME = 'titan_iac'
     PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
+    SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
+    SONARQUBE_PROJECT_KEY = 'titan_iac'
+    SONARQUBE_TOKEN = credentials('sonarqube-token')
     VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
+    QUALITY_GATE_SONARQUBE_ENFORCE = '0'
+    QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
+    QUALITY_GATE_IRONBANK_ENFORCE = '1'
+    QUALITY_GATE_IRONBANK_REQUIRED = '0'
+    QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
+  }
+  options {
+    disableConcurrentBuilds()
+    buildDiscarder(logRotator(daysToKeepStr: '30', numToKeepStr: '200', artifactDaysToKeepStr: '30', artifactNumToKeepStr: '120'))
   }
   stages {
     stage('Checkout') {
@@ -35,7 +85,175 @@ spec:
     }
     stage('Install deps') {
       steps {
-        sh 'pip install --no-cache-dir -r ci/requirements.txt'
+        sh '''
+          set -eu
+          if ! command -v git >/dev/null 2>&1; then
+            apt-get update
+            apt-get install -y --no-install-recommends git ca-certificates
+            rm -rf /var/lib/apt/lists/*
+          fi
+          pip install --no-cache-dir -r ci/requirements.txt
+        '''
+      }
+    }
+    stage('Prepare local quality evidence') {
+      steps {
+        sh '''
+          set -eu
+          mkdir -p build
+          set +e
+          python3 -m testing.quality_gate --profile local --build-dir build
+          local_quality_rc=$?
+          set -e
+          printf '%s\n' "${local_quality_rc}" > build/local-quality-gate.rc
+        '''
+      }
+    }
+    stage('Collect SonarQube evidence') {
+      steps {
+        container('quality-tools') {
+          sh '''#!/usr/bin/env bash
+            set -euo pipefail
+            mkdir -p build
+            args=(
+              "-Dsonar.host.url=${SONARQUBE_HOST_URL}"
+              "-Dsonar.login=${SONARQUBE_TOKEN}"
+              "-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}"
+              "-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}"
+              "-Dsonar.sources=."
+              "-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**,services/monitoring/dashboards/**,services/monitoring/grafana-dashboard-*.yaml,services/game-stream/**"
+              "-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
+            )
+            [ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
+            set +e
+            sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
+            rc=${PIPESTATUS[0]}
+            set -e
+            printf '%s\n' "${rc}" > build/sonarqube-analysis.rc
+          '''
+        }
+        sh '''
+          set -eu
+          mkdir -p build
+          python3 - <<'PY'
+import base64
+import json
+import os
+import time
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/')
+project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip()
+token = os.getenv('SONARQUBE_TOKEN', '').strip()
+report_path = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json')
+
+payload = {
+    "status": "ERROR",
+    "note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY",
+}
+if host and project_key:
+    task_file = Path('.scannerwork/report-task.txt')
+    task_id = ''
+    if task_file.exists():
+        for line in task_file.read_text(encoding='utf-8').splitlines():
+            key, _, value = line.partition('=')
+            if key == 'ceTaskId':
+                task_id = value.strip()
+                break
+    if task_id:
+        ce_query = urllib.parse.urlencode({"id": task_id})
+        deadline = time.monotonic() + 180
+        while time.monotonic() < deadline:
+            ce_request = urllib.request.Request(f"{host}/api/ce/task?{ce_query}", method="GET")
+            if token:
+                encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
+                ce_request.add_header("Authorization", f"Basic {encoded}")
+            try:
+                with urllib.request.urlopen(ce_request, timeout=12) as response:
+                    ce_payload = json.loads(response.read().decode("utf-8"))
+            except Exception:
+                time.sleep(3)
+                continue
+            status = str(ce_payload.get("task", {}).get("status", "")).upper()
+            if status in {"SUCCESS", "FAILED", "CANCELED"}:
+                break
+            time.sleep(3)
+
+    query = urllib.parse.urlencode({"projectKey": project_key})
+    request = urllib.request.Request(
+        f"{host}/api/qualitygates/project_status?{query}",
+        method="GET",
+    )
+    if token:
+        encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
+        request.add_header("Authorization", f"Basic {encoded}")
+    try:
+        with urllib.request.urlopen(request, timeout=12) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+    except Exception as exc:  # noqa: BLE001
+        payload = {"status": "ERROR", "error": str(exc)}
+
+with open(report_path, "w", encoding="utf-8") as handle:
+    json.dump(payload, handle, indent=2, sort_keys=True)
+    handle.write("\\n")
+PY
+        '''
+      }
+    }
+    stage('Collect IronBank evidence') {
+      steps {
+        container('quality-tools') {
+          sh '''#!/usr/bin/env bash
+            set -euo pipefail
+            mkdir -p build
+            set +e
+            trivy fs --cache-dir "${TRIVY_CACHE_DIR}" --skip-db-update --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output build/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
+            trivy_rc=$?
+            set -e
+            if [ ! -s build/trivy-fs.json ]; then
+              cat > build/ironbank-compliance.json <<EOF
+{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy did not produce JSON output","trivy_rc":${trivy_rc}}
+EOF
+              exit 0
+            fi
+          '''
+        }
+        sh '''
+          set -eu
+          mkdir -p build
+          if [ -s build/trivy-fs.json ]; then
+            python3 ci/scripts/supply_chain_report.py --trivy-json build/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output build/ironbank-compliance.json
+            exit 0
+          fi
+          python3 - <<'PY'
+import json
+import os
+from pathlib import Path
+
+report_path = Path(os.getenv('QUALITY_GATE_IRONBANK_REPORT', 'build/ironbank-compliance.json'))
+if report_path.exists():
+    raise SystemExit(0)
+
+status = os.getenv('IRONBANK_COMPLIANCE_STATUS', '').strip()
+compliant = os.getenv('IRONBANK_COMPLIANT', '').strip().lower()
+payload = {
+    "status": status or "unknown",
+    "compliant": compliant in {"1", "true", "yes", "on"} if compliant else None,
+}
+payload = {k: v for k, v in payload.items() if v is not None}
+if "status" not in payload:
+    payload["status"] = "unknown"
+payload["note"] = (
+    "Set IRONBANK_COMPLIANCE_STATUS/IRONBANK_COMPLIANT "
+    "or write build/ironbank-compliance.json in image-building repos."
+)
+
+report_path.parent.mkdir(parents=True, exist_ok=True)
+report_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\\n", encoding="utf-8")
+PY
+        '''
       }
     }
     stage('Run quality gate') {
@@ -65,8 +283,96 @@ spec:
     stage('Enforce quality gate') {
       steps {
         sh '''
-          set -eu
+          set -euo pipefail
-          test "$(cat build/quality-gate.rc 2>/dev/null || echo 1)" -eq 0
+          gate_rc="$(cat build/quality-gate.rc 2>/dev/null || echo 1)"
+          fail=0
+          if [ "${gate_rc}" -ne 0 ]; then
+            echo "quality gate failed with rc=${gate_rc}" >&2
+            fail=1
+          fi
+
+          enabled() {
+            case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in
+              1|true|yes|on) return 0 ;;
+              *) return 1 ;;
+            esac
+          }
+
+          if enabled "${QUALITY_GATE_SONARQUBE_ENFORCE:-1}"; then
+            sonar_status="$(python3 - <<'PY'
+import json
+from pathlib import Path
+
+path = Path("build/sonarqube-quality-gate.json")
+if not path.exists():
+    print("missing")
+    raise SystemExit(0)
+try:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+except Exception:  # noqa: BLE001
+    print("error")
+    raise SystemExit(0)
+status = (payload.get("status") or payload.get("projectStatus", {}).get("status") or payload.get("qualityGate", {}).get("status") or "").strip().lower()
+print(status or "missing")
+PY
+)"
+            case "${sonar_status}" in
+              ok|pass|passed|success) ;;
+              *)
+                echo "sonarqube gate failed: ${sonar_status}" >&2
+                fail=1
+                ;;
+            esac
+          fi
+
+          ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
+          if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
+            ironbank_required=1
+          fi
+          if enabled "${QUALITY_GATE_IRONBANK_ENFORCE:-1}"; then
+            supply_status="$(python3 - <<'PY'
+import json
+from pathlib import Path
+
+path = Path("build/ironbank-compliance.json")
+if not path.exists():
+    print("missing")
+    raise SystemExit(0)
+try:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+except Exception:  # noqa: BLE001
+    print("error")
+    raise SystemExit(0)
+compliant = payload.get("compliant")
+if compliant is True:
+    print("ok")
+elif compliant is False:
+    print("failed")
+else:
+    status = str(payload.get("status") or payload.get("result") or payload.get("compliance") or "").strip().lower()
+    print(status or "missing")
+PY
+)"
+            case "${supply_status}" in
+              ok|pass|passed|success|compliant) ;;
+              not_applicable|na|n/a)
+                if enabled "${ironbank_required}"; then
+                  echo "supply chain gate required but status=${supply_status}" >&2
+                  fail=1
+                fi
+                ;;
+              *)
+                if enabled "${ironbank_required}"; then
+                  echo "supply chain gate failed: ${supply_status}" >&2
+                  fail=1
+                else
+                  echo "supply chain gate not passing (${supply_status}) but not required for this run" >&2
+                fi
+                ;;
+            esac
+          fi
+
+          exit "${fail}"
         '''
       }
     }
@@ -75,7 +381,7 @@ spec:
         script {
           env.FLUX_BRANCH = sh(
             returnStdout: true,
-            script: '''awk '/branch:/{print $2; exit}' clusters/atlas/flux-system/gotk-sync.yaml'''
+            script: "grep -m1 '^\\s*branch:' clusters/atlas/flux-system/gotk-sync.yaml | sed 's/^\\s*branch:\\s*//'"
           ).trim()
           if (!env.FLUX_BRANCH) {
             error('Flux branch not found in gotk-sync.yaml')
@@ -92,16 +398,28 @@ spec:
         }
       }
       steps {
-        container('jnlp') {
+        withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
-          withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
+          sh '''
-            sh '''
+            set -euo pipefail
-              set +x
+            if ! command -v git >/dev/null 2>&1; then
-              git config user.email "jenkins@bstein.dev"
+              if command -v apk >/dev/null 2>&1; then
-              git config user.name "jenkins"
+                apk add --no-cache git >/dev/null
-              git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
+              elif command -v apt-get >/dev/null 2>&1; then
-              git push origin HEAD:${FLUX_BRANCH}
+                apt-get update >/dev/null
-            '''
+                apt-get install -y git >/dev/null
-          }
+              fi
+            fi
+            cd "${WORKSPACE:-$PWD}"
+            if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+              echo "workspace is not a git checkout; skipping promote"
+              exit 0
+            fi
+            set +x
+            git config user.email "jenkins@bstein.dev"
+            git config user.name "jenkins"
+            git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
+            git push origin HEAD:${FLUX_BRANCH}
+          '''
         }
       }
     }
@@ -109,15 +427,23 @@ spec:
   post {
     always {
       script {
-        if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
+        try {
-          try {
+          if (fileExists('build/junit-unit.xml') || fileExists('build/junit-glue.xml')) {
-            junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
+            try {
-          } catch (Throwable err) {
+              junit allowEmptyResults: true, testResults: 'build/junit-*.xml'
-            echo "junit step unavailable: ${err.class.simpleName}"
+            } catch (Throwable err) {
+              echo "junit step unavailable: ${err.class.simpleName}"
+            }
+          }
+          archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
+        } catch (Throwable err) {
+          if (err.class.simpleName == 'MissingContextVariableException') {
+            echo 'workspace unavailable; skipping post-build artifact collection'
+          } else {
+            throw err
           }
         }
       }
-      archiveArtifacts artifacts: 'build/**', allowEmptyArchive: true, fingerprint: true
     }
   }
 }

ci/scripts/publish_test_metrics.py

@@ -6,30 +6,50 @@ from __future__ import annotations
 import json
 import os
 from glob import glob
+from pathlib import Path
+import sys
 import urllib.error
 import urllib.request
 import xml.etree.ElementTree as ET
 
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
+from ci.scripts import publish_test_metrics_quality as _quality_helpers
+
+CANONICAL_CHECKS = _quality_helpers.CANONICAL_CHECKS
+_build_check_statuses = _quality_helpers._build_check_statuses
+_combine_statuses = _quality_helpers._combine_statuses
+_infer_sonarqube_status = _quality_helpers._infer_sonarqube_status
+_infer_source_lines_over_500 = _quality_helpers._infer_source_lines_over_500
+_infer_supply_chain_status = _quality_helpers._infer_supply_chain_status
+_infer_workspace_coverage_percent = _quality_helpers._infer_workspace_coverage_percent
+_load_optional_json = _quality_helpers._load_optional_json
+_normalize_result_status = _quality_helpers._normalize_result_status
+
 
 def _escape_label(value: str) -> str:
+    """Escape a Prometheus label value without changing its content."""
     return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
 
 
 def _label_str(labels: dict[str, str]) -> str:
+    """Render a stable Prometheus label set from a mapping."""
     parts = [f'{key}="{_escape_label(val)}"' for key, val in labels.items() if val]
     return "{" + ",".join(parts) + "}" if parts else ""
 
 
 def _read_text(url: str) -> str:
+    """Fetch a plain-text response body from the given URL."""
     with urllib.request.urlopen(url, timeout=10) as response:
         return response.read().decode("utf-8")
 
 
 def _post_text(url: str, payload: str) -> None:
+    """PUT a plain-text payload and fail on any 4xx/5xx response."""
     request = urllib.request.Request(
         url,
         data=payload.encode("utf-8"),
-        method="POST",
+        method="PUT",
         headers={"Content-Type": "text/plain"},
     )
     with urllib.request.urlopen(request, timeout=10) as response:
@@ -38,6 +58,7 @@ def _post_text(url: str, payload: str) -> None:
 
 
 def _parse_junit(path: str) -> dict[str, int]:
+    """Parse a JUnit XML file into aggregate test counters."""
     if not os.path.exists(path):
         return {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
 
@@ -64,6 +85,7 @@ def _parse_junit(path: str) -> dict[str, int]:
 
 
 def _collect_junit_totals(pattern: str) -> dict[str, int]:
+    """Sum JUnit counters across every XML file matching the pattern."""
     totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0}
     for path in sorted(glob(pattern)):
         parsed = _parse_junit(path)
@@ -72,7 +94,38 @@ def _collect_junit_totals(pattern: str) -> dict[str, int]:
     return totals
 
 
+def _collect_junit_cases(pattern: str) -> list[tuple[str, str]]:
+    """Collect individual JUnit test-case statuses for flaky-test trend panels."""
+    cases: list[tuple[str, str]] = []
+    for path in sorted(glob(pattern)):
+        if not os.path.exists(path):
+            continue
+        root = ET.parse(path).getroot()
+        suites: list[ET.Element]
+        if root.tag == "testsuite":
+            suites = [root]
+        elif root.tag == "testsuites":
+            suites = [elem for elem in root if elem.tag == "testsuite"]
+        else:
+            suites = []
+        for suite in suites:
+            for test_case in suite.findall("testcase"):
+                case_name = test_case.attrib.get("name", "").strip()
+                class_name = test_case.attrib.get("classname", "").strip()
+                if not case_name:
+                    continue
+                full_name = f"{class_name}.{case_name}" if class_name else case_name
+                status = "passed"
+                if test_case.find("failure") is not None or test_case.find("error") is not None:
+                    status = "failed"
+                elif test_case.find("skipped") is not None:
+                    status = "skipped"
+                cases.append((full_name, status))
+    return cases
+
+
 def _read_exit_code(path: str) -> int:
+    """Read the quality-gate exit code, defaulting to failure if missing."""
     try:
         with open(path, "r", encoding="utf-8") as handle:
             return int(handle.read().strip())
@@ -81,6 +134,7 @@ def _read_exit_code(path: str) -> int:
 
 
 def _load_summary(path: str) -> dict:
+    """Load the JSON quality-gate summary, returning an empty mapping on error."""
     try:
         with open(path, "r", encoding="utf-8") as handle:
             return json.load(handle)
@@ -88,7 +142,26 @@ def _load_summary(path: str) -> dict:
         return {}
 
 
+def _summary_float(summary: dict, key: str) -> float:
+    """Extract a float-like value from the summary, defaulting to 0.0."""
+    value = summary.get(key)
+    if isinstance(value, (int, float)):
+        return float(value)
+    return 0.0
+
+
+def _summary_int(summary: dict, key: str) -> int:
+    """Extract an int-like value from the summary, defaulting to 0."""
+    value = summary.get(key)
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return int(value)
+    return 0
+
+
 def _fetch_existing_counter(pushgateway_url: str, metric: str, labels: dict[str, str]) -> float:
+    """Return the current counter value for a labeled metric if present."""
     text = _read_text(f"{pushgateway_url.rstrip('/')}/metrics")
     for line in text.splitlines():
         if not line.startswith(metric + "{"):
@@ -109,20 +182,34 @@ def _build_payload(
     suite: str,
     status: str,
     tests: dict[str, int],
+    test_cases: list[tuple[str, str]],
     ok_count: int,
     failed_count: int,
     branch: str,
     build_number: str,
+    jenkins_job: str,
     summary: dict | None = None,
+    workspace_line_coverage_percent: float = 0.0,
+    source_files_total: int = 0,
+    source_lines_over_500: int = 0,
+    check_statuses: dict[str, str] | None = None,
 ) -> str:
+    """Build the Pushgateway payload for the current suite run."""
     passed = max(tests["tests"] - tests["failures"] - tests["errors"] - tests["skipped"], 0)
     build_labels = _label_str(
         {
             "suite": suite,
             "branch": branch or "unknown",
             "build_number": build_number or "unknown",
+            "jenkins_job": jenkins_job or suite,
         }
     )
+    test_case_base_labels = {
+        "suite": suite,
+        "branch": branch or "unknown",
+        "build_number": build_number or "unknown",
+        "jenkins_job": jenkins_job or suite,
+    }
     lines = [
         "# TYPE platform_quality_gate_runs_total counter",
         f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count}',
@@ -135,37 +222,85 @@ def _build_payload(
         "# TYPE titan_iac_quality_gate_run_status gauge",
         f'titan_iac_quality_gate_run_status{{suite="{suite}",status="ok"}} {1 if status == "ok" else 0}',
         f'titan_iac_quality_gate_run_status{{suite="{suite}",status="failed"}} {1 if status == "failed" else 0}',
+        "# TYPE platform_quality_gate_build_info gauge",
+        f"platform_quality_gate_build_info{build_labels} 1",
         "# TYPE titan_iac_quality_gate_build_info gauge",
         f"titan_iac_quality_gate_build_info{build_labels} 1",
+        "# TYPE platform_quality_gate_workspace_line_coverage_percent gauge",
+        f'platform_quality_gate_workspace_line_coverage_percent{{suite="{suite}"}} {workspace_line_coverage_percent:.3f}',
+        "# TYPE platform_quality_gate_source_files_total gauge",
+        f'platform_quality_gate_source_files_total{{suite="{suite}"}} {source_files_total}',
+        "# TYPE platform_quality_gate_source_lines_over_500_total gauge",
+        f'platform_quality_gate_source_lines_over_500_total{{suite="{suite}"}} {source_lines_over_500}',
     ]
-    results = summary.get("results", []) if isinstance(summary, dict) else []
+    if check_statuses:
-    if results:
         lines.append("# TYPE titan_iac_quality_gate_checks_total gauge")
-        for result in results:
+        for check_name in CANONICAL_CHECKS:
-            check_name = result.get("name")
+            check_status = check_statuses.get(check_name, "not_applicable")
-            check_status = result.get("status")
-            if not check_name or not check_status:
-                continue
             lines.append(
-                f'titan_iac_quality_gate_checks_total{{suite="{suite}",check="{_escape_label(str(check_name))}",result="{_escape_label(str(check_status))}"}} 1'
+                f'titan_iac_quality_gate_checks_total{{suite="{suite}",check="{_escape_label(check_name)}",result="{_escape_label(check_status)}"}} 1'
             )
+    lines.append("# TYPE platform_quality_gate_test_case_result gauge")
+    if test_cases:
+        for test_name, test_status in test_cases:
+            labels = {
+                **test_case_base_labels,
+                "test": test_name,
+                "status": test_status,
+            }
+            lines.append(
+                f"platform_quality_gate_test_case_result{_label_str(labels)} 1"
+            )
+    else:
+        labels = {**test_case_base_labels, "test": "__no_test_cases__", "status": "skipped"}
+        lines.append(
+            f"platform_quality_gate_test_case_result{_label_str(labels)} 1"
+        )
     return "\n".join(lines) + "\n"
 
 
 def main() -> int:
-    suite = os.getenv("SUITE_NAME", "titan-iac")
+    """Publish the quality-gate metrics and print a compact run summary."""
+    suite = os.getenv("SUITE_NAME", "titan_iac")
     pushgateway_url = os.getenv("PUSHGATEWAY_URL", "http://platform-quality-gateway.monitoring.svc.cluster.local:9091")
     job_name = os.getenv("QUALITY_GATE_JOB_NAME", "platform-quality-ci")
     junit_glob = os.getenv("JUNIT_GLOB", os.getenv("JUNIT_PATH", "build/junit-*.xml"))
     exit_code_path = os.getenv("QUALITY_GATE_EXIT_CODE_PATH", os.getenv("GLUE_EXIT_CODE_PATH", "build/quality-gate.rc"))
     summary_path = os.getenv("QUALITY_GATE_SUMMARY_PATH", "build/quality-gate-summary.json")
-    branch = os.getenv("BRANCH_NAME", os.getenv("GIT_BRANCH", ""))
+    branch = os.getenv("BRANCH_NAME") or os.getenv("GIT_BRANCH") or "unknown"
+    if branch.startswith("origin/"):
+        branch = branch[len("origin/") :]
     build_number = os.getenv("BUILD_NUMBER", "")
+    jenkins_job = os.getenv("JOB_NAME", "titan-iac")
 
     tests = _collect_junit_totals(junit_glob)
+    test_cases = _collect_junit_cases(junit_glob)
     exit_code = _read_exit_code(exit_code_path)
     status = "ok" if exit_code == 0 else "failed"
     summary = _load_summary(summary_path)
+    workspace_line_coverage_percent = _summary_float(summary, "workspace_line_coverage_percent")
+    if workspace_line_coverage_percent <= 0:
+        workspace_line_coverage_percent = _infer_workspace_coverage_percent(summary, "build/coverage-unit.xml")
+    source_files_total = _summary_int(summary, "source_files_total")
+    source_lines_over_500 = _summary_int(summary, "source_lines_over_500")
+    if source_lines_over_500 <= 0:
+        source_lines_over_500 = _infer_source_lines_over_500(summary)
+    sonarqube_report = _load_optional_json(os.getenv("QUALITY_GATE_SONARQUBE_REPORT", "build/sonarqube-quality-gate.json"))
+    supply_chain_report = _load_optional_json(os.getenv("QUALITY_GATE_IRONBANK_REPORT", "build/ironbank-compliance.json"))
+    truthy = {"1", "true", "yes", "on"}
+    supply_chain_required = (
+        os.getenv("QUALITY_GATE_IRONBANK_REQUIRED", "0").strip().lower() in truthy
+        or os.getenv("PUBLISH_IMAGES", "false").strip().lower() in truthy
+    )
+    check_statuses = _build_check_statuses(
+        summary=summary,
+        tests=tests,
+        workspace_line_coverage_percent=workspace_line_coverage_percent,
+        source_lines_over_500=source_lines_over_500,
+        sonarqube_report=sonarqube_report,
+        supply_chain_report=supply_chain_report,
+        supply_chain_required=supply_chain_required,
+    )
 
     ok_count = int(
         _fetch_existing_counter(
@@ -190,11 +325,17 @@ def main() -> int:
         suite=suite,
         status=status,
         tests=tests,
+        test_cases=test_cases,
         ok_count=ok_count,
         failed_count=failed_count,
         branch=branch,
         build_number=build_number,
+        jenkins_job=jenkins_job,
         summary=summary,
+        workspace_line_coverage_percent=workspace_line_coverage_percent,
+        source_files_total=source_files_total,
+        source_lines_over_500=source_lines_over_500,
+        check_statuses=check_statuses,
     )
     push_url = f"{pushgateway_url.rstrip('/')}/metrics/job/{job_name}/suite/{suite}"
     _post_text(push_url, payload)
@@ -208,11 +349,14 @@ def main() -> int:
         "tests_skipped": tests["skipped"],
         "ok_count": ok_count,
         "failed_count": failed_count,
-        "checks_recorded": len(summary.get("results", [])) if isinstance(summary, dict) else 0,
+        "checks_recorded": len(check_statuses),
+        "workspace_line_coverage_percent": workspace_line_coverage_percent,
+        "source_files_total": source_files_total,
+        "source_lines_over_500": source_lines_over_500,
     }
     print(json.dumps(summary, sort_keys=True))
     return 0
 
 
-if __name__ == "__main__":
+if __name__ == "__main__":  # pragma: no cover
     raise SystemExit(main())

ci/scripts/publish_test_metrics_quality.py

@@ -0,0 +1,204 @@
+#!/usr/bin/env python3
+"""Quality/status helpers for publish_test_metrics."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+import xml.etree.ElementTree as ET
+
+SUCCESS_STATUSES = {"ok", "pass", "passed", "success", "compliant"}
+NOT_APPLICABLE_STATUSES = {"not_applicable", "n/a", "na", "none", "skipped"}
+FAILED_STATUSES = {"failed", "fail", "error", "errors", "warn", "warning", "red"}
+
+CANONICAL_CHECKS = [
+    "tests",
+    "coverage",
+    "loc",
+    "docs_naming",
+    "gate_glue",
+    "sonarqube",
+    "supply_chain",
+]
+
+
+def _infer_workspace_coverage_percent(summary: dict, default_xml: str) -> float:
+    """Infer workspace line coverage from quality summary coverage XML metadata."""
+    results = summary.get("results", []) if isinstance(summary, dict) else []
+    coverage_xml = default_xml
+    for result in results:
+        if not isinstance(result, dict):
+            continue
+        if str(result.get("name") or "").strip().lower() != "coverage":
+            continue
+        candidate = str(result.get("coverage_xml") or "").strip()
+        if candidate:
+            coverage_xml = candidate
+            break
+    xml_path = Path(coverage_xml)
+    if not xml_path.exists():
+        return 0.0
+    try:
+        root = ET.parse(xml_path).getroot()
+        line_rate = root.attrib.get("line-rate")
+        if line_rate is None:
+            return 0.0
+        return float(line_rate) * 100.0
+    except (ET.ParseError, OSError, ValueError):
+        return 0.0
+
+
+def _infer_source_lines_over_500(summary: dict) -> int:
+    """Infer over-limit source file count from hygiene issue payloads."""
+    results = summary.get("results", []) if isinstance(summary, dict) else []
+    for result in results:
+        if not isinstance(result, dict):
+            continue
+        if str(result.get("name") or "").strip().lower() not in {"hygiene", "loc", "smell"}:
+            continue
+        issues = result.get("issues")
+        if not isinstance(issues, list):
+            continue
+        return sum(1 for item in issues if isinstance(item, str) and item.startswith("file exceeds"))
+    return 0
+
+
+def _normalize_result_status(value: str | None, default: str = "failed") -> str:
+    """Map arbitrary check status text into canonical check result buckets."""
+    if not value:
+        return default
+    normalized = value.strip().lower()
+    if normalized in SUCCESS_STATUSES:
+        return "ok"
+    if normalized in NOT_APPLICABLE_STATUSES:
+        return "not_applicable"
+    if normalized in FAILED_STATUSES:
+        return "failed"
+    return default
+
+
+def _load_optional_json(path: str | None) -> dict:
+    """Load an optional JSON report file, returning an empty object when absent."""
+    if not path:
+        return {}
+    candidate = Path(path)
+    if not candidate.exists():
+        return {}
+    try:
+        return json.loads(candidate.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        return {}
+
+
+def _combine_statuses(statuses: list[str]) -> str:
+    """Roll up many check statuses into one canonical result."""
+    if not statuses:
+        return "not_applicable"
+    if any(status == "failed" for status in statuses):
+        return "failed"
+    if all(status == "not_applicable" for status in statuses):
+        return "not_applicable"
+    if all(status in {"ok", "not_applicable"} for status in statuses):
+        return "ok"
+    return "failed"
+
+
+def _infer_sonarqube_status(report: dict) -> str:
+    """Infer canonical SonarQube check status from its JSON report payload."""
+    if not report:
+        return "not_applicable"
+    status = (
+        report.get("projectStatus", {}).get("status")
+        or report.get("qualityGate", {}).get("status")
+        or report.get("status")
+    )
+    return _normalize_result_status(str(status) if status is not None else None, default="failed")
+
+
+def _infer_supply_chain_status(report: dict, required: bool) -> str:
+    """Infer canonical supply-chain status from IronBank/artifact report payload."""
+    if not report:
+        return "failed" if required else "not_applicable"
+    compliant = report.get("compliant")
+    if isinstance(compliant, bool):
+        if compliant:
+            return "ok"
+        return "failed" if required else "not_applicable"
+    status = report.get("status")
+    if status is None:
+        return "failed" if required else "not_applicable"
+    normalized = _normalize_result_status(str(status), default="failed")
+    if normalized == "failed" and not required:
+        return "not_applicable"
+    if normalized == "not_applicable" and required:
+        return "failed"
+    return normalized
+
+
+def _build_check_statuses(
+    summary: dict | None,
+    tests: dict[str, int],
+    workspace_line_coverage_percent: float,
+    source_lines_over_500: int,
+    sonarqube_report: dict,
+    supply_chain_report: dict,
+    supply_chain_required: bool,
+) -> dict[str, str]:
+    """Generate the canonical quality-check status map for dashboarding."""
+    raw_results = summary.get("results", []) if isinstance(summary, dict) else []
+    status_by_name: dict[str, str] = {}
+    for result in raw_results:
+        if not isinstance(result, dict):
+            continue
+        check_name = str(result.get("name") or "").strip().lower()
+        if not check_name:
+            continue
+        status_by_name[check_name] = _normalize_result_status(result.get("status"), default="failed")
+
+    tests_status = status_by_name.get("tests")
+    if not tests_status:
+        candidate_keys = ["unit", "integration", "e2e", "pytest", "test", "tests"]
+        candidates = [status_by_name[key] for key in candidate_keys if key in status_by_name]
+        if candidates:
+            tests_status = _combine_statuses(candidates)
+        elif tests["tests"] > 0:
+            tests_status = "ok" if (tests["failures"] + tests["errors"]) == 0 else "failed"
+        else:
+            tests_status = "not_applicable"
+
+    coverage_status = status_by_name.get("coverage")
+    if not coverage_status:
+        if workspace_line_coverage_percent > 0:
+            coverage_status = "ok" if workspace_line_coverage_percent >= 95.0 else "failed"
+        else:
+            coverage_status = "not_applicable"
+
+    loc_status = status_by_name.get("loc")
+    if not loc_status:
+        loc_status = "ok" if source_lines_over_500 == 0 else "failed"
+
+    docs_naming_status = status_by_name.get("docs_naming")
+    if not docs_naming_status:
+        candidates = [status_by_name[key] for key in ["docs", "hygiene", "smell", "lint", "naming"] if key in status_by_name]
+        docs_naming_status = _combine_statuses(candidates) if candidates else "not_applicable"
+
+    gate_glue_status = status_by_name.get("gate_glue")
+    if not gate_glue_status:
+        candidates = [status_by_name[key] for key in ["gate_glue", "glue", "gate"] if key in status_by_name]
+        gate_glue_status = _combine_statuses(candidates) if candidates else "not_applicable"
+
+    sonarqube_status = status_by_name.get("sonarqube") or _infer_sonarqube_status(sonarqube_report)
+    supply_chain_status = status_by_name.get("supply_chain") or _infer_supply_chain_status(
+        supply_chain_report,
+        required=supply_chain_required,
+    )
+
+    return {
+        "tests": tests_status,
+        "coverage": coverage_status,
+        "loc": loc_status,
+        "docs_naming": docs_naming_status,
+        "gate_glue": gate_glue_status,
+        "sonarqube": sonarqube_status,
+        "supply_chain": supply_chain_status,
+    }

ci/scripts/supply_chain_report.py

@@ -0,0 +1,173 @@
+"""Build a titan-iac supply-chain compliance report from Trivy evidence."""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+from pathlib import Path
+from typing import Any
+
+
+FAIL_SEVERITIES = {"HIGH", "CRITICAL"}
+
+
+def _read_json(path: Path) -> dict[str, Any]:
+    """Read a JSON object from disk for use as pipeline evidence."""
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(payload, dict):
+        raise ValueError(f"{path} must contain a JSON object")
+    return payload
+
+
+def _parse_day(raw: str | None) -> dt.date | None:
+    """Parse an ISO day while letting optional waiver dates stay optional."""
+    if not raw:
+        return None
+    return dt.date.fromisoformat(raw)
+
+
+def _today(override: str | None = None) -> dt.date:
+    """Return the policy day so tests can pin expiry behavior."""
+    return _parse_day(override) or dt.date.today()
+
+
+def _load_waiver_pairs(path: Path | None, policy_day: dt.date) -> tuple[set[tuple[str, str]], int]:
+    """Return active ``(misconfiguration id, target)`` waivers and expired count."""
+    if path is None or not path.exists():
+        return set(), 0
+
+    payload = _read_json(path)
+    default_expires_at = payload.get("default_expires_at")
+    active: set[tuple[str, str]] = set()
+    expired = 0
+
+    for entry in payload.get("misconfigurations", []):
+        if not isinstance(entry, dict):
+            continue
+        misconfiguration_id = str(entry.get("id") or "").strip()
+        if not misconfiguration_id:
+            continue
+        expires_at = _parse_day(str(entry.get("expires_at") or default_expires_at or ""))
+        targets = entry.get("targets", [])
+        if not isinstance(targets, list):
+            continue
+
+        if expires_at and expires_at < policy_day:
+            expired += len(targets)
+            continue
+
+        # Waivers are target-specific so a new unsafe manifest fails until it is
+        # either fixed or deliberately accepted with a fresh expiration.
+        for target in targets:
+            if isinstance(target, str) and target:
+                active.add((misconfiguration_id, target))
+
+    return active, expired
+
+
+def _iter_failed_misconfigurations(payload: dict[str, Any]):
+    """Yield failed high/critical Trivy misconfiguration records."""
+    for result in payload.get("Results", []):
+        if not isinstance(result, dict):
+            continue
+        target = str(result.get("Target") or "")
+        for item in result.get("Misconfigurations") or []:
+            if not isinstance(item, dict):
+                continue
+            if item.get("Status") != "FAIL":
+                continue
+            if str(item.get("Severity") or "").upper() not in FAIL_SEVERITIES:
+                continue
+            yield target, item
+
+
+def _count_vulnerabilities(payload: dict[str, Any], severity: str) -> int:
+    """Count Trivy vulnerabilities at a specific severity."""
+    count = 0
+    for result in payload.get("Results", []):
+        if not isinstance(result, dict):
+            continue
+        for item in result.get("Vulnerabilities") or []:
+            if isinstance(item, dict) and str(item.get("Severity") or "").upper() == severity:
+                count += 1
+    return count
+
+
+def _count_secrets(payload: dict[str, Any]) -> int:
+    """Count detected secrets in the Trivy filesystem report."""
+    count = 0
+    for result in payload.get("Results", []):
+        if isinstance(result, dict):
+            count += len(result.get("Secrets") or [])
+    return count
+
+
+def build_report(
+    trivy_payload: dict[str, Any],
+    waiver_path: Path | None = None,
+    today_override: str | None = None,
+) -> dict[str, Any]:
+    """Build the compliance summary consumed by the quality gate."""
+    policy_day = _today(today_override)
+    active_waivers, expired_waivers = _load_waiver_pairs(waiver_path, policy_day)
+
+    open_misconfigs: list[dict[str, str]] = []
+    waived_misconfigs = 0
+    for target, item in _iter_failed_misconfigurations(trivy_payload):
+        misconfiguration_id = str(item.get("ID") or "")
+        if (misconfiguration_id, target) in active_waivers:
+            waived_misconfigs += 1
+            continue
+        open_misconfigs.append(
+            {
+                "id": misconfiguration_id,
+                "target": target,
+                "severity": str(item.get("Severity") or ""),
+                "title": str(item.get("Title") or ""),
+            }
+        )
+
+    critical = _count_vulnerabilities(trivy_payload, "CRITICAL")
+    high = _count_vulnerabilities(trivy_payload, "HIGH")
+    secrets = _count_secrets(trivy_payload)
+    status = "ok" if critical == 0 and secrets == 0 and not open_misconfigs else "failed"
+
+    return {
+        "status": status,
+        "compliant": status == "ok",
+        "category": "artifact_security",
+        "scan_type": "filesystem",
+        "scanner": "trivy",
+        "critical_vulnerabilities": critical,
+        "high_vulnerabilities": high,
+        "high_vulnerability_policy": "observe",
+        "secrets": secrets,
+        "high_or_critical_misconfigurations": len(open_misconfigs),
+        "waived_misconfigurations": waived_misconfigs,
+        "expired_waivers": expired_waivers,
+        "waiver_file": str(waiver_path) if waiver_path else "",
+        "open_misconfiguration_examples": open_misconfigs[:20],
+    }
+
+
+def main(argv: list[str] | None = None) -> int:
+    """CLI entrypoint used by Jenkins after the Trivy scan completes."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--trivy-json", required=True)
+    parser.add_argument("--waivers")
+    parser.add_argument("--output", required=True)
+    parser.add_argument("--today")
+    args = parser.parse_args(argv)
+
+    trivy_payload = _read_json(Path(args.trivy_json))
+    waiver_path = Path(args.waivers) if args.waivers else None
+    report = build_report(trivy_payload, waiver_path=waiver_path, today_override=args.today)
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

ci/tests/glue/test_ariadne_schedules.py

@@ -0,0 +1,108 @@
+"""Glue checks for Ariadne schedules exported to VictoriaMetrics."""
+
+from __future__ import annotations
+
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+
+import requests
+import yaml
+
+
+CONFIG_PATH = Path(__file__).with_name("config.yaml")
+
+
+def _load_config() -> dict:
+    with CONFIG_PATH.open("r", encoding="utf-8") as handle:
+        return yaml.safe_load(handle) or {}
+
+
+def _query(promql: str) -> list[dict]:
+    vm_url = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
+    response = requests.get(f"{vm_url}/api/v1/query", params={"query": promql}, timeout=10)
+    response.raise_for_status()
+    payload = response.json()
+    return payload.get("data", {}).get("result", [])
+
+
+def _expected_tasks() -> list[dict]:
+    cfg = _load_config()
+    tasks = [
+        _normalize_task(item, cfg)
+        for item in cfg.get("ariadne_schedule_tasks", [])
+    ]
+    assert tasks, "No Ariadne schedule tasks configured"
+    return tasks
+
+
+def _normalize_task(item: object, cfg: dict) -> dict:
+    if isinstance(item, str):
+        return {
+            "task": item,
+            "check_last_success": True,
+            "max_success_age_hours": cfg.get("max_success_age_hours", 48),
+        }
+    if isinstance(item, dict):
+        normalized = dict(item)
+        normalized.setdefault("check_last_success", True)
+        normalized.setdefault("max_success_age_hours", cfg.get("max_success_age_hours", 48))
+        return normalized
+    raise TypeError(f"Unsupported Ariadne schedule task config entry: {item!r}")
+
+
+def _tracked_tasks(tasks: list[dict]) -> list[dict]:
+    tracked = [item for item in tasks if item.get("check_last_success")]
+    assert tracked, "No Ariadne schedule tasks are marked for success tracking"
+    return tracked
+
+
+def _task_regex(tasks: list[dict]) -> str:
+    return "|".join(item["task"] for item in tasks)
+
+
+def test_ariadne_schedule_series_exist():
+    tasks = _expected_tasks()
+    selector = _task_regex(tasks)
+    series = _query(f'ariadne_schedule_next_run_timestamp_seconds{{task=~"{selector}"}}')
+    seen = {item.get("metric", {}).get("task") for item in series}
+    missing = [item["task"] for item in tasks if item["task"] not in seen]
+    assert not missing, f"Missing next-run metrics for: {', '.join(missing)}"
+
+
+def test_ariadne_schedule_recent_success():
+    tasks = _tracked_tasks(_expected_tasks())
+    selector = _task_regex(tasks)
+    series = _query(f'ariadne_schedule_last_success_timestamp_seconds{{task=~"{selector}"}}')
+    seen = {item.get("metric", {}).get("task") for item in series}
+    missing = [item["task"] for item in tasks if item["task"] not in seen]
+    assert not missing, f"Missing last-success metrics for: {', '.join(missing)}"
+
+    now = datetime.now(timezone.utc)
+    age_by_task = {
+        item.get("metric", {}).get("task"): (now - datetime.fromtimestamp(float(item["value"][1]), tz=timezone.utc)).total_seconds() / 3600
+        for item in series
+    }
+    too_old = [
+        f"{task} ({age_by_task[task]:.1f}h > {item['max_success_age_hours']}h)"
+        for item in tasks
+        if (task := item["task"]) in age_by_task and age_by_task[task] > float(item["max_success_age_hours"])
+    ]
+    assert not too_old, "Ariadne schedules are stale: " + ", ".join(too_old)
+
+
+def test_ariadne_schedule_last_status_present_and_boolean():
+    tasks = _tracked_tasks(_expected_tasks())
+    selector = _task_regex(tasks)
+    series = _query(f'ariadne_schedule_last_status{{task=~"{selector}"}}')
+    seen = {item.get("metric", {}).get("task") for item in series}
+    missing = [item["task"] for item in tasks if item["task"] not in seen]
+    assert not missing, f"Missing last-status metrics for: {', '.join(missing)}"
+
+    invalid = []
+    for item in series:
+        task = item.get("metric", {}).get("task")
+        value = float(item["value"][1])
+        if value not in (0.0, 1.0):
+            invalid.append(f"{task}={value}")
+    assert not invalid, f"Unexpected Ariadne last-status values: {', '.join(invalid)}"

ci/tests/glue/test_glue_metrics.py

@@ -1,3 +1,5 @@
+"""Glue checks for the metrics the quality-gate publishes."""
+
 from __future__ import annotations
 
 import os
@@ -23,26 +25,63 @@ def _query(promql: str) -> list[dict]:
     return payload.get("data", {}).get("result", [])
 
 
-def test_glue_metrics_present():
+def _expected_tasks() -> list[dict]:
-    series = _query('kube_cronjob_labels{label_atlas_bstein_dev_glue="true"}')
+    cfg = _load_config()
-    assert series, "No glue cronjob label series found"
+    tasks = [
+        _normalize_task(item, cfg)
+        for item in cfg.get("ariadne_schedule_tasks", [])
+    ]
+    assert tasks, "No Ariadne schedule tasks configured"
+    return tasks
 
 
-def test_glue_metrics_success_join():
+def _normalize_task(item: object, cfg: dict) -> dict:
-    query = (
+    if isinstance(item, str):
-        "kube_cronjob_status_last_successful_time "
+        return {
-        'and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue="true"}'
+            "task": item,
-    )
+            "check_last_success": True,
-    series = _query(query)
+            "max_success_age_hours": cfg.get("max_success_age_hours", 48),
-    assert series, "No glue cronjob last success series found"
+        }
+    if isinstance(item, dict):
+        normalized = dict(item)
+        normalized.setdefault("check_last_success", True)
+        normalized.setdefault("max_success_age_hours", cfg.get("max_success_age_hours", 48))
+        return normalized
+    raise TypeError(f"Unsupported Ariadne schedule task config entry: {item!r}")
+
+
+def _tracked_tasks(tasks: list[dict]) -> list[dict]:
+    tracked = [item for item in tasks if item.get("check_last_success")]
+    assert tracked, "No Ariadne schedule tasks are marked for success tracking"
+    return tracked
+
+
+def _task_regex(tasks: list[dict]) -> str:
+    return "|".join(item["task"] for item in tasks)
 
 
 def test_ariadne_schedule_metrics_present():
-    cfg = _load_config()
+    tasks = _expected_tasks()
-    expected = cfg.get("ariadne_schedule_tasks", [])
+    selector = _task_regex(tasks)
-    if not expected:
+    series = _query(f'ariadne_schedule_next_run_timestamp_seconds{{task=~"{selector}"}}')
-        return
+    seen = {item.get("metric", {}).get("task") for item in series}
-    series = _query("ariadne_schedule_next_run_timestamp_seconds")
+    missing = [item["task"] for item in tasks if item["task"] not in seen]
-    tasks = {item.get("metric", {}).get("task") for item in series}
-    missing = [task for task in expected if task not in tasks]
     assert not missing, f"Missing Ariadne schedule metrics for: {', '.join(missing)}"
+
+
+def test_ariadne_schedule_success_and_status_metrics_present():
+    tasks = _tracked_tasks(_expected_tasks())
+    selector = _task_regex(tasks)
+
+    success = _query(f'ariadne_schedule_last_success_timestamp_seconds{{task=~"{selector}"}}')
+    status = _query(f'ariadne_schedule_last_status{{task=~"{selector}"}}')
+
+    success_tasks = {item.get("metric", {}).get("task") for item in success}
+    status_tasks = {item.get("metric", {}).get("task") for item in status}
+    expected = {item["task"] for item in tasks}
+
+    missing_success = sorted(expected - success_tasks)
+    missing_status = sorted(expected - status_tasks)
+
+    assert not missing_success, f"Missing Ariadne success metrics for: {', '.join(missing_success)}"
+    assert not missing_status, f"Missing Ariadne status metrics for: {', '.join(missing_status)}"

ci/titan-iac-trivy-waivers.json

@@ -0,0 +1,407 @@
+{
+  "version": 1,
+  "generated_from": "Jenkins titan-iac build 225 Trivy filesystem scan",
+  "default_expires_at": "2026-05-22",
+  "ticket": "atlas-quality-wave-k8s-hardening",
+  "default_reason": "Existing Kubernetes manifest hardening baseline accepted only for the first quality-gate rollout; fix or renew explicitly before expiry.",
+  "misconfigurations": [
+    {
+      "id": "DS-0002",
+      "targets": [
+        "dockerfiles/Dockerfile.ananke-node-helper"
+      ]
+    },
+    {
+      "id": "KSV-0009",
+      "targets": [
+        "services/mailu/vip-controller.yaml",
+        "services/maintenance/k3s-agent-restart-daemonset.yaml"
+      ]
+    },
+    {
+      "id": "KSV-0010",
+      "targets": [
+        "services/maintenance/k3s-agent-restart-daemonset.yaml",
+        "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
+        "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
+        "services/monitoring/jetson-tegrastats-exporter.yaml"
+      ]
+    },
+    {
+      "id": "KSV-0014",
+      "targets": [
+        "infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml",
+        "infrastructure/core/node-prefer-noschedule-cronjob.yaml",
+        "infrastructure/core/ntp-sync-daemonset.yaml",
+        "infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml",
+        "infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml",
+        "infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml",
+        "infrastructure/longhorn/core/vault-sync-deployment.yaml",
+        "infrastructure/longhorn/ui-ingress/oauth2-proxy-longhorn.yaml",
+        "infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
+        "infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
+        "infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
+        "infrastructure/postgres/statefulset.yaml",
+        "infrastructure/vault-csi/vault-csi-provider.yaml",
+        "services/ai-llm/deployment.yaml",
+        "services/bstein-dev-home/backend-deployment.yaml",
+        "services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
+        "services/bstein-dev-home/frontend-deployment.yaml",
+        "services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml",
+        "services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml",
+        "services/bstein-dev-home/vault-sync-deployment.yaml",
+        "services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
+        "services/comms/atlasbot-deployment.yaml",
+        "services/comms/coturn.yaml",
+        "services/comms/element-call-deployment.yaml",
+        "services/comms/guest-name-job.yaml",
+        "services/comms/guest-register-deployment.yaml",
+        "services/comms/livekit-token-deployment.yaml",
+        "services/comms/livekit.yaml",
+        "services/comms/mas-deployment.yaml",
+        "services/comms/oneoffs/bstein-force-leave-job.yaml",
+        "services/comms/oneoffs/comms-secrets-ensure-job.yaml",
+        "services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml",
+        "services/comms/oneoffs/mas-db-ensure-job.yaml",
+        "services/comms/oneoffs/mas-local-users-ensure-job.yaml",
+        "services/comms/oneoffs/othrys-kick-numeric-job.yaml",
+        "services/comms/oneoffs/synapse-admin-ensure-job.yaml",
+        "services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml",
+        "services/comms/oneoffs/synapse-signingkey-ensure-job.yaml",
+        "services/comms/oneoffs/synapse-user-seed-job.yaml",
+        "services/comms/pin-othrys-job.yaml",
+        "services/comms/reset-othrys-room-job.yaml",
+        "services/comms/seed-othrys-room.yaml",
+        "services/comms/vault-sync-deployment.yaml",
+        "services/comms/wellknown.yaml",
+        "services/crypto/monerod/deployment.yaml",
+        "services/crypto/wallet-monero-temp/deployment.yaml",
+        "services/crypto/xmr-miner/deployment.yaml",
+        "services/crypto/xmr-miner/vault-sync-deployment.yaml",
+        "services/crypto/xmr-miner/xmrig-daemonset.yaml",
+        "services/finance/actual-budget-deployment.yaml",
+        "services/finance/firefly-cronjob.yaml",
+        "services/finance/firefly-deployment.yaml",
+        "services/finance/firefly-user-sync-cronjob.yaml",
+        "services/finance/oneoffs/finance-secrets-ensure-job.yaml",
+        "services/gitea/deployment.yaml",
+        "services/harbor/vault-sync-deployment.yaml",
+        "services/health/wger-admin-ensure-cronjob.yaml",
+        "services/health/wger-deployment.yaml",
+        "services/health/wger-user-sync-cronjob.yaml",
+        "services/jellyfin/deployment.yaml",
+        "services/jellyfin/loader.yaml",
+        "services/jenkins/deployment.yaml",
+        "services/jenkins/vault-sync-deployment.yaml",
+        "services/keycloak/deployment.yaml",
+        "services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/ldap-federation-job.yaml",
+        "services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/mas-secrets-ensure-job.yaml",
+        "services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/portal-e2e-client-job.yaml",
+        "services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml",
+        "services/keycloak/oneoffs/portal-e2e-target-client-job.yaml",
+        "services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml",
+        "services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml",
+        "services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/realm-settings-job.yaml",
+        "services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/user-overrides-job.yaml",
+        "services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml",
+        "services/keycloak/vault-sync-deployment.yaml",
+        "services/logging/node-image-gc-rpi4-daemonset.yaml",
+        "services/logging/node-image-prune-rpi5-daemonset.yaml",
+        "services/logging/node-log-rotation-daemonset.yaml",
+        "services/logging/oauth2-proxy.yaml",
+        "services/logging/oneoffs/opensearch-dashboards-setup-job.yaml",
+        "services/logging/oneoffs/opensearch-ism-job.yaml",
+        "services/logging/oneoffs/opensearch-observability-setup-job.yaml",
+        "services/logging/opensearch-prune-cronjob.yaml",
+        "services/logging/vault-sync-deployment.yaml",
+        "services/mailu/mailu-sync-cronjob.yaml",
+        "services/mailu/mailu-sync-listener.yaml",
+        "services/mailu/oneoffs/mailu-sync-job.yaml",
+        "services/mailu/vault-sync-deployment.yaml",
+        "services/mailu/vip-controller.yaml",
+        "services/maintenance/ariadne-deployment.yaml",
+        "services/maintenance/disable-k3s-traefik-daemonset.yaml",
+        "services/maintenance/image-sweeper-cronjob.yaml",
+        "services/maintenance/k3s-agent-restart-daemonset.yaml",
+        "services/maintenance/metis-deployment.yaml",
+        "services/maintenance/metis-k3s-token-sync-cronjob.yaml",
+        "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
+        "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
+        "services/maintenance/node-image-sweeper-daemonset.yaml",
+        "services/maintenance/node-nofile-daemonset.yaml",
+        "services/maintenance/oauth2-proxy-metis.yaml",
+        "services/maintenance/oauth2-proxy-soteria.yaml",
+        "services/maintenance/oneoffs/ariadne-migrate-job.yaml",
+        "services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml",
+        "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
+        "services/maintenance/pod-cleaner-cronjob.yaml",
+        "services/maintenance/soteria-deployment.yaml",
+        "services/maintenance/vault-sync-deployment.yaml",
+        "services/monitoring/dcgm-exporter.yaml",
+        "services/monitoring/jetson-tegrastats-exporter.yaml",
+        "services/monitoring/oneoffs/grafana-org-bootstrap.yaml",
+        "services/monitoring/oneoffs/grafana-user-dedupe-job.yaml",
+        "services/monitoring/platform-quality-gateway-deployment.yaml",
+        "services/monitoring/platform-quality-suite-probe-cronjob.yaml",
+        "services/monitoring/postmark-exporter-deployment.yaml",
+        "services/monitoring/vmalert-atlas-availability.yaml",
+        "services/monitoring/vault-sync-deployment.yaml",
+        "services/nextcloud-mail-sync/cronjob.yaml",
+        "services/nextcloud/collabora.yaml",
+        "services/nextcloud/cronjob.yaml",
+        "services/nextcloud/deployment.yaml",
+        "services/nextcloud/maintenance-cronjob.yaml",
+        "services/oauth2-proxy/deployment.yaml",
+        "services/openldap/statefulset.yaml",
+        "services/outline/deployment.yaml",
+        "services/outline/redis-deployment.yaml",
+        "services/pegasus/deployment.yaml",
+        "services/pegasus/vault-sync-deployment.yaml",
+        "services/planka/deployment.yaml",
+        "services/quality/oauth2-proxy-sonarqube.yaml",
+        "services/quality/sonarqube-deployment.yaml",
+        "services/quality/sonarqube-exporter-deployment.yaml",
+        "services/sui-metrics/base/deployment.yaml",
+        "services/typhon/vault-sync-deployment.yaml",
+        "services/vault/k8s-auth-config-cronjob.yaml",
+        "services/vault/oidc-config-cronjob.yaml",
+        "services/vault/statefulset.yaml",
+        "services/vaultwarden/deployment.yaml"
+      ]
+    },
+    {
+      "id": "KSV-0017",
+      "targets": [
+        "infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
+        "infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
+        "infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
+        "services/logging/node-image-gc-rpi4-daemonset.yaml",
+        "services/logging/node-image-prune-rpi5-daemonset.yaml",
+        "services/logging/node-log-rotation-daemonset.yaml",
+        "services/maintenance/disable-k3s-traefik-daemonset.yaml",
+        "services/maintenance/image-sweeper-cronjob.yaml",
+        "services/maintenance/k3s-agent-restart-daemonset.yaml",
+        "services/maintenance/metis-deployment.yaml",
+        "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
+        "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
+        "services/maintenance/node-image-sweeper-daemonset.yaml",
+        "services/maintenance/node-nofile-daemonset.yaml",
+        "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
+        "services/monitoring/dcgm-exporter.yaml",
+        "services/monitoring/jetson-tegrastats-exporter.yaml"
+      ]
+    },
+    {
+      "id": "KSV-0041",
+      "targets": [
+        "infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml",
+        "infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml",
+        "infrastructure/traefik/clusterrole.yaml",
+        "services/bstein-dev-home/rbac.yaml",
+        "services/comms/comms-secrets-ensure-rbac.yaml",
+        "services/comms/mas-db-ensure-rbac.yaml",
+        "services/comms/mas-secrets-ensure-rbac.yaml",
+        "services/maintenance/soteria-rbac.yaml"
+      ]
+    },
+    {
+      "id": "KSV-0047",
+      "targets": [
+        "services/monitoring/rbac.yaml"
+      ]
+    },
+    {
+      "id": "KSV-0053",
+      "targets": [
+        "services/comms/comms-secrets-ensure-rbac.yaml",
+        "services/comms/mas-db-ensure-rbac.yaml",
+        "services/jenkins/serviceaccount.yaml",
+        "services/maintenance/ariadne-rbac.yaml"
+      ]
+    },
+    {
+      "id": "KSV-0056",
+      "targets": [
+        "infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml",
+        "infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml",
+        "services/jenkins/serviceaccount.yaml",
+        "services/maintenance/disable-k3s-traefik-rbac.yaml",
+        "services/maintenance/k3s-traefik-cleanup-rbac.yaml"
+      ]
+    },
+    {
+      "id": "KSV-0114",
+      "targets": [
+        "infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml"
+      ]
+    },
+    {
+      "id": "KSV-0118",
+      "targets": [
+        "infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml",
+        "infrastructure/core/coredns-deployment.yaml",
+        "infrastructure/core/node-prefer-noschedule-cronjob.yaml",
+        "infrastructure/core/ntp-sync-daemonset.yaml",
+        "infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml",
+        "infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml",
+        "infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml",
+        "infrastructure/longhorn/core/vault-sync-deployment.yaml",
+        "infrastructure/longhorn/ui-ingress/oauth2-proxy-longhorn.yaml",
+        "infrastructure/modules/profiles/components/device-plugin-jetson/daemonset.yaml",
+        "infrastructure/modules/profiles/components/device-plugin-minipc/daemonset.yaml",
+        "infrastructure/modules/profiles/components/device-plugin-tethys/daemonset.yaml",
+        "infrastructure/postgres/statefulset.yaml",
+        "infrastructure/vault-csi/vault-csi-provider.yaml",
+        "services/ai-llm/deployment.yaml",
+        "services/bstein-dev-home/backend-deployment.yaml",
+        "services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
+        "services/bstein-dev-home/frontend-deployment.yaml",
+        "services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml",
+        "services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml",
+        "services/bstein-dev-home/vault-sync-deployment.yaml",
+        "services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
+        "services/comms/atlasbot-deployment.yaml",
+        "services/comms/coturn.yaml",
+        "services/comms/element-call-deployment.yaml",
+        "services/comms/guest-name-job.yaml",
+        "services/comms/livekit-token-deployment.yaml",
+        "services/comms/livekit.yaml",
+        "services/comms/mas-deployment.yaml",
+        "services/comms/oneoffs/bstein-force-leave-job.yaml",
+        "services/comms/oneoffs/comms-secrets-ensure-job.yaml",
+        "services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml",
+        "services/comms/oneoffs/mas-db-ensure-job.yaml",
+        "services/comms/oneoffs/mas-local-users-ensure-job.yaml",
+        "services/comms/oneoffs/othrys-kick-numeric-job.yaml",
+        "services/comms/oneoffs/synapse-admin-ensure-job.yaml",
+        "services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml",
+        "services/comms/oneoffs/synapse-signingkey-ensure-job.yaml",
+        "services/comms/oneoffs/synapse-user-seed-job.yaml",
+        "services/comms/pin-othrys-job.yaml",
+        "services/comms/reset-othrys-room-job.yaml",
+        "services/comms/seed-othrys-room.yaml",
+        "services/comms/vault-sync-deployment.yaml",
+        "services/comms/wellknown.yaml",
+        "services/crypto/monerod/deployment.yaml",
+        "services/crypto/wallet-monero-temp/deployment.yaml",
+        "services/crypto/xmr-miner/deployment.yaml",
+        "services/crypto/xmr-miner/vault-sync-deployment.yaml",
+        "services/crypto/xmr-miner/xmrig-daemonset.yaml",
+        "services/finance/firefly-cronjob.yaml",
+        "services/finance/firefly-deployment.yaml",
+        "services/finance/firefly-user-sync-cronjob.yaml",
+        "services/finance/oneoffs/finance-secrets-ensure-job.yaml",
+        "services/gitea/deployment.yaml",
+        "services/harbor/vault-sync-deployment.yaml",
+        "services/health/wger-admin-ensure-cronjob.yaml",
+        "services/health/wger-deployment.yaml",
+        "services/health/wger-user-sync-cronjob.yaml",
+        "services/jellyfin/loader.yaml",
+        "services/jenkins/deployment.yaml",
+        "services/jenkins/vault-sync-deployment.yaml",
+        "services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/ldap-federation-job.yaml",
+        "services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/mas-secrets-ensure-job.yaml",
+        "services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/portal-e2e-client-job.yaml",
+        "services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml",
+        "services/keycloak/oneoffs/portal-e2e-target-client-job.yaml",
+        "services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml",
+        "services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml",
+        "services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/realm-settings-job.yaml",
+        "services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml",
+        "services/keycloak/oneoffs/user-overrides-job.yaml",
+        "services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml",
+        "services/keycloak/vault-sync-deployment.yaml",
+        "services/logging/node-image-gc-rpi4-daemonset.yaml",
+        "services/logging/node-image-prune-rpi5-daemonset.yaml",
+        "services/logging/node-log-rotation-daemonset.yaml",
+        "services/logging/oauth2-proxy.yaml",
+        "services/logging/oneoffs/opensearch-dashboards-setup-job.yaml",
+        "services/logging/oneoffs/opensearch-ism-job.yaml",
+        "services/logging/oneoffs/opensearch-observability-setup-job.yaml",
+        "services/logging/opensearch-prune-cronjob.yaml",
+        "services/logging/vault-sync-deployment.yaml",
+        "services/mailu/mailu-sync-cronjob.yaml",
+        "services/mailu/mailu-sync-listener.yaml",
+        "services/mailu/oneoffs/mailu-sync-job.yaml",
+        "services/mailu/vault-sync-deployment.yaml",
+        "services/mailu/vip-controller.yaml",
+        "services/maintenance/ariadne-deployment.yaml",
+        "services/maintenance/disable-k3s-traefik-daemonset.yaml",
+        "services/maintenance/image-sweeper-cronjob.yaml",
+        "services/maintenance/k3s-agent-restart-daemonset.yaml",
+        "services/maintenance/metis-deployment.yaml",
+        "services/maintenance/metis-k3s-token-sync-cronjob.yaml",
+        "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
+        "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
+        "services/maintenance/node-image-sweeper-daemonset.yaml",
+        "services/maintenance/node-nofile-daemonset.yaml",
+        "services/maintenance/oauth2-proxy-metis.yaml",
+        "services/maintenance/oauth2-proxy-soteria.yaml",
+        "services/maintenance/oneoffs/ariadne-migrate-job.yaml",
+        "services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml",
+        "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
+        "services/maintenance/pod-cleaner-cronjob.yaml",
+        "services/maintenance/soteria-deployment.yaml",
+        "services/maintenance/vault-sync-deployment.yaml",
+        "services/monitoring/dcgm-exporter.yaml",
+        "services/monitoring/jetson-tegrastats-exporter.yaml",
+        "services/monitoring/oneoffs/grafana-org-bootstrap.yaml",
+        "services/monitoring/oneoffs/grafana-user-dedupe-job.yaml",
+        "services/monitoring/platform-quality-gateway-deployment.yaml",
+        "services/monitoring/platform-quality-suite-probe-cronjob.yaml",
+        "services/monitoring/postmark-exporter-deployment.yaml",
+        "services/monitoring/vmalert-atlas-availability.yaml",
+        "services/monitoring/vault-sync-deployment.yaml",
+        "services/nextcloud/collabora.yaml",
+        "services/oauth2-proxy/deployment.yaml",
+        "services/openldap/statefulset.yaml",
+        "services/outline/deployment.yaml",
+        "services/outline/redis-deployment.yaml",
+        "services/pegasus/vault-sync-deployment.yaml",
+        "services/quality/oauth2-proxy-sonarqube.yaml",
+        "services/quality/sonarqube-deployment.yaml",
+        "services/quality/sonarqube-exporter-deployment.yaml",
+        "services/sui-metrics/base/deployment.yaml",
+        "services/sui-metrics/overlays/atlas/patch-node-selector.yaml",
+        "services/typhon/deployment.yaml",
+        "services/typhon/vault-sync-deployment.yaml",
+        "services/vault/k8s-auth-config-cronjob.yaml",
+        "services/vault/oidc-config-cronjob.yaml",
+        "services/vaultwarden/deployment.yaml"
+      ]
+    },
+    {
+      "id": "KSV-0121",
+      "targets": [
+        "services/logging/node-image-gc-rpi4-daemonset.yaml",
+        "services/logging/node-image-prune-rpi5-daemonset.yaml",
+        "services/logging/node-log-rotation-daemonset.yaml",
+        "services/maintenance/disable-k3s-traefik-daemonset.yaml",
+        "services/maintenance/image-sweeper-cronjob.yaml",
+        "services/maintenance/metis-deployment.yaml",
+        "services/maintenance/node-image-sweeper-daemonset.yaml",
+        "services/maintenance/node-nofile-daemonset.yaml",
+        "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml"
+      ]
+    }
+  ]
+}

clusters/atlas/flux-system/applications/ai-llm/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: ai-llm
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/ai-llm

clusters/atlas/flux-system/applications/bstein-dev-home-migrations/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: bstein-dev-home-migrations
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/bstein-dev-home/oneoffs/migrations

clusters/atlas/flux-system/applications/bstein-dev-home/image-automation.yaml

@@ -13,14 +13,14 @@ spec:
   git:
     checkout:
       ref:
-        branch: feature/ariadne
+        branch: main
     commit:
       author:
         email: ops@bstein.dev
         name: flux-bot
       messageTemplate: "chore(bstein-dev-home): automated image update"
     push:
-      branch: feature/ariadne
+      branch: main
   update:
     strategy: Setters
     path: services/bstein-dev-home

clusters/atlas/flux-system/applications/bstein-dev-home/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: bstein-dev-home
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/bstein-dev-home

clusters/atlas/flux-system/applications/comms/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: comms
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   prune: true
@@ -13,5 +15,3 @@ spec:
   path: ./services/comms
   targetNamespace: comms
   timeout: 2m
-  dependsOn:
-    - name: traefik

clusters/atlas/flux-system/applications/crypto/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: crypto
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/crypto

clusters/atlas/flux-system/applications/finance/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: finance
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/finance

clusters/atlas/flux-system/applications/game-stream/kustomization.yaml

@@ -0,0 +1,29 @@
+# clusters/atlas/flux-system/applications/game-stream/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: game-stream
+  namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+spec:
+  interval: 10m
+  path: ./services/game-stream
+  targetNamespace: game-stream
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  dependsOn:
+    - name: cert-manager
+    - name: keycloak
+    - name: traefik
+    - name: vault
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: oauth2-proxy-wolf
+      namespace: game-stream
+  wait: false
+  timeout: 10m

clusters/atlas/flux-system/applications/gitea/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: gitea
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/gitea

clusters/atlas/flux-system/applications/harbor/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: harbor
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/harbor

clusters/atlas/flux-system/applications/health/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: health
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/health
@@ -15,7 +17,6 @@ spec:
   dependsOn:
     - name: keycloak
     - name: postgres
-    - name: traefik
     - name: vault
   healthChecks:
     - apiVersion: apps/v1

clusters/atlas/flux-system/applications/jellyfin/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: jellyfin
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/jellyfin

clusters/atlas/flux-system/applications/jenkins/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: jenkins
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/jenkins
@@ -14,7 +16,6 @@ spec:
   targetNamespace: jenkins
   dependsOn:
     - name: helm
-    - name: traefik
   healthChecks:
     - apiVersion: apps/v1
       kind: Deployment

clusters/atlas/flux-system/applications/keycloak/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: keycloak
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   prune: true

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -21,10 +21,15 @@ resources:
   - sui-metrics/kustomization.yaml
   - openldap/kustomization.yaml
   - keycloak/kustomization.yaml
+  - quality/kustomization.yaml
   - oauth2-proxy/kustomization.yaml
   - mailu/kustomization.yaml
   - jenkins/kustomization.yaml
   - ai-llm/kustomization.yaml
+  - openclaw/kustomization.yaml
+  - game-stream/kustomization.yaml
+  - veles/kustomization.yaml
+  - typhon/kustomization.yaml
   - nextcloud/kustomization.yaml
   - nextcloud-mail-sync/kustomization.yaml
   - outline/kustomization.yaml

clusters/atlas/flux-system/applications/mailu/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: mailu
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   sourceRef:

clusters/atlas/flux-system/applications/monerod/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: monerod
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/crypto/monerod

clusters/atlas/flux-system/applications/nextcloud-mail-sync/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: nextcloud-mail-sync
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   prune: true

clusters/atlas/flux-system/applications/nextcloud/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: nextcloud
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/nextcloud

clusters/atlas/flux-system/applications/oauth2-proxy/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: oauth2-proxy
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   prune: true

clusters/atlas/flux-system/applications/openclaw/kustomization.yaml

@@ -0,0 +1,34 @@
+# clusters/atlas/flux-system/applications/openclaw/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: openclaw
+  namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+spec:
+  interval: 10m
+  path: ./services/openclaw
+  targetNamespace: openclaw
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true
+  timeout: 30m
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: openclaw-ollama
+      namespace: openclaw
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: openclaw
+      namespace: openclaw
+  dependsOn:
+    - name: cert-manager
+    - name: core
+    - name: longhorn
+    - name: traefik
+

clusters/atlas/flux-system/applications/openldap/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: openldap
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   prune: true

clusters/atlas/flux-system/applications/outline/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: outline
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/outline
@@ -15,7 +17,6 @@ spec:
   dependsOn:
     - name: keycloak
     - name: mailu
-    - name: traefik
   healthChecks:
     - apiVersion: apps/v1
       kind: Deployment

clusters/atlas/flux-system/applications/pegasus/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: pegasus
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/pegasus

clusters/atlas/flux-system/applications/planka/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: planka
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/planka
@@ -15,7 +17,6 @@ spec:
   dependsOn:
     - name: keycloak
     - name: mailu
-    - name: traefik
   healthChecks:
     - apiVersion: apps/v1
       kind: Deployment

clusters/atlas/flux-system/applications/quality/kustomization.yaml

@@ -0,0 +1,36 @@
+# clusters/atlas/flux-system/applications/quality/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: quality
+  namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+spec:
+  interval: 10m
+  path: ./services/quality
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: quality
+  dependsOn:
+    - name: cert-manager
+    - name: keycloak
+    - name: vault
+    - name: postgres
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: sonarqube
+      namespace: quality
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: sonarqube-exporter
+      namespace: quality
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: oauth2-proxy-sonarqube
+      namespace: quality
+  wait: false
+  timeout: 20m

clusters/atlas/flux-system/applications/sui-metrics/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: sui-metrics
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/sui-metrics/overlays/atlas

clusters/atlas/flux-system/applications/typhon/kustomization.yaml

@@ -0,0 +1,31 @@
+# clusters/atlas/flux-system/applications/typhon/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: typhon
+  namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+spec:
+  interval: 10m
+  path: ./services/typhon
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: climate
+  dependsOn:
+    - name: vault
+    - name: vault-csi
+    - name: monitoring
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: typhon
+      namespace: climate
+    - apiVersion: v1
+      kind: Service
+      name: typhon
+      namespace: climate
+  wait: false
+  timeout: 20m

clusters/atlas/flux-system/applications/vault/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: vault
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   sourceRef:

clusters/atlas/flux-system/applications/vaultwarden/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: vaultwarden
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   suspend: false
@@ -17,4 +19,3 @@ spec:
   wait: true
   dependsOn:
     - name: helm
-    - name: traefik

clusters/atlas/flux-system/applications/veles/image-automation.yaml

@@ -0,0 +1,29 @@
+# clusters/atlas/flux-system/applications/veles/image-automation.yaml
+# Staged for the first Veles image rollout. Add this file to the parent
+# applications kustomization after the namespace exists and the Harbor repos
+# have initial tags.
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageUpdateAutomation
+metadata:
+  name: veles
+  namespace: veles
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  git:
+    checkout:
+      ref:
+        branch: main
+    commit:
+      author:
+        email: ops@bstein.dev
+        name: flux-bot
+      messageTemplate: "chore(veles): automated image update"
+    push:
+      branch: main
+  update:
+    strategy: Setters
+    path: services/veles

clusters/atlas/flux-system/applications/veles/kustomization.yaml

@@ -0,0 +1,28 @@
+# clusters/atlas/flux-system/applications/veles/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: veles
+  namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+spec:
+  interval: 10m
+  path: ./services/veles
+  targetNamespace: veles
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  dependsOn:
+    - name: cert-manager
+    - name: core
+    - name: keycloak
+    - name: longhorn
+    - name: traefik
+    - name: vault
+    - name: vault-csi
+    - name: vault-injector
+  wait: false
+  timeout: 20m

clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: wallet-monero-temp
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/crypto/wallet-monero-temp

clusters/atlas/flux-system/applications/xmr-miner/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: xmr-miner
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/crypto/xmr-miner

clusters/atlas/flux-system/gotk-components.yaml

@@ -5966,6 +5966,9 @@ spec:
       - args:
         - --events-addr=http://notification-controller.$(RUNTIME_NAMESPACE).svc.cluster.local./
         - --watch-all-namespaces=true
+        - --concurrent=1
+        - --requeue-dependency=5s
+        - --interval-jitter-percentage=30
         - --log-level=info
         - --log-encoding=json
         - --enable-leader-election

clusters/atlas/flux-system/gotk-sync.yaml

@@ -7,7 +7,7 @@ metadata:
   name: flux-system
   namespace: flux-system
 spec:
-  interval: 1m0s
+  interval: 15m0s
   ref:
     branch: main
   secretRef:
@@ -20,7 +20,7 @@ metadata:
   name: flux-system
   namespace: flux-system
 spec:
-  interval: 10m0s
+  interval: 1h0m0s
   path: ./clusters/atlas/flux-system
   prune: true
   sourceRef:

clusters/atlas/flux-system/platform/cert-manager-cleanup/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: cert-manager-cleanup
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 30m
   path: ./infrastructure/cert-manager/cleanup

clusters/atlas/flux-system/platform/cert-manager/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: cert-manager
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 30m
   path: ./infrastructure/cert-manager

clusters/atlas/flux-system/platform/core/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: core
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./infrastructure/core

clusters/atlas/flux-system/platform/descheduler/kustomization.yaml

@@ -0,0 +1,21 @@
+# clusters/atlas/flux-system/platform/descheduler/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: descheduler
+  namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+spec:
+  interval: 30m
+  path: ./infrastructure/descheduler
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: kube-system
+  dependsOn:
+    - name: helm
+    - name: core
+  wait: true

clusters/atlas/flux-system/platform/gitops-ui/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: gitops-ui
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   timeout: 10m
@@ -16,5 +18,4 @@ spec:
   targetNamespace: flux-system
   dependsOn:
     - name: helm
-    - name: traefik
   wait: true

clusters/atlas/flux-system/platform/helm/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: helm
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 30m
   sourceRef:

clusters/atlas/flux-system/platform/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 resources:
   - core/kustomization.yaml
   - helm/kustomization.yaml
+  - descheduler/kustomization.yaml
+  - resource-guardrails/kustomization.yaml
   - cert-manager/kustomization.yaml
   - metallb/kustomization.yaml
   - traefik/kustomization.yaml

clusters/atlas/flux-system/platform/logging/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: logging
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/logging

clusters/atlas/flux-system/platform/longhorn-adopt/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: longhorn-adopt
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 30m
   path: ./infrastructure/longhorn/adopt

clusters/atlas/flux-system/platform/longhorn-ui/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: longhorn-ui
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./infrastructure/longhorn/ui-ingress

clusters/atlas/flux-system/platform/longhorn/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: longhorn
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 30m
   path: ./infrastructure/longhorn/core

clusters/atlas/flux-system/platform/maintenance/image-automation.yaml

@@ -13,14 +13,14 @@ spec:
   git:
     checkout:
       ref:
-        branch: feature/ariadne
+        branch: main
     commit:
       author:
         email: ops@bstein.dev
         name: flux-bot
       messageTemplate: "chore(maintenance): automated image update"
     push:
-      branch: feature/ariadne
+      branch: main
   update:
     strategy: Setters
     path: services/maintenance

clusters/atlas/flux-system/platform/maintenance/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: maintenance
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/maintenance

clusters/atlas/flux-system/platform/metallb/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: metallb
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 30m
   sourceRef:

clusters/atlas/flux-system/platform/monitoring/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: monitoring
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./services/monitoring

clusters/atlas/flux-system/platform/postgres/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: postgres
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./infrastructure/postgres

clusters/atlas/flux-system/platform/resource-guardrails/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/platform/resource-guardrails/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: resource-guardrails
+  namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
+spec:
+  interval: 10m
+  path: ./infrastructure/resource-guardrails
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  dependsOn:
+    - name: core
+  wait: true

clusters/atlas/flux-system/platform/traefik/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: traefik
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 10m
   path: ./infrastructure/traefik

clusters/atlas/flux-system/platform/vault-csi/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: vault-csi
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 30m
   sourceRef:

clusters/atlas/flux-system/platform/vault-injector/kustomization.yaml

@@ -4,6 +4,8 @@ kind: Kustomization
 metadata:
   name: vault-injector
   namespace: flux-system
+  annotations:
+    kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
 spec:
   interval: 30m
   path: ./infrastructure/vault-injector

dockerfiles/Dockerfile.comms-guest-tools

@@ -2,4 +2,8 @@ FROM python:3.11-slim
 
 ENV PIP_DISABLE_PIP_VERSION_CHECK=1
 
-RUN pip install --no-cache-dir requests psycopg2-binary
+RUN pip install --no-cache-dir requests psycopg2-binary \
+    && groupadd --system guest-tools \
+    && useradd --system --uid 65532 --gid guest-tools --home-dir /nonexistent --shell /usr/sbin/nologin guest-tools
+
+USER guest-tools

dockerfiles/Dockerfile.data-prepper

@@ -1,15 +1,12 @@
-FROM --platform=$BUILDPLATFORM opensearchproject/data-prepper:2.8.0 AS source
+# Use the mirrored Harbor artifact so CI does not depend on Docker Hub egress.
-
+FROM registry.bstein.dev/streaming/data-prepper@sha256:32ac6ad42e0f12da08bebee307e290b17d127b30def9b06eeaffbcbbc5033e83
-FROM --platform=$TARGETPLATFORM eclipse-temurin:17-jre
 
 ENV DATA_PREPPER_PATH=/usr/share/data-prepper
 
-RUN useradd -u 10001 -M -U -d / -s /usr/sbin/nologin data_prepper \
+USER root
-  && mkdir -p /var/log/data-prepper
+RUN apt-get update \
-
+    && apt-get install -y --no-install-recommends bc \
-COPY --from=source /usr/share/data-prepper /usr/share/data-prepper
+    && rm -rf /var/lib/apt/lists/*
-
-RUN chown -R 10001:10001 /usr/share/data-prepper /var/log/data-prepper
 
 USER 10001
 WORKDIR /usr/share/data-prepper

dockerfiles/Dockerfile.livekit-token-vault

@@ -1,10 +1,13 @@
 FROM ghcr.io/element-hq/lk-jwt-service:0.3.0 AS base
 
 FROM alpine:3.20
-RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache ca-certificates \
+    && addgroup -S livekit-token \
+    && adduser -S -D -H -u 65532 -G livekit-token livekit-token
 COPY --from=base /lk-jwt-service /lk-jwt-service
 COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
 RUN chmod 0755 /entrypoint.sh
 
+USER livekit-token
 ENTRYPOINT ["/entrypoint.sh"]
 CMD ["/lk-jwt-service"]

dockerfiles/Dockerfile.monero-p2pool

@@ -29,10 +29,12 @@ FROM ${DEBIAN_IMAGE}
 RUN set -eux; \
     apt-get update; \
     apt-get install -y --no-install-recommends ca-certificates; \
-    update-ca-certificates; rm -rf /var/lib/apt/lists/*
+    update-ca-certificates; rm -rf /var/lib/apt/lists/*; \
+    groupadd --system p2pool; \
+    useradd --system --uid 65532 --gid p2pool --home-dir /nonexistent --shell /usr/sbin/nologin p2pool
 COPY --from=fetch /out/p2pool /usr/local/bin/p2pool
 
 RUN /usr/local/bin/p2pool --version || true
 EXPOSE 3333
+USER p2pool
 ENTRYPOINT ["/usr/local/bin/p2pool"]
-

dockerfiles/Dockerfile.monero-wallet-rpc

@@ -26,9 +26,12 @@ RUN set -eux; \
     curl -fsSL "$URL" -o /opt/monero/monero.tar.bz2; \
     tar -xjf /opt/monero/monero.tar.bz2 -C /opt/monero --strip-components=1; \
     install -m 0755 /opt/monero/monero-wallet-rpc /usr/local/bin/monero-wallet-rpc; \
-    rm -f /opt/monero/monero.tar.bz2
+    rm -f /opt/monero/monero.tar.bz2; \
+    groupadd --system monero; \
+    useradd --system --uid 1000 --gid monero --home-dir /nonexistent --shell /usr/sbin/nologin monero
 
 ENV PATH="/usr/local/bin:/usr/bin:/bin"
 RUN /usr/local/bin/monero-wallet-rpc --version || true
 
 EXPOSE 18083
+USER monero

dockerfiles/Dockerfile.monerod

@@ -23,10 +23,14 @@ RUN set -eux; \
     mkdir -p /opt/monero; \
     tar -xjf /tmp/monero.tar.bz2 -C /opt/monero --strip-components=1; \
     rm -f /tmp/monero.tar.bz2; \
+    groupadd --system monero; \
+    useradd --system --uid 1000 --gid monero --home-dir /nonexistent --shell /usr/sbin/nologin monero; \
     mkdir -p /data; \
+    chown monero:monero /data; \
     chmod 0770 /data
 
 ENV LD_LIBRARY_PATH=/opt/monero:/opt/monero/lib \
     PATH="/opt/monero:${PATH}"
 
+USER monero
 CMD ["/opt/monero/monerod", "--version"]

dockerfiles/Dockerfile.oauth2-proxy-vault

@@ -1,10 +1,13 @@
 FROM quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 AS base
 
 FROM alpine:3.20
-RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache ca-certificates \
+    && addgroup -S oauth2-proxy \
+    && adduser -S -D -H -u 65532 -G oauth2-proxy oauth2-proxy
 COPY --from=base /bin/oauth2-proxy /bin/oauth2-proxy
 COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
 RUN chmod 0755 /entrypoint.sh
 
+USER oauth2-proxy
 ENTRYPOINT ["/entrypoint.sh"]
 CMD ["/bin/oauth2-proxy"]

dockerfiles/Dockerfile.pegasus-vault

@@ -1,10 +1,13 @@
 FROM registry.bstein.dev/streaming/pegasus:1.2.32 AS base
 
 FROM alpine:3.20
-RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache ca-certificates \
+    && addgroup -S pegasus \
+    && adduser -S -D -H -u 65532 -G pegasus pegasus
 COPY --from=base /pegasus /pegasus
 COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
 RUN chmod 0755 /entrypoint.sh
 
+USER pegasus
 ENTRYPOINT ["/entrypoint.sh"]
 CMD ["/pegasus"]

dockerfiles/Dockerfile.quality-tools

@@ -0,0 +1,48 @@
+# dockerfiles/Dockerfile.quality-tools
+FROM debian:bookworm-slim
+
+ARG SONAR_SCANNER_VERSION=8.0.1.6346
+ARG TRIVY_VERSION=0.70.0
+ENV TRIVY_CACHE_DIR=/opt/trivy-cache
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+    bash \
+    ca-certificates \
+    curl \
+    git \
+    jq \
+    unzip \
+  && rm -rf /var/lib/apt/lists/* \
+  && groupadd --system quality-tools \
+  && useradd --system --uid 65532 --gid quality-tools --home-dir /nonexistent --shell /usr/sbin/nologin quality-tools
+
+RUN set -eux; \
+  scanner_zip="sonar-scanner-cli-${SONAR_SCANNER_VERSION}-linux-aarch64.zip"; \
+  base_url="https://binaries.sonarsource.com/Distribution/sonar-scanner-cli"; \
+  curl -fsSL "${base_url}/${scanner_zip}" -o "/tmp/${scanner_zip}"; \
+  curl -fsSL "${base_url}/${scanner_zip}.sha256" -o "/tmp/${scanner_zip}.sha256"; \
+  printf '%s  %s\n' "$(cat "/tmp/${scanner_zip}.sha256")" "/tmp/${scanner_zip}" | sha256sum -c -; \
+  unzip -q "/tmp/${scanner_zip}" -d /opt; \
+  ln -s "/opt/sonar-scanner-${SONAR_SCANNER_VERSION}-linux-aarch64/bin/sonar-scanner" /usr/local/bin/sonar-scanner; \
+  rm -f "/tmp/${scanner_zip}" "/tmp/${scanner_zip}.sha256"
+
+RUN set -eux; \
+  trivy_tgz="trivy_${TRIVY_VERSION}_Linux-ARM64.tar.gz"; \
+  curl -fsSL "https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/${trivy_tgz}" -o "/tmp/${trivy_tgz}"; \
+  tar -C /usr/local/bin -xzf "/tmp/${trivy_tgz}" trivy; \
+  rm -f "/tmp/${trivy_tgz}"; \
+  trivy --version; \
+  sonar-scanner -v
+
+RUN set -eux; \
+  mkdir -p "${TRIVY_CACHE_DIR}"; \
+  trivy image --download-db-only --cache-dir "${TRIVY_CACHE_DIR}"; \
+  chmod -R a+rX "${TRIVY_CACHE_DIR}"; \
+  mkdir -p /workspace; \
+  chown quality-tools:quality-tools /workspace
+
+WORKDIR /workspace
+USER quality-tools

infrastructure/cert-manager/helmrelease.yaml

@@ -27,12 +27,53 @@ spec:
     timeout: 10m
   values:
     installCRDs: true
+    replicaCount: 2
+    podDisruptionBudget:
+      enabled: true
+      minAvailable: 1
     extraArgs:
       - --acme-http01-solver-nameservers=1.1.1.1:53,8.8.8.8:53
+    resources:
+      requests:
+        cpu: 50m
+        memory: 128Mi
+      limits:
+        cpu: 500m
+        memory: 512Mi
     nodeSelector:
       node-role.kubernetes.io/worker: "true"
     affinity:
       nodeAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+                - key: atlas.bstein.dev/spillover
+                  operator: DoesNotExist
+          - weight: 95
+            preference:
+              matchExpressions:
+                - key: kubernetes.io/hostname
+                  operator: NotIn
+                  values:
+                    - titan-13
+                    - titan-15
+                    - titan-17
+                    - titan-19
+          - weight: 90
+            preference:
+              matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                    - rpi5
+          - weight: 50
+            preference:
+              matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                    - rpi4
         requiredDuringSchedulingIgnoredDuringExecution:
           nodeSelectorTerms:
             - matchExpressions:
@@ -42,10 +83,63 @@ spec:
                     - rpi5
                     - rpi4
     webhook:
+      replicaCount: 2
+      podDisruptionBudget:
+        enabled: true
+        minAvailable: 1
+      resources:
+        requests:
+          cpu: 100m
+          memory: 128Mi
+        limits:
+          cpu: 500m
+          memory: 512Mi
+      livenessProbe:
+        failureThreshold: 8
+        initialDelaySeconds: 90
+        periodSeconds: 10
+        successThreshold: 1
+        timeoutSeconds: 5
+      readinessProbe:
+        failureThreshold: 8
+        initialDelaySeconds: 10
+        periodSeconds: 5
+        successThreshold: 1
+        timeoutSeconds: 5
       nodeSelector:
         node-role.kubernetes.io/worker: "true"
       affinity:
         nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: atlas.bstein.dev/spillover
+                    operator: DoesNotExist
+            - weight: 95
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-13
+                      - titan-15
+                      - titan-17
+                      - titan-19
+            - weight: 90
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+            - weight: 50
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi4
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
               - matchExpressions:
@@ -55,10 +149,51 @@ spec:
                       - rpi5
                       - rpi4
     cainjector:
+      replicaCount: 2
+      podDisruptionBudget:
+        enabled: true
+        minAvailable: 1
+      resources:
+        requests:
+          cpu: 50m
+          memory: 128Mi
+        limits:
+          cpu: 500m
+          memory: 512Mi
       nodeSelector:
         node-role.kubernetes.io/worker: "true"
       affinity:
         nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: atlas.bstein.dev/spillover
+                    operator: DoesNotExist
+            - weight: 95
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-13
+                      - titan-15
+                      - titan-17
+                      - titan-19
+            - weight: 90
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+            - weight: 50
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi4
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
               - matchExpressions:

infrastructure/core/coredns-custom.yaml

@@ -10,6 +10,7 @@ data:
       errors
       cache 30
       hosts {
+        192.168.22.9 agent.bstein.dev
         192.168.22.9 alerts.bstein.dev
         192.168.22.9 auth.bstein.dev
         192.168.22.9 bstein.dev
@@ -28,6 +29,7 @@ data:
         192.168.22.9 matrix.live.bstein.dev
         192.168.22.9 metrics.bstein.dev
         192.168.22.9 monero.bstein.dev
+        192.168.22.9 moonlight.bstein.dev
         10.43.6.87 money.bstein.dev
         192.168.22.9 notes.bstein.dev
         192.168.22.9 office.bstein.dev
@@ -40,6 +42,7 @@ data:
         192.168.22.9 secret.bstein.dev
         192.168.22.9 sso.bstein.dev
         192.168.22.9 stream.bstein.dev
+        192.168.22.9 wolf.bstein.dev
         192.168.22.9 tasks.bstein.dev
         192.168.22.9 vault.bstein.dev
         fallthrough

infrastructure/core/kustomization.yaml

@@ -4,8 +4,12 @@ kind: Kustomization
 resources:
   - ../modules/base
   - ../modules/profiles/atlas-ha
+  - node-prefer-noschedule-serviceaccount.yaml
+  - node-prefer-noschedule-rbac.yaml
+  - node-prefer-noschedule-cronjob.yaml
   - coredns-custom.yaml
   - coredns-deployment.yaml
   - ntp-sync-daemonset.yaml
+  - workload-profiles.yaml
   - ../sources/cert-manager/letsencrypt.yaml
   - ../sources/cert-manager/letsencrypt-prod.yaml

infrastructure/core/node-prefer-noschedule-cronjob.yaml

@@ -0,0 +1,80 @@
+# infrastructure/core/node-prefer-noschedule-cronjob.yaml
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: node-prefer-noschedule
+  namespace: kube-system
+spec:
+  schedule: "* * * * *"
+  concurrencyPolicy: Replace
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      template:
+        spec:
+          serviceAccountName: node-prefer-noschedule
+          restartPolicy: Never
+          containers:
+            - name: taint
+              image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+              command:
+                - /usr/bin/env
+                - bash
+                - -ceu
+                - |
+                  k() {
+                    kubectl --request-timeout=10s "$@"
+                  }
+
+                  clear_worker() {
+                    local node="${1}"
+                    local hardware="${2}"
+                    if k get node "${node}" >/dev/null 2>&1; then
+                      k label node "${node}" node-role.kubernetes.io/worker=true "hardware=${hardware}" --overwrite=true || true
+                      k label node "${node}" atlas.bstein.dev/spillover- || true
+                      k taint node "${node}" node.kubernetes.io/unschedulable:NoSchedule- || true
+                      k uncordon "${node}" || true
+                    else
+                      echo "skipping missing node ${node}"
+                    fi
+                  }
+
+                  clear_worker titan-04 rpi5
+                  clear_worker titan-05 rpi5
+                  clear_worker titan-07 rpi5
+                  clear_worker titan-08 rpi5
+                  clear_worker titan-11 rpi5
+                  clear_worker titan-12 rpi4
+                  clear_worker titan-14 rpi4
+                  clear_worker titan-18 rpi4
+                  clear_worker titan-22 amd64
+
+                  if k get node titan-22 >/dev/null 2>&1; then
+                    k label node titan-22 atlas.bstein.dev/general-compute=last-resort --overwrite=true || true
+                  fi
+
+                  if k get node titan-23 >/dev/null 2>&1; then
+                    k label node titan-23 \
+                      veles.bstein.dev/simulation=true \
+                      veles.bstein.dev/node-pool=oceanus \
+                      node-role.kubernetes.io/veles-sim=true \
+                      longhorn-host=true \
+                      hardware=oceanus \
+                      --overwrite=true || true
+                    k label node titan-23 node-role.kubernetes.io/worker- || true
+                    k taint node titan-23 veles.bstein.dev/simulation=true:NoSchedule --overwrite=true || true
+                  else
+                    echo "skipping missing node titan-23"
+                  fi
+
+                  for node in titan-13 titan-15 titan-17 titan-19; do
+                    if k get node "${node}" >/dev/null 2>&1; then
+                      k label node "${node}" atlas.bstein.dev/spillover=true longhorn-host=true --overwrite=true || true
+                      k taint node "${node}" longhorn=true:PreferNoSchedule --overwrite=true || true
+                      k taint node "${node}" atlas.bstein.dev/spillover=true:PreferNoSchedule --overwrite=true || true
+                    else
+                      echo "skipping missing node ${node}"
+                    fi
+                  done

infrastructure/core/node-prefer-noschedule-rbac.yaml

@@ -0,0 +1,22 @@
+# infrastructure/core/node-prefer-noschedule-rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: node-prefer-noschedule
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: node-prefer-noschedule
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: node-prefer-noschedule
+subjects:
+  - kind: ServiceAccount
+    name: node-prefer-noschedule
+    namespace: kube-system

infrastructure/core/node-prefer-noschedule-serviceaccount.yaml

@@ -0,0 +1,6 @@
+# infrastructure/core/node-prefer-noschedule-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: node-prefer-noschedule
+  namespace: kube-system

infrastructure/core/workload-profiles.yaml

@@ -0,0 +1,27 @@
+# infrastructure/core/workload-profiles.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: atlas-workload-profiles
+  namespace: kube-system
+data:
+  profiles.yaml: |
+    profiles:
+      tiny:
+        request: { cpu: 25m, memory: 64Mi }
+        limit: { cpu: 200m, memory: 256Mi }
+      light:
+        request: { cpu: 50m, memory: 128Mi }
+        limit: { cpu: 500m, memory: 512Mi }
+      standard:
+        request: { cpu: 250m, memory: 512Mi }
+        limit: { cpu: "1", memory: 1Gi }
+      heavy:
+        request: { cpu: 500m, memory: 1Gi }
+        limit: { cpu: 1500m, memory: 3Gi }
+      ci:
+        request: { cpu: 512m, memory: 512Mi }
+        limit: { cpu: 1500m, memory: 2Gi }
+      scavenger:
+        request: { cpu: 10m, memory: 32Mi }
+        limit: { cpu: 250m, memory: 256Mi }

infrastructure/descheduler/helmrelease.yaml

@@ -0,0 +1,97 @@
+# infrastructure/descheduler/helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: descheduler
+  namespace: kube-system
+spec:
+  interval: 30m
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    remediation:
+      retries: 3
+  chart:
+    spec:
+      chart: descheduler
+      version: 0.33.0
+      sourceRef:
+        kind: HelmRepository
+        name: descheduler
+        namespace: flux-system
+  values:
+    kind: CronJob
+    schedule: "*/20 * * * *"
+    successfulJobsHistoryLimit: 1
+    failedJobsHistoryLimit: 3
+    resources:
+      requests:
+        cpu: 50m
+        memory: 96Mi
+      limits:
+        cpu: 200m
+        memory: 256Mi
+    deschedulerPolicyAPIVersion: descheduler/v1alpha2
+    deschedulerPolicy:
+      maxNoOfPodsToEvictPerNode: 2
+      maxNoOfPodsToEvictPerNamespace: 2
+      profiles:
+        - name: atlas-rpi-balance
+          pluginConfig:
+            - name: DefaultEvictor
+              args:
+                nodeFit: true
+                minPodAge: 10m
+                ignorePvcPods: true
+                evictLocalStoragePods: false
+            - name: RemovePodsHavingTooManyRestarts
+              args:
+                podRestartThreshold: 12
+                includingInitContainers: true
+            - name: RemovePodsViolatingNodeAffinity
+              args:
+                nodeAffinityType:
+                  - requiredDuringSchedulingIgnoredDuringExecution
+            - name: RemovePodsViolatingTopologySpreadConstraint
+            - name: RemovePodsViolatingNodeTaints
+            - name: LowNodeUtilization
+              args:
+                thresholds:
+                  cpu: 45
+                  memory: 45
+                  pods: 45
+                targetThresholds:
+                  cpu: 75
+                  memory: 75
+                  pods: 75
+          plugins:
+            balance:
+              enabled:
+                - RemovePodsViolatingTopologySpreadConstraint
+                - LowNodeUtilization
+            deschedule:
+              enabled:
+                - RemovePodsHavingTooManyRestarts
+                - RemovePodsViolatingNodeTaints
+                - RemovePodsViolatingNodeAffinity
+    priorityClassName: system-cluster-critical
+    nodeSelector:
+      node-role.kubernetes.io/control-plane: "true"
+    affinity:
+      nodeAffinity:
+        preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+                - key: kubernetes.io/hostname
+                  operator: NotIn
+                  values:
+                    - titan-0a
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule

infrastructure/descheduler/kustomization.yaml

@@ -0,0 +1,5 @@
+# infrastructure/descheduler/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - helmrelease.yaml

infrastructure/longhorn/core/helmrelease.yaml

@@ -26,6 +26,9 @@ spec:
     cleanupOnFail: true
     timeout: 15m
   values:
+    global:
+      nodeSelector:
+        longhorn-host: "true"
     service:
       ui:
         type: NodePort
@@ -78,3 +81,23 @@ spec:
           tag: v2.16.0
     defaultSettings:
       systemManagedPodsImagePullPolicy: Always
+      taintToleration: veles.bstein.dev/simulation=true:NoSchedule
+    longhornManager:
+      tolerations:
+        - key: veles.bstein.dev/simulation
+          operator: Equal
+          value: "true"
+          effect: NoSchedule
+      nodeSelector:
+        longhorn-host: "true"
+    longhornDriver:
+      tolerations:
+        - key: veles.bstein.dev/simulation
+          operator: Equal
+          value: "true"
+          effect: NoSchedule
+      nodeSelector:
+        longhorn-host: "true"
+    longhornUI:
+      nodeSelector:
+        longhorn-host: "true"

infrastructure/longhorn/core/kustomization.yaml

@@ -7,7 +7,9 @@ resources:
   - secretproviderclass.yaml
   - vault-sync-deployment.yaml
   - helmrelease.yaml
+  - veles-recurring-jobs.yaml
   - longhorn-settings-ensure-job.yaml
+  - longhorn-csi-toleration-ensure-job.yaml
   - longhorn-disk-tags-ensure-job.yaml
 
 configMapGenerator:

infrastructure/longhorn/core/longhorn-csi-toleration-ensure-job.yaml

@@ -0,0 +1,106 @@
+# infrastructure/longhorn/core/longhorn-csi-toleration-ensure-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: longhorn-csi-toleration-ensure-4
+  namespace: longhorn-system
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 240
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      serviceAccountName: longhorn-service-account
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: titan-11
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: ["arm64"]
+                  - key: node-role.kubernetes.io/worker
+                    operator: Exists
+      containers:
+        - name: patch
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -euo pipefail
+
+              ns="longhorn-system"
+              ds="longhorn-csi-plugin"
+              key="veles.bstein.dev/simulation"
+              value="true"
+              effect="NoSchedule"
+
+              patch_daemonset() {
+                target="$1"
+                current="$(kubectl -n "${ns}" get daemonset "${target}" -o json)"
+                if echo "${current}" | jq -e \
+                  --arg key "${key}" \
+                  --arg value "${value}" \
+                  --arg effect "${effect}" \
+                  '.spec.template.spec.tolerations[]? | select(.key == $key and .value == $value and .effect == $effect)' >/dev/null; then
+                  echo "${target} already tolerates ${key}=${value}:${effect}"
+                  return 0
+                fi
+
+                patch="$(echo "${current}" | jq -c \
+                  --arg key "${key}" \
+                  --arg value "${value}" \
+                  --arg effect "${effect}" \
+                  '{
+                    spec: {
+                      template: {
+                        spec: {
+                          tolerations: ((.spec.template.spec.tolerations // []) + [
+                            {key: $key, operator: "Equal", value: $value, effect: $effect}
+                          ])
+                        }
+                      }
+                    }
+                  }')"
+                kubectl -n "${ns}" patch daemonset "${target}" --type=merge -p "${patch}"
+              }
+
+              patch_daemonset "${ds}"
+              engine_daemonsets="$(kubectl -n "${ns}" get daemonset -l longhorn.io/component=engine-image -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')"
+              for engine_ds in ${engine_daemonsets}; do
+                patch_daemonset "${engine_ds}"
+              done
+
+              csi_ready="false"
+              for attempt in $(seq 1 90); do
+                if kubectl get csinode titan-23 -o json | jq -e '.spec.drivers[]? | select(.name == "driver.longhorn.io")' >/dev/null; then
+                  echo "driver.longhorn.io registered on titan-23"
+                  csi_ready="true"
+                  break
+                fi
+                sleep 2
+              done
+
+              if [ "${csi_ready}" != "true" ]; then
+                echo "driver.longhorn.io did not register on titan-23 before timeout" >&2
+                exit 1
+              fi
+
+              for engine_ds in ${engine_daemonsets}; do
+                for attempt in $(seq 1 90); do
+                  if kubectl -n "${ns}" get pods -o json | jq -e \
+                    --arg engine_ds "${engine_ds}" \
+                    '.items[] | select(.spec.nodeName == "titan-23") | select(.metadata.ownerReferences[]?.name == $engine_ds) | select([.status.containerStatuses[]?.ready] | all)' >/dev/null; then
+                    echo "${engine_ds} ready on titan-23"
+                    break
+                  fi
+                  if [ "${attempt}" = "90" ]; then
+                    echo "${engine_ds} did not become ready on titan-23 before timeout" >&2
+                    exit 1
+                  fi
+                  sleep 2
+                done
+              done

infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: longhorn-disk-tags-ensure-1
+  name: longhorn-disk-tags-ensure-3
   namespace: longhorn-system
 spec:
   backoffLimit: 0

infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml

@@ -2,15 +2,18 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: longhorn-settings-ensure-4
+  name: longhorn-settings-ensure-10
   namespace: longhorn-system
 spec:
   backoffLimit: 0
+  activeDeadlineSeconds: 240
   ttlSecondsAfterFinished: 3600
   template:
     spec:
       serviceAccountName: longhorn-service-account
       restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/hostname: titan-11
       volumes:
         - name: longhorn-settings-ensure-script
           configMap:

infrastructure/longhorn/core/scripts/longhorn_disk_tags_ensure.py

@@ -17,10 +17,28 @@ import urllib.request
 
 LONGHORN_NS = "longhorn-system"
 LONGHORN_API = "/apis/longhorn.io/v1beta2/namespaces/{namespace}/nodes"
-DESIRED_TAGS = {
+DESIRED_DISK_TAGS = {
-    "/mnt/astreae": "astreae",
+    "/mnt/astreae": ["astreae"],
-    "/mnt/asteria": "asteria",
+    "/mnt/asteria": ["asteria"],
+    "/mnt/veles": ["veles-oceanus", "veles-db", "veles-artifacts"],
+    "/mnt/veles-db": ["veles-oceanus", "veles-db"],
+    "/mnt/veles-artifacts": ["veles-oceanus", "veles-artifacts"],
 }
+DESIRED_NODE_TAGS = {
+    "titan-23": ["veles-oceanus"],
+}
+DESIRED_NODE_DISKS = {
+    "titan-23": {
+        "veles-oceanus": {
+            "path": "/mnt/veles",
+            "allowScheduling": True,
+            "evictionRequested": False,
+            "storageReserved": 0,
+            "tags": ["veles-oceanus", "veles-db", "veles-artifacts"],
+        }
+    }
+}
+DISABLE_DEFAULT_DISK_NODES = {"titan-23"}
 
 
 def api_base() -> str:
@@ -63,8 +81,30 @@ def list_nodes() -> list[dict]:
     return data.get("items", [])
 
 
-def patch_disk_tags(node_name: str, disk_name: str, desired_tag: str) -> None:
+def merged_tags(current_tags: list[str], desired_tags: list[str]) -> list[str]:
-    body = {"spec": {"disks": {disk_name: {"tags": [desired_tag]}}}}
+    return sorted(dict.fromkeys([*current_tags, *desired_tags]))
+
+
+def patch_node_tags(node_name: str, desired_tags: list[str]) -> None:
+    body = {"spec": {"tags": desired_tags}}
+    request_json(
+        "PATCH",
+        f"{LONGHORN_API.format(namespace=LONGHORN_NS)}/{node_name}",
+        body=body,
+    )
+
+
+def patch_disk_tags(node_name: str, disk_name: str, desired_tags: list[str]) -> None:
+    body = {"spec": {"disks": {disk_name: {"tags": desired_tags}}}}
+    request_json(
+        "PATCH",
+        f"{LONGHORN_API.format(namespace=LONGHORN_NS)}/{node_name}",
+        body=body,
+    )
+
+
+def patch_disks(node_name: str, disks: dict) -> None:
+    body = {"spec": {"disks": disks}}
     request_json(
         "PATCH",
         f"{LONGHORN_API.format(namespace=LONGHORN_NS)}/{node_name}",
@@ -78,18 +118,52 @@ def main() -> int:
 
     for node in list_nodes():
         name = node.get("metadata", {}).get("name", "")
+        desired_node_tags = DESIRED_NODE_TAGS.get(name)
+        if desired_node_tags:
+            current_node_tags = node.get("spec", {}).get("tags") or []
+            next_node_tags = merged_tags(current_node_tags, desired_node_tags)
+            if current_node_tags != next_node_tags:
+                print(f"patching {name} node tags={current_node_tags!r} -> {next_node_tags!r}")
+                patch_node_tags(name, next_node_tags)
+                changed += 1
+            else:
+                skipped += 1
+
         spec_disks = node.get("spec", {}).get("disks", {}) or {}
+        desired_disks = DESIRED_NODE_DISKS.get(name, {})
+        missing_disks = {
+            disk_name: disk_spec
+            for disk_name, disk_spec in desired_disks.items()
+            if disk_name not in spec_disks
+        }
+        if missing_disks:
+            print(f"adding {name} disks={sorted(missing_disks)}")
+            patch_disks(name, missing_disks)
+            changed += len(missing_disks)
+            spec_disks = {**spec_disks, **missing_disks}
+
+        if name in DISABLE_DEFAULT_DISK_NODES:
+            disable_patch = {}
+            for disk_name, disk in spec_disks.items():
+                disk_path = (disk.get("path") or "").rstrip("/")
+                if disk_path == "/var/lib/longhorn" and disk.get("allowScheduling", True):
+                    disable_patch[disk_name] = {"allowScheduling": False}
+            if disable_patch:
+                print(f"disabling default Longhorn scheduling on {name} disks={sorted(disable_patch)}")
+                patch_disks(name, disable_patch)
+                changed += len(disable_patch)
+
         for disk_name, disk in spec_disks.items():
             disk_path = disk.get("path")
-            desired_tag = DESIRED_TAGS.get(disk_path)
+            desired_disk_tags = DESIRED_DISK_TAGS.get(disk_path)
-            if not desired_tag:
+            if not desired_disk_tags:
                 continue
             current_tags = disk.get("tags") or []
-            if current_tags == [desired_tag]:
+            if current_tags == desired_disk_tags:
                 skipped += 1
                 continue
-            print(f"patching {name}:{disk_name} path={disk_path} tags={current_tags!r} -> {[desired_tag]!r}")
+            print(f"patching {name}:{disk_name} path={disk_path} tags={current_tags!r} -> {desired_disk_tags!r}")
-            patch_disk_tags(name, disk_name, desired_tag)
+            patch_disk_tags(name, disk_name, desired_disk_tags)
             changed += 1
 
     print(f"done: changed={changed} skipped={skipped}")

infrastructure/longhorn/core/scripts/longhorn_settings_ensure.sh

@@ -4,11 +4,12 @@ set -eu
 # Longhorn blocks direct CR patches for some settings; use the internal API instead.
 
 api_base="http://longhorn-backend.longhorn-system.svc:9500/v1/settings"
+curl_opts="-fsS --connect-timeout 3 --max-time 15"
 
 wait_for_api() {
   attempts=30
   while [ "${attempts}" -gt 0 ]; do
-    if curl -fsS "${api_base}" >/dev/null 2>&1; then
+    if curl ${curl_opts} "${api_base}" >/dev/null 2>&1; then
       return 0
     fi
     attempts=$((attempts - 1))
@@ -22,17 +23,32 @@ update_setting() {
   name="$1"
   value="$2"
 
-  current="$(curl -fsS "${api_base}/${name}" || true)"
+  current="$(curl ${curl_opts} "${api_base}/${name}" || true)"
   if echo "${current}" | grep -Fq "\"value\":\"${value}\""; then
     echo "Setting ${name} already set."
     return 0
   fi
 
   echo "Setting ${name} -> ${value}"
-  curl -fsS -X PUT \
+  out="$(mktemp)"
+  if curl ${curl_opts} -o "${out}" -X PUT \
     -H "Content-Type: application/json" \
     -d "{\"value\":\"${value}\"}" \
-    "${api_base}/${name}" >/dev/null
+    "${api_base}/${name}"; then
+    rm -f "${out}"
+    return 0
+  fi
+
+  current="$(curl ${curl_opts} "${api_base}/${name}" || true)"
+  if echo "${current}" | grep -Fq "\"value\":\"${value}\""; then
+    echo "Setting ${name} stored; Longhorn will apply it when current state allows."
+    rm -f "${out}"
+    return 0
+  fi
+
+  cat "${out}" >&2 || true
+  rm -f "${out}"
+  return 1
 }
 
 wait_for_api
@@ -40,3 +56,8 @@ update_setting default-engine-image "registry.bstein.dev/infra/longhorn-engine:v
 update_setting default-instance-manager-image "registry.bstein.dev/infra/longhorn-instance-manager:v1.8.2"
 update_setting default-backing-image-manager-image "registry.bstein.dev/infra/longhorn-backing-image-manager:v1.8.2"
 update_setting support-bundle-manager-image "registry.bstein.dev/infra/longhorn-support-bundle-kit:v0.0.56"
+update_setting taint-toleration "veles.bstein.dev/simulation=true:NoSchedule"
+# Keep storage-heavy nodes from getting hammered by rebuild storms and skew.
+update_setting replica-auto-balance "best-effort"
+update_setting concurrent-replica-rebuild-per-node-limit "2"
+update_setting node-down-pod-deletion-policy "delete-both-statefulset-and-deployment-pod"

infrastructure/longhorn/core/secretproviderclass.yaml

@@ -13,9 +13,27 @@ spec:
       - objectName: "harbor-pull__dockerconfigjson"
         secretPath: "kv/data/atlas/shared/harbor-pull"
         secretKey: "dockerconfigjson"
+      - objectName: "longhorn-backup-b2__AWS_ACCESS_KEY_ID"
+        secretPath: "kv/data/atlas/longhorn/backup-b2"
+        secretKey: "AWS_ACCESS_KEY_ID"
+      - objectName: "longhorn-backup-b2__AWS_SECRET_ACCESS_KEY"
+        secretPath: "kv/data/atlas/longhorn/backup-b2"
+        secretKey: "AWS_SECRET_ACCESS_KEY"
+      - objectName: "longhorn-backup-b2__AWS_ENDPOINTS"
+        secretPath: "kv/data/atlas/longhorn/backup-b2"
+        secretKey: "AWS_ENDPOINTS"
   secretObjects:
     - secretName: longhorn-registry
       type: kubernetes.io/dockerconfigjson
       data:
         - objectName: harbor-pull__dockerconfigjson
           key: .dockerconfigjson
+    - secretName: longhorn-backup-b2
+      type: Opaque
+      data:
+        - objectName: longhorn-backup-b2__AWS_ACCESS_KEY_ID
+          key: AWS_ACCESS_KEY_ID
+        - objectName: longhorn-backup-b2__AWS_SECRET_ACCESS_KEY
+          key: AWS_SECRET_ACCESS_KEY
+        - objectName: longhorn-backup-b2__AWS_ENDPOINTS
+          key: AWS_ENDPOINTS

infrastructure/longhorn/core/vault-sync-deployment.yaml

@@ -26,6 +26,16 @@ spec:
                   - key: hardware
                     operator: In
                     values: ["rpi5", "rpi4"]
+            - weight: 90
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values:
+                      - titan-13
+                      - titan-15
+                      - titan-17
+                      - titan-19
       containers:
         - name: sync
           image: alpine:3.20

infrastructure/longhorn/core/veles-recurring-jobs.yaml

@@ -0,0 +1,60 @@
+# infrastructure/longhorn/core/veles-recurring-jobs.yaml
+apiVersion: longhorn.io/v1beta2
+kind: RecurringJob
+metadata:
+  name: veles-postgres-backup
+  namespace: longhorn-system
+spec:
+  name: veles-postgres-backup
+  cron: "30 5 * * *"
+  task: backup
+  groups:
+    - veles
+    - veles-postgres
+  retain: 7
+  concurrency: 1
+---
+apiVersion: longhorn.io/v1beta2
+kind: RecurringJob
+metadata:
+  name: veles-postgres-snapshot
+  namespace: longhorn-system
+spec:
+  name: veles-postgres-snapshot
+  cron: "*/30 * * * *"
+  task: snapshot
+  groups:
+    - veles
+    - veles-postgres
+  retain: 8
+  concurrency: 1
+---
+apiVersion: longhorn.io/v1beta2
+kind: RecurringJob
+metadata:
+  name: veles-artifacts-backup
+  namespace: longhorn-system
+spec:
+  name: veles-artifacts-backup
+  cron: "45 5 * * *"
+  task: backup
+  groups:
+    - veles
+    - veles-artifacts
+  retain: 7
+  concurrency: 1
+---
+apiVersion: longhorn.io/v1beta2
+kind: RecurringJob
+metadata:
+  name: veles-artifacts-snapshot
+  namespace: longhorn-system
+spec:
+  name: veles-artifacts-snapshot
+  cron: "15 */6 * * *"
+  task: snapshot
+  groups:
+    - veles
+    - veles-artifacts
+  retain: 8
+  concurrency: 1

infrastructure/modules/base/priorityclass/kustomization.yaml

@@ -3,3 +3,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - scavenger.yaml
+  - veles.yaml

infrastructure/modules/base/priorityclass/veles.yaml

@@ -0,0 +1,17 @@
+# infrastructure/modules/base/priorityclass/veles.yaml
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: veles-core
+value: 500
+globalDefault: false
+description: "For Veles core database, API, and controller workloads"
+---
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: veles-sim
+value: 50
+globalDefault: false
+preemptionPolicy: Never
+description: "For Veles simulation jobs; lower than core and non-preempting"

infrastructure/modules/base/storageclass/kustomization.yaml

@@ -5,3 +5,6 @@ resources:
   - asteria.yaml
   - asteria-encrypted.yaml
   - astreae.yaml
+  - veles-oceanus-db.yaml
+  - veles-oceanus-artifacts.yaml
+  - veles-oceanus-policy.yaml

infrastructure/modules/base/storageclass/veles-oceanus-artifacts.yaml

@@ -0,0 +1,21 @@
+# infrastructure/modules/base/storageclass/veles-oceanus-artifacts.yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: veles-oceanus-artifacts
+  annotations:
+    veles.bstein.dev/allowed-namespace: veles
+provisioner: driver.longhorn.io
+parameters:
+  nodeSelector: veles-oceanus
+  diskSelector: veles-oceanus,veles-artifacts
+  fromBackup: ""
+  numberOfReplicas: "1"
+  staleReplicaTimeout: "30"
+  fsType: ext4
+  replicaAutoBalance: disabled
+  dataLocality: strict-local
+  recurringJobSelector: '[{"name":"veles-artifacts-backup","isGroup":false},{"name":"veles-artifacts-snapshot","isGroup":false}]'
+reclaimPolicy: Retain
+allowVolumeExpansion: true
+volumeBindingMode: WaitForFirstConsumer

infrastructure/modules/base/storageclass/veles-oceanus-db.yaml

@@ -0,0 +1,21 @@
+# infrastructure/modules/base/storageclass/veles-oceanus-db.yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: veles-oceanus-db
+  annotations:
+    veles.bstein.dev/allowed-namespace: veles
+provisioner: driver.longhorn.io
+parameters:
+  nodeSelector: veles-oceanus
+  diskSelector: veles-oceanus,veles-db
+  fromBackup: ""
+  numberOfReplicas: "1"
+  staleReplicaTimeout: "30"
+  fsType: ext4
+  replicaAutoBalance: disabled
+  dataLocality: strict-local
+  recurringJobSelector: '[{"name":"veles-postgres-backup","isGroup":false},{"name":"veles-postgres-snapshot","isGroup":false}]'
+reclaimPolicy: Retain
+allowVolumeExpansion: true
+volumeBindingMode: WaitForFirstConsumer

infrastructure/modules/base/storageclass/veles-oceanus-policy.yaml

@@ -0,0 +1,25 @@
+# infrastructure/modules/base/storageclass/veles-oceanus-policy.yaml
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicy
+metadata:
+  name: veles-oceanus-storage-namespace
+spec:
+  failurePolicy: Fail
+  matchConstraints:
+    resourceRules:
+      - apiGroups: [""]
+        apiVersions: ["v1"]
+        operations: ["CREATE", "UPDATE"]
+        resources: ["persistentvolumeclaims"]
+  validations:
+    - expression: "!has(object.spec.storageClassName) || !(object.spec.storageClassName in ['veles-oceanus-db', 'veles-oceanus-artifacts']) || object.metadata.namespace == 'veles'"
+      message: "Veles Oceanus storage classes are reserved for namespace veles"
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingAdmissionPolicyBinding
+metadata:
+  name: veles-oceanus-storage-namespace
+spec:
+  policyName: veles-oceanus-storage-namespace
+  validationActions:
+    - Deny

infrastructure/postgres/statefulset.yaml

@@ -25,6 +25,7 @@ spec:
       serviceAccountName: postgres-vault
       nodeSelector:
         node-role.kubernetes.io/worker: "true"
+        hardware: rpi5
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -35,7 +36,17 @@ spec:
                     values: ["true"]
                   - key: hardware
                     operator: In
-                    values: ["rpi4", "rpi5"]
+                    values: ["rpi5"]
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: ["titan-06"]
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/hostname
+                    operator: In
+                    values: ["titan-05", "titan-07", "titan-08", "titan-11"]
       containers:
         - name: postgres
           image: postgres:15

infrastructure/resource-guardrails/kustomization.yaml

@@ -0,0 +1,5 @@
+# infrastructure/resource-guardrails/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - limitranges.yaml