Merge pull request 'feature/ariadne' (#11) from feature/ariadne into main

Reviewed-on: #11

.gitignore

@@ -6,3 +6,5 @@ __pycache__/
 *.py[cod]
 .pytest_cache
 .venv
+.venv-ci
+tmp/

Jenkinsfile

@@ -0,0 +1,77 @@
+// Mirror of ci/Jenkinsfile.titan-iac for multibranch discovery.
+pipeline {
+  agent {
+    kubernetes {
+      defaultContainer 'python'
+      yaml """
+apiVersion: v1
+kind: Pod
+spec:
+  nodeSelector:
+    hardware: rpi5
+    kubernetes.io/arch: arm64
+    node-role.kubernetes.io/worker: "true"
+  containers:
+    - name: python
+      image: python:3.12-slim
+      command:
+        - cat
+      tty: true
+"""
+    }
+  }
+  environment {
+    PIP_DISABLE_PIP_VERSION_CHECK = '1'
+    PYTHONUNBUFFERED = '1'
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        checkout scm
+      }
+    }
+    stage('Install deps') {
+      steps {
+        sh 'pip install --no-cache-dir -r ci/requirements.txt'
+      }
+    }
+    stage('Glue tests') {
+      steps {
+        sh 'pytest -q ci/tests/glue'
+      }
+    }
+    stage('Resolve Flux branch') {
+      steps {
+        script {
+          env.FLUX_BRANCH = sh(
+            returnStdout: true,
+            script: "awk '/branch:/{print $2; exit}' clusters/atlas/flux-system/gotk-sync.yaml"
+          ).trim()
+          if (!env.FLUX_BRANCH) {
+            error('Flux branch not found in gotk-sync.yaml')
+          }
+          echo "Flux branch: ${env.FLUX_BRANCH}"
+        }
+      }
+    }
+    stage('Promote') {
+      when {
+        expression {
+          def branch = env.BRANCH_NAME ?: (env.GIT_BRANCH ?: '').replaceFirst('origin/', '')
+          return env.FLUX_BRANCH && branch == env.FLUX_BRANCH
+        }
+      }
+      steps {
+        withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
+          sh '''
+            set +x
+            git config user.email "jenkins@bstein.dev"
+            git config user.name "jenkins"
+            git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
+            git push origin HEAD:${FLUX_BRANCH}
+          '''
+        }
+      }
+    }
+  }
+}

ci/Jenkinsfile.titan-iac

@@ -0,0 +1,76 @@
+pipeline {
+  agent {
+    kubernetes {
+      defaultContainer 'python'
+      yaml """
+apiVersion: v1
+kind: Pod
+spec:
+  nodeSelector:
+    hardware: rpi5
+    kubernetes.io/arch: arm64
+    node-role.kubernetes.io/worker: "true"
+  containers:
+    - name: python
+      image: python:3.12-slim
+      command:
+        - cat
+      tty: true
+"""
+    }
+  }
+  environment {
+    PIP_DISABLE_PIP_VERSION_CHECK = '1'
+    PYTHONUNBUFFERED = '1'
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        checkout scm
+      }
+    }
+    stage('Install deps') {
+      steps {
+        sh 'pip install --no-cache-dir -r ci/requirements.txt'
+      }
+    }
+    stage('Glue tests') {
+      steps {
+        sh 'pytest -q ci/tests/glue'
+      }
+    }
+    stage('Resolve Flux branch') {
+      steps {
+        script {
+          env.FLUX_BRANCH = sh(
+            returnStdout: true,
+            script: "awk '/branch:/{print $2; exit}' clusters/atlas/flux-system/gotk-sync.yaml"
+          ).trim()
+          if (!env.FLUX_BRANCH) {
+            error('Flux branch not found in gotk-sync.yaml')
+          }
+          echo "Flux branch: ${env.FLUX_BRANCH}"
+        }
+      }
+    }
+    stage('Promote') {
+      when {
+        expression {
+          def branch = env.BRANCH_NAME ?: (env.GIT_BRANCH ?: '').replaceFirst('origin/', '')
+          return env.FLUX_BRANCH && branch == env.FLUX_BRANCH
+        }
+      }
+      steps {
+        withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
+          sh '''
+            set +x
+            git config user.email "jenkins@bstein.dev"
+            git config user.name "jenkins"
+            git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
+            git push origin HEAD:${FLUX_BRANCH}
+          '''
+        }
+      }
+    }
+  }
+}

ci/requirements.txt

@@ -0,0 +1,4 @@
+pytest==8.3.4
+kubernetes==30.1.0
+PyYAML==6.0.2
+requests==2.32.3

ci/tests/glue/config.yaml

@@ -0,0 +1,16 @@
+max_success_age_hours: 48
+allow_suspended:
+  - bstein-dev-home/vaultwarden-cred-sync
+  - comms/othrys-room-reset
+  - comms/pin-othrys-invite
+  - comms/seed-othrys-room
+  - finance/firefly-user-sync
+  - health/wger-admin-ensure
+  - health/wger-user-sync
+  - mailu-mailserver/mailu-sync-nightly
+  - nextcloud/nextcloud-mail-sync
+ariadne_schedule_tasks:
+  - schedule.mailu_sync
+  - schedule.nextcloud_sync
+  - schedule.vaultwarden_sync
+  - schedule.wger_admin

ci/tests/glue/test_glue_cronjobs.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+import yaml
+from kubernetes import client, config
+
+
+CONFIG_PATH = Path(__file__).with_name("config.yaml")
+
+
+def _load_config() -> dict:
+    with CONFIG_PATH.open("r", encoding="utf-8") as handle:
+        return yaml.safe_load(handle) or {}
+
+
+def _load_kube():
+    try:
+        config.load_incluster_config()
+    except config.ConfigException:
+        config.load_kube_config()
+
+
+def test_glue_cronjobs_recent_success():
+    cfg = _load_config()
+    max_age_hours = int(cfg.get("max_success_age_hours", 48))
+    allow_suspended = set(cfg.get("allow_suspended", []))
+
+    _load_kube()
+    batch = client.BatchV1Api()
+    cronjobs = batch.list_cron_job_for_all_namespaces(label_selector="atlas.bstein.dev/glue=true").items
+
+    assert cronjobs, "No glue cronjobs found with atlas.bstein.dev/glue=true"
+
+    now = datetime.now(timezone.utc)
+    for cronjob in cronjobs:
+        name = f"{cronjob.metadata.namespace}/{cronjob.metadata.name}"
+        if cronjob.spec.suspend:
+            assert name in allow_suspended, f"{name} is suspended but not in allow_suspended"
+            continue
+
+        last_success = cronjob.status.last_successful_time
+        assert last_success is not None, f"{name} has no lastSuccessfulTime"
+        age_hours = (now - last_success).total_seconds() / 3600
+        assert age_hours <= max_age_hours, f"{name} last success {age_hours:.1f}h ago"

ci/tests/glue/test_glue_metrics.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import requests
+import yaml
+
+
+VM_URL = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
+CONFIG_PATH = Path(__file__).with_name("config.yaml")
+
+
+def _load_config() -> dict:
+    with CONFIG_PATH.open("r", encoding="utf-8") as handle:
+        return yaml.safe_load(handle) or {}
+
+
+def _query(promql: str) -> list[dict]:
+    response = requests.get(f"{VM_URL}/api/v1/query", params={"query": promql}, timeout=10)
+    response.raise_for_status()
+    payload = response.json()
+    return payload.get("data", {}).get("result", [])
+
+
+def test_glue_metrics_present():
+    series = _query('kube_cronjob_labels{label_atlas_bstein_dev_glue="true"}')
+    assert series, "No glue cronjob label series found"
+
+
+def test_glue_metrics_success_join():
+    query = (
+        "kube_cronjob_status_last_successful_time "
+        'and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue="true"}'
+    )
+    series = _query(query)
+    assert series, "No glue cronjob last success series found"
+
+
+def test_ariadne_schedule_metrics_present():
+    cfg = _load_config()
+    expected = cfg.get("ariadne_schedule_tasks", [])
+    if not expected:
+        return
+    series = _query("ariadne_schedule_next_run_timestamp_seconds")
+    tasks = {item.get("metric", {}).get("task") for item in series}
+    missing = [task for task in expected if task not in tasks]
+    assert not missing, f"Missing Ariadne schedule metrics for: {', '.join(missing)}"

clusters/atlas/applications/kustomization.yaml

@@ -1,13 +0,0 @@
-# clusters/atlas/applications/kustomization.yaml
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - ../../services/crypto
-  - ../../services/gitea
-  - ../../services/jellyfin
-  - ../../services/comms
-  - ../../services/monitoring
-  - ../../services/logging
-  - ../../services/pegasus
-  - ../../services/vault
-  - ../../services/bstein-dev-home

clusters/atlas/flux-system/applications/bstein-dev-home-migrations/kustomization.yaml

@@ -0,0 +1,17 @@
+# clusters/atlas/flux-system/applications/bstein-dev-home-migrations/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: bstein-dev-home-migrations
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/bstein-dev-home/oneoffs/migrations
+  prune: true
+  force: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: bstein-dev-home
+  wait: false
+  suspend: true

clusters/atlas/flux-system/applications/bstein-dev-home/image-automation.yaml

@@ -3,7 +3,7 @@ apiVersion: image.toolkit.fluxcd.io/v1
 kind: ImageUpdateAutomation
 metadata:
   name: bstein-dev-home
-  namespace: flux-system
+  namespace: bstein-dev-home
 spec:
   interval: 1m0s
   sourceRef:
@@ -13,14 +13,14 @@ spec:
   git:
     checkout:
       ref:
-        branch: main
+        branch: feature/ariadne
     commit:
       author:
         email: ops@bstein.dev
         name: flux-bot
-      messageTemplate: "chore(bstein-dev-home): update images to {{range .Updated.Images}}{{.}}{{end}}"
+      messageTemplate: "chore(bstein-dev-home): automated image update"
     push:
-      branch: main
+      branch: feature/ariadne
   update:
     strategy: Setters
     path: services/bstein-dev-home

clusters/atlas/flux-system/applications/comms/kustomization.yaml

@@ -1,4 +1,4 @@
-# clusters/atlas/flux-system/applications/communication/kustomization.yaml
+# clusters/atlas/flux-system/applications/comms/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:

clusters/atlas/flux-system/applications/finance/kustomization.yaml

@@ -0,0 +1,24 @@
+# clusters/atlas/flux-system/applications/finance/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: finance
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/finance
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: finance
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: actual-budget
+      namespace: finance
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: firefly
+      namespace: finance
+  wait: false

clusters/atlas/flux-system/applications/harbor/kustomization.yaml

@@ -13,11 +13,6 @@ spec:
     kind: GitRepository
     name: flux-system
     namespace: flux-system
-  healthChecks:
-    - apiVersion: helm.toolkit.fluxcd.io/v2
-      kind: HelmRelease
-      name: harbor
-      namespace: harbor
   wait: false
   dependsOn:
     - name: core

clusters/atlas/flux-system/applications/health/kustomization.yaml

@@ -0,0 +1,25 @@
+# clusters/atlas/flux-system/applications/health/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: health
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/health
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: health
+  dependsOn:
+    - name: keycloak
+    - name: postgres
+    - name: traefik
+    - name: vault
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: wger
+      namespace: health
+  wait: false

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -12,10 +12,12 @@ resources:
   - pegasus/image-automation.yaml
   - bstein-dev-home/kustomization.yaml
   - bstein-dev-home/image-automation.yaml
+  - bstein-dev-home-migrations/kustomization.yaml
   - harbor/kustomization.yaml
   - harbor/image-automation.yaml
   - jellyfin/kustomization.yaml
   - xmr-miner/kustomization.yaml
+  - wallet-monero-temp/kustomization.yaml
   - sui-metrics/kustomization.yaml
   - openldap/kustomization.yaml
   - keycloak/kustomization.yaml
@@ -25,6 +27,7 @@ resources:
   - ai-llm/kustomization.yaml
   - nextcloud/kustomization.yaml
   - nextcloud-mail-sync/kustomization.yaml
-  - postgres/kustomization.yaml
   - outline/kustomization.yaml
   - planka/kustomization.yaml
+  - finance/kustomization.yaml
+  - health/kustomization.yaml

clusters/atlas/flux-system/applications/pegasus/image-automation.yaml

@@ -3,7 +3,7 @@ apiVersion: image.toolkit.fluxcd.io/v1
 kind: ImageUpdateAutomation
 metadata:
   name: pegasus
-  namespace: flux-system
+  namespace: jellyfin
 spec:
   interval: 1m0s
   sourceRef:

clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: wallet-monero-temp
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/crypto/wallet-monero-temp
+  targetNamespace: crypto
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  dependsOn:
+    - name: crypto
+    - name: xmr-miner
+  wait: true

clusters/atlas/flux-system/gotk-components.yaml

@@ -1,3 +1,4 @@
+# clusters/atlas/flux-system/gotk-components.yaml
 ---
 # This manifest was generated by flux. DO NOT EDIT.
 # Flux Version: v2.7.5

clusters/atlas/flux-system/gotk-sync.yaml

@@ -1,3 +1,4 @@
+# clusters/atlas/flux-system/gotk-sync.yaml
 # This manifest was generated by flux. DO NOT EDIT.
 ---
 apiVersion: source.toolkit.fluxcd.io/v1
@@ -8,7 +9,7 @@ metadata:
 spec:
   interval: 1m0s
   ref:
-    branch: main
+    branch: feature/ariadne
   secretRef:
     name: flux-system-gitea
   url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git

clusters/atlas/flux-system/platform/cert-manager-cleanup/kustomization.yaml

@@ -0,0 +1,17 @@
+# clusters/atlas/flux-system/platform/cert-manager-cleanup/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cert-manager-cleanup
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./infrastructure/cert-manager/cleanup
+  prune: true
+  force: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: cert-manager
+  wait: true

clusters/atlas/flux-system/platform/cert-manager/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/platform/cert-manager/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cert-manager
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./infrastructure/cert-manager
+  prune: true
+  force: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: cert-manager
+  dependsOn:
+    - name: helm
+  wait: true

clusters/atlas/flux-system/platform/kustomization.yaml

@@ -4,11 +4,17 @@ kind: Kustomization
 resources:
   - core/kustomization.yaml
   - helm/kustomization.yaml
+  - cert-manager/kustomization.yaml
   - metallb/kustomization.yaml
   - traefik/kustomization.yaml
   - gitops-ui/kustomization.yaml
   - monitoring/kustomization.yaml
   - logging/kustomization.yaml
   - maintenance/kustomization.yaml
+  - maintenance/image-automation.yaml
+  - longhorn-adopt/kustomization.yaml
+  - longhorn/kustomization.yaml
   - longhorn-ui/kustomization.yaml
+  - postgres/kustomization.yaml
   - ../platform/vault-csi/kustomization.yaml
+  - ../platform/vault-injector/kustomization.yaml

clusters/atlas/flux-system/platform/longhorn-adopt/kustomization.yaml

@@ -0,0 +1,17 @@
+# clusters/atlas/flux-system/platform/longhorn-adopt/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: longhorn-adopt
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./infrastructure/longhorn/adopt
+  prune: true
+  force: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: longhorn-system
+  wait: true

clusters/atlas/flux-system/platform/longhorn-ui/kustomization.yaml

@@ -15,4 +15,5 @@ spec:
     namespace: flux-system
   dependsOn:
     - name: core
+    - name: longhorn
   wait: true

clusters/atlas/flux-system/platform/longhorn/kustomization.yaml

@@ -0,0 +1,20 @@
+# clusters/atlas/flux-system/platform/longhorn/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: longhorn
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./infrastructure/longhorn/core
+  prune: true
+  force: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: longhorn-system
+  dependsOn:
+    - name: helm
+    - name: longhorn-adopt
+  wait: false

clusters/atlas/flux-system/platform/maintenance/image-automation.yaml

@@ -0,0 +1,26 @@
+# clusters/atlas/flux-system/platform/maintenance/image-automation.yaml
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageUpdateAutomation
+metadata:
+  name: maintenance
+  namespace: maintenance
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  git:
+    checkout:
+      ref:
+        branch: feature/ariadne
+    commit:
+      author:
+        email: ops@bstein.dev
+        name: flux-bot
+      messageTemplate: "chore(maintenance): automated image update"
+    push:
+      branch: feature/ariadne
+  update:
+    strategy: Setters
+    path: services/maintenance

clusters/atlas/flux-system/platform/maintenance/kustomization.yaml

@@ -8,6 +8,7 @@ spec:
   interval: 10m
   path: ./services/maintenance
   prune: true
+  force: true
   sourceRef:
     kind: GitRepository
     name: flux-system

clusters/atlas/flux-system/platform/postgres/kustomization.yaml

@@ -1,4 +1,4 @@
-# clusters/atlas/flux-system/applications/postgres/kustomization.yaml
+# clusters/atlas/flux-system/platform/postgres/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
@@ -6,7 +6,7 @@ metadata:
   namespace: flux-system
 spec:
   interval: 10m
-  path: ./services/postgres
+  path: ./infrastructure/postgres
   prune: true
   force: true
   sourceRef:

clusters/atlas/flux-system/platform/vault-injector/kustomization.yaml

@@ -0,0 +1,16 @@
+# clusters/atlas/flux-system/platform/vault-injector/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: vault-injector
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./infrastructure/vault-injector
+  targetNamespace: vault
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true

clusters/atlas/platform/kustomization.yaml

@@ -1,8 +0,0 @@
-# clusters/atlas/platform/kustomization.yaml
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - ../../../infrastructure/modules/base
-  - ../../../infrastructure/modules/profiles/atlas-ha
-  - ../../../infrastructure/sources/cert-manager/letsencrypt.yaml
-  - ../../../infrastructure/metallb

dockerfiles/Dockerfile.comms-guest-tools

@@ -0,0 +1,5 @@
+FROM python:3.11-slim
+
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1
+
+RUN pip install --no-cache-dir requests psycopg2-binary

dockerfiles/Dockerfile.harbor-core-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-core:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/harbor/entrypoint.sh"]

dockerfiles/Dockerfile.harbor-jobservice-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-jobservice:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/harbor/entrypoint.sh"]

dockerfiles/Dockerfile.harbor-registry-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-registry:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/home/harbor/entrypoint.sh"]

dockerfiles/Dockerfile.harbor-registryctl-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-registryctl:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/home/harbor/start.sh"]

dockerfiles/Dockerfile.livekit-token-vault

@@ -0,0 +1,10 @@
+FROM ghcr.io/element-hq/lk-jwt-service:0.3.0 AS base
+
+FROM alpine:3.20
+RUN apk add --no-cache ca-certificates
+COPY --from=base /lk-jwt-service /lk-jwt-service
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/lk-jwt-service"]

dockerfiles/Dockerfile.oauth2-proxy-vault

@@ -0,0 +1,10 @@
+FROM quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 AS base
+
+FROM alpine:3.20
+RUN apk add --no-cache ca-certificates
+COPY --from=base /bin/oauth2-proxy /bin/oauth2-proxy
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/bin/oauth2-proxy"]

dockerfiles/Dockerfile.pegasus-vault

@@ -0,0 +1,10 @@
+FROM registry.bstein.dev/streaming/pegasus:1.2.32 AS base
+
+FROM alpine:3.20
+RUN apk add --no-cache ca-certificates
+COPY --from=base /pegasus /pegasus
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/pegasus"]

dockerfiles/vault-entrypoint.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+set -eu
+
+if [ -n "${VAULT_ENV_FILE:-}" ]; then
+  if [ -f "${VAULT_ENV_FILE}" ]; then
+    # shellcheck disable=SC1090
+    . "${VAULT_ENV_FILE}"
+  else
+    echo "Vault env file not found: ${VAULT_ENV_FILE}" >&2
+    exit 1
+  fi
+fi
+
+if [ -n "${VAULT_COPY_FILES:-}" ]; then
+  old_ifs="$IFS"
+  IFS=','
+  for pair in ${VAULT_COPY_FILES}; do
+    src="${pair%%:*}"
+    dest="${pair#*:}"
+    if [ -z "${src}" ] || [ -z "${dest}" ]; then
+      echo "Vault copy entry malformed: ${pair}" >&2
+      exit 1
+    fi
+    if [ ! -f "${src}" ]; then
+      echo "Vault file not found: ${src}" >&2
+      exit 1
+    fi
+    mkdir -p "$(dirname "${dest}")"
+    cp "${src}" "${dest}"
+  done
+  IFS="$old_ifs"
+fi
+
+exec "$@"

infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml

@@ -0,0 +1,40 @@
+# infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cert-manager-cleanup-2
+  namespace: cert-manager
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      serviceAccountName: cert-manager-cleanup
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/worker
+                    operator: Exists
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: ["arm64"]
+      containers:
+        - name: cleanup
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command: ["/usr/bin/env", "bash"]
+          args: ["/scripts/cert_manager_cleanup.sh"]
+          volumeMounts:
+            - name: script
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: script
+          configMap:
+            name: cert-manager-cleanup-script
+            defaultMode: 0555

infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml

@@ -0,0 +1,58 @@
+# infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cert-manager-cleanup
+  namespace: cert-manager
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cert-manager-cleanup
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - services
+      - endpoints
+      - configmaps
+      - secrets
+      - serviceaccounts
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["apps"]
+    resources:
+      - deployments
+      - daemonsets
+      - statefulsets
+      - replicasets
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["batch"]
+    resources:
+      - jobs
+      - cronjobs
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["rbac.authorization.k8s.io"]
+    resources:
+      - roles
+      - rolebindings
+      - clusterroles
+      - clusterrolebindings
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["admissionregistration.k8s.io"]
+    resources:
+      - validatingwebhookconfigurations
+      - mutatingwebhookconfigurations
+    verbs: ["get", "list", "watch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cert-manager-cleanup
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cert-manager-cleanup
+subjects:
+  - kind: ServiceAccount
+    name: cert-manager-cleanup
+    namespace: cert-manager

infrastructure/cert-manager/cleanup/kustomization.yaml

@@ -0,0 +1,15 @@
+# infrastructure/cert-manager/cleanup/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - cert-manager-cleanup-rbac.yaml
+  - cert-manager-cleanup-job.yaml
+
+configMapGenerator:
+  - name: cert-manager-cleanup-script
+    namespace: cert-manager
+    files:
+      - cert_manager_cleanup.sh=scripts/cert_manager_cleanup.sh
+    options:
+      disableNameSuffixHash: true

infrastructure/cert-manager/cleanup/namespace.yaml

@@ -0,0 +1,5 @@
+# infrastructure/cert-manager/cleanup/namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager

infrastructure/cert-manager/cleanup/scripts/cert_manager_cleanup.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+namespace="cert-manager"
+selectors=(
+  "app.kubernetes.io/name=cert-manager"
+  "app.kubernetes.io/instance=cert-manager"
+  "app.kubernetes.io/instance=certmanager-prod"
+)
+
+delete_namespaced() {
+  local selector="$1"
+  kubectl -n "${namespace}" delete deployment,daemonset,statefulset,replicaset \
+    --selector "${selector}" --ignore-not-found --wait=false
+  kubectl -n "${namespace}" delete pod,service,endpoints,serviceaccount,configmap,secret \
+    --selector "${selector}" --ignore-not-found --wait=false
+  kubectl -n "${namespace}" delete role,rolebinding \
+    --selector "${selector}" --ignore-not-found --wait=false
+  kubectl -n "${namespace}" delete job,cronjob \
+    --selector "${selector}" --ignore-not-found --wait=false
+}
+
+delete_cluster_scoped() {
+  local selector="$1"
+  kubectl delete clusterrole,clusterrolebinding \
+    --selector "${selector}" --ignore-not-found --wait=false
+  kubectl delete mutatingwebhookconfiguration,validatingwebhookconfiguration \
+    --selector "${selector}" --ignore-not-found --wait=false
+}
+
+for selector in "${selectors[@]}"; do
+  delete_namespaced "${selector}"
+  delete_cluster_scoped "${selector}"
+done
+
+kubectl delete mutatingwebhookconfiguration cert-manager-webhook --ignore-not-found --wait=false
+kubectl delete validatingwebhookconfiguration cert-manager-webhook --ignore-not-found --wait=false

infrastructure/cert-manager/helmrelease.yaml

@@ -0,0 +1,67 @@
+# infrastructure/cert-manager/helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: cert-manager
+  namespace: cert-manager
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: cert-manager
+      version: v1.17.0
+      sourceRef:
+        kind: HelmRepository
+        name: jetstack
+        namespace: flux-system
+  install:
+    crds: CreateReplace
+    remediation: { retries: 3 }
+    timeout: 10m
+  upgrade:
+    crds: CreateReplace
+    remediation:
+      retries: 3
+      remediateLastFailure: true
+    cleanupOnFail: true
+    timeout: 10m
+  values:
+    installCRDs: true
+    nodeSelector:
+      node-role.kubernetes.io/worker: "true"
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                    - rpi5
+                    - rpi4
+    webhook:
+      nodeSelector:
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+                      - rpi4
+    cainjector:
+      nodeSelector:
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+                      - rpi4

infrastructure/cert-manager/kustomization.yaml

@@ -0,0 +1,6 @@
+# infrastructure/cert-manager/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helmrelease.yaml

infrastructure/cert-manager/namespace.yaml

@@ -0,0 +1,5 @@
+# infrastructure/cert-manager/namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager

infrastructure/core/coredns-custom.yaml

@@ -0,0 +1,47 @@
+# infrastructure/core/coredns-custom.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom
+  namespace: kube-system
+data:
+  bstein-dev.server: |
+    bstein.dev:53 {
+      errors
+      cache 30
+      hosts {
+        192.168.22.9 alerts.bstein.dev
+        192.168.22.9 auth.bstein.dev
+        192.168.22.9 bstein.dev
+        10.43.6.87 budget.bstein.dev
+        192.168.22.9 call.live.bstein.dev
+        192.168.22.9 cd.bstein.dev
+        192.168.22.9 chat.ai.bstein.dev
+        192.168.22.9 ci.bstein.dev
+        192.168.22.9 cloud.bstein.dev
+        192.168.22.9 health.bstein.dev
+        192.168.22.9 kit.live.bstein.dev
+        192.168.22.9 live.bstein.dev
+        192.168.22.9 logs.bstein.dev
+        192.168.22.9 longhorn.bstein.dev
+        192.168.22.4 mail.bstein.dev
+        192.168.22.9 matrix.live.bstein.dev
+        192.168.22.9 metrics.bstein.dev
+        192.168.22.9 monero.bstein.dev
+        10.43.6.87 money.bstein.dev
+        192.168.22.9 notes.bstein.dev
+        192.168.22.9 office.bstein.dev
+        192.168.22.9 pegasus.bstein.dev
+        3.136.224.193 pm-bounces.bstein.dev
+        3.150.68.49 pm-bounces.bstein.dev
+        18.189.137.81 pm-bounces.bstein.dev
+        192.168.22.9 registry.bstein.dev
+        192.168.22.9 scm.bstein.dev
+        192.168.22.9 secret.bstein.dev
+        192.168.22.9 sso.bstein.dev
+        192.168.22.9 stream.bstein.dev
+        192.168.22.9 tasks.bstein.dev
+        192.168.22.9 vault.bstein.dev
+        fallthrough
+      }
+    }

infrastructure/core/coredns-deployment.yaml

@@ -0,0 +1,141 @@
+# infrastructure/core/coredns-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/name: CoreDNS
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 2
+  revisionHistoryLimit: 0
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-dns
+    spec:
+      containers:
+        - name: coredns
+          image: registry.bstein.dev/infra/coredns:1.12.1
+          imagePullPolicy: IfNotPresent
+          args:
+            - -conf
+            - /etc/coredns/Corefile
+          ports:
+            - containerPort: 53
+              name: dns
+              protocol: UDP
+            - containerPort: 53
+              name: dns-tcp
+              protocol: TCP
+            - containerPort: 9153
+              name: metrics
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+              scheme: HTTP
+            initialDelaySeconds: 60
+            periodSeconds: 10
+            timeoutSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: 8181
+              scheme: HTTP
+            periodSeconds: 2
+            timeoutSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+          resources:
+            limits:
+              memory: 170Mi
+            requests:
+              cpu: 100m
+              memory: 70Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              add:
+                - NET_BIND_SERVICE
+              drop:
+                - all
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/coredns
+              readOnly: true
+            - name: custom-config-volume
+              mountPath: /etc/coredns/custom
+              readOnly: true
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+                      - rpi4
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values:
+                      - "true"
+      dnsPolicy: Default
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      serviceAccountName: coredns
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              k8s-app: kube-dns
+        - maxSkew: 1
+          topologyKey: topology.kubernetes.io/zone
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      volumes:
+        - name: config-volume
+          configMap:
+            name: coredns
+            defaultMode: 420
+            items:
+              - key: Corefile
+                path: Corefile
+              - key: NodeHosts
+                path: NodeHosts
+        - name: custom-config-volume
+          configMap:
+            name: coredns-custom
+            optional: true
+            defaultMode: 420

infrastructure/core/kustomization.yaml

@@ -4,5 +4,8 @@ kind: Kustomization
 resources:
   - ../modules/base
   - ../modules/profiles/atlas-ha
+  - coredns-custom.yaml
+  - coredns-deployment.yaml
+  - ntp-sync-daemonset.yaml
   - ../sources/cert-manager/letsencrypt.yaml
   - ../sources/cert-manager/letsencrypt-prod.yaml

infrastructure/core/ntp-sync-daemonset.yaml

@@ -0,0 +1,50 @@
+# infrastructure/core/ntp-sync-daemonset.yaml
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: ntp-sync
+  namespace: kube-system
+  labels:
+    app: ntp-sync
+spec:
+  selector:
+    matchLabels:
+      app: ntp-sync
+  template:
+    metadata:
+      labels:
+        app: ntp-sync
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: DoesNotExist
+                  - key: node-role.kubernetes.io/master
+                    operator: DoesNotExist
+      containers:
+        - name: ntp-sync
+          image: public.ecr.aws/docker/library/busybox:1.36.1
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -eu
+              while true; do
+                ntpd -q -p pool.ntp.org || true
+                sleep 300
+              done
+          securityContext:
+            capabilities:
+              add: ["SYS_TIME"]
+            runAsUser: 0
+            runAsGroup: 0
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              cpu: 50m
+              memory: 64Mi

infrastructure/longhorn/adopt/kustomization.yaml

@@ -0,0 +1,15 @@
+# infrastructure/longhorn/adopt/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - longhorn-adopt-rbac.yaml
+  - longhorn-helm-adopt-job.yaml
+
+configMapGenerator:
+  - name: longhorn-helm-adopt-script
+    namespace: longhorn-system
+    files:
+      - longhorn_helm_adopt.sh=scripts/longhorn_helm_adopt.sh
+    options:
+      disableNameSuffixHash: true

infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml

@@ -0,0 +1,56 @@
+# infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-helm-adopt
+  namespace: longhorn-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: longhorn-helm-adopt
+rules:
+  - apiGroups: [""]
+    resources:
+      - configmaps
+      - services
+      - serviceaccounts
+      - secrets
+    verbs: ["get", "list", "watch", "patch", "update"]
+  - apiGroups: ["apps"]
+    resources:
+      - deployments
+      - daemonsets
+    verbs: ["get", "list", "watch", "patch", "update"]
+  - apiGroups: ["batch"]
+    resources:
+      - jobs
+    verbs: ["get", "list", "watch", "patch", "update"]
+  - apiGroups: ["rbac.authorization.k8s.io"]
+    resources:
+      - roles
+      - rolebindings
+      - clusterroles
+      - clusterrolebindings
+    verbs: ["get", "list", "watch", "patch", "update"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources:
+      - customresourcedefinitions
+    verbs: ["get", "list", "watch", "patch", "update"]
+  - apiGroups: ["scheduling.k8s.io"]
+    resources:
+      - priorityclasses
+    verbs: ["get", "list", "watch", "patch", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: longhorn-helm-adopt
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: longhorn-helm-adopt
+subjects:
+  - kind: ServiceAccount
+    name: longhorn-helm-adopt
+    namespace: longhorn-system

infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml

@@ -0,0 +1,40 @@
+# infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: longhorn-helm-adopt-2
+  namespace: longhorn-system
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      serviceAccountName: longhorn-helm-adopt
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/worker
+                    operator: Exists
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: ["arm64"]
+      containers:
+        - name: adopt
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command: ["/usr/bin/env", "bash"]
+          args: ["/scripts/longhorn_helm_adopt.sh"]
+          volumeMounts:
+            - name: script
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: script
+          configMap:
+            name: longhorn-helm-adopt-script
+            defaultMode: 0555

infrastructure/longhorn/adopt/namespace.yaml

@@ -0,0 +1,5 @@
+# infrastructure/longhorn/adopt/namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system

infrastructure/longhorn/adopt/scripts/longhorn_helm_adopt.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+release_name="longhorn"
+release_namespace="longhorn-system"
+selector="app.kubernetes.io/instance=${release_name}"
+
+annotate_and_label() {
+  local scope="$1"
+  local kind="$2"
+  if [ "${scope}" = "namespaced" ]; then
+    kubectl -n "${release_namespace}" annotate "${kind}" -l "${selector}" \
+      meta.helm.sh/release-name="${release_name}" \
+      meta.helm.sh/release-namespace="${release_namespace}" \
+      --overwrite >/dev/null 2>&1 || true
+    kubectl -n "${release_namespace}" label "${kind}" -l "${selector}" \
+      app.kubernetes.io/managed-by=Helm --overwrite >/dev/null 2>&1 || true
+  else
+    kubectl annotate "${kind}" -l "${selector}" \
+      meta.helm.sh/release-name="${release_name}" \
+      meta.helm.sh/release-namespace="${release_namespace}" \
+      --overwrite >/dev/null 2>&1 || true
+    kubectl label "${kind}" -l "${selector}" \
+      app.kubernetes.io/managed-by=Helm --overwrite >/dev/null 2>&1 || true
+  fi
+}
+
+namespaced_kinds=(
+  configmap
+  service
+  serviceaccount
+  deployment
+  daemonset
+  job
+  role
+  rolebinding
+)
+
+cluster_kinds=(
+  clusterrole
+  clusterrolebinding
+  customresourcedefinition
+  priorityclass
+)
+
+for kind in "${namespaced_kinds[@]}"; do
+  annotate_and_label "namespaced" "${kind}"
+done
+
+for kind in "${cluster_kinds[@]}"; do
+  annotate_and_label "cluster" "${kind}"
+done

infrastructure/longhorn/core/helmrelease.yaml

@@ -0,0 +1,80 @@
+# infrastructure/longhorn/core/helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: longhorn
+      version: 1.8.2
+      sourceRef:
+        kind: HelmRepository
+        name: longhorn
+        namespace: flux-system
+  install:
+    crds: Skip
+    remediation: { retries: 3 }
+    timeout: 15m
+  upgrade:
+    crds: Skip
+    remediation:
+      retries: 3
+      remediateLastFailure: true
+    cleanupOnFail: true
+    timeout: 15m
+  values:
+    service:
+      ui:
+        type: NodePort
+        nodePort: 30824
+    privateRegistry:
+      createSecret: false
+      registrySecret: longhorn-registry
+    image:
+      pullPolicy: Always
+      longhorn:
+        engine:
+          repository: registry.bstein.dev/infra/longhorn-engine
+          tag: v1.8.2
+        manager:
+          repository: registry.bstein.dev/infra/longhorn-manager
+          tag: v1.8.2
+        ui:
+          repository: registry.bstein.dev/infra/longhorn-ui
+          tag: v1.8.2
+        instanceManager:
+          repository: registry.bstein.dev/infra/longhorn-instance-manager
+          tag: v1.8.2
+        shareManager:
+          repository: registry.bstein.dev/infra/longhorn-share-manager
+          tag: v1.8.2
+        backingImageManager:
+          repository: registry.bstein.dev/infra/longhorn-backing-image-manager
+          tag: v1.8.2
+        supportBundleKit:
+          repository: registry.bstein.dev/infra/longhorn-support-bundle-kit
+          tag: v0.0.56
+      csi:
+        attacher:
+          repository: registry.bstein.dev/infra/longhorn-csi-attacher
+          tag: v4.9.0
+        provisioner:
+          repository: registry.bstein.dev/infra/longhorn-csi-provisioner
+          tag: v5.3.0
+        nodeDriverRegistrar:
+          repository: registry.bstein.dev/infra/longhorn-csi-node-driver-registrar
+          tag: v2.14.0
+        resizer:
+          repository: registry.bstein.dev/infra/longhorn-csi-resizer
+          tag: v1.13.2
+        snapshotter:
+          repository: registry.bstein.dev/infra/longhorn-csi-snapshotter
+          tag: v8.2.0
+        livenessProbe:
+          repository: registry.bstein.dev/infra/longhorn-livenessprobe
+          tag: v2.16.0
+    defaultSettings:
+      systemManagedPodsImagePullPolicy: Always

infrastructure/longhorn/core/kustomization.yaml

@@ -0,0 +1,18 @@
+# infrastructure/longhorn/core/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - vault-serviceaccount.yaml
+  - secretproviderclass.yaml
+  - vault-sync-deployment.yaml
+  - helmrelease.yaml
+  - longhorn-settings-ensure-job.yaml
+
+configMapGenerator:
+  - name: longhorn-settings-ensure-script
+    files:
+      - longhorn_settings_ensure.sh=scripts/longhorn_settings_ensure.sh
+
+generatorOptions:
+  disableNameSuffixHash: true

infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml

@@ -0,0 +1,36 @@
+# infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: longhorn-settings-ensure-4
+  namespace: longhorn-system
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      serviceAccountName: longhorn-service-account
+      restartPolicy: Never
+      volumes:
+        - name: longhorn-settings-ensure-script
+          configMap:
+            name: longhorn-settings-ensure-script
+            defaultMode: 0555
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: ["arm64"]
+                  - key: node-role.kubernetes.io/worker
+                    operator: Exists
+      containers:
+        - name: apply
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command: ["/scripts/longhorn_settings_ensure.sh"]
+          volumeMounts:
+            - name: longhorn-settings-ensure-script
+              mountPath: /scripts
+              readOnly: true

infrastructure/longhorn/core/namespace.yaml

@@ -0,0 +1,5 @@
+# infrastructure/longhorn/core/namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system

infrastructure/longhorn/core/scripts/longhorn_settings_ensure.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env sh
+set -eu
+
+# Longhorn blocks direct CR patches for some settings; use the internal API instead.
+
+api_base="http://longhorn-backend.longhorn-system.svc:9500/v1/settings"
+
+wait_for_api() {
+  attempts=30
+  while [ "${attempts}" -gt 0 ]; do
+    if curl -fsS "${api_base}" >/dev/null 2>&1; then
+      return 0
+    fi
+    attempts=$((attempts - 1))
+    sleep 2
+  done
+  echo "Longhorn API not ready after retries." >&2
+  return 1
+}
+
+update_setting() {
+  name="$1"
+  value="$2"
+
+  current="$(curl -fsS "${api_base}/${name}" || true)"
+  if echo "${current}" | grep -Fq "\"value\":\"${value}\""; then
+    echo "Setting ${name} already set."
+    return 0
+  fi
+
+  echo "Setting ${name} -> ${value}"
+  curl -fsS -X PUT \
+    -H "Content-Type: application/json" \
+    -d "{\"value\":\"${value}\"}" \
+    "${api_base}/${name}" >/dev/null
+}
+
+wait_for_api
+update_setting default-engine-image "registry.bstein.dev/infra/longhorn-engine:v1.8.2"
+update_setting default-instance-manager-image "registry.bstein.dev/infra/longhorn-instance-manager:v1.8.2"
+update_setting default-backing-image-manager-image "registry.bstein.dev/infra/longhorn-backing-image-manager:v1.8.2"
+update_setting support-bundle-manager-image "registry.bstein.dev/infra/longhorn-support-bundle-kit:v0.0.56"

infrastructure/longhorn/core/secretproviderclass.yaml

@@ -0,0 +1,21 @@
+# infrastructure/longhorn/core/secretproviderclass.yaml
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: longhorn-vault
+  namespace: longhorn-system
+spec:
+  provider: vault
+  parameters:
+    vaultAddress: "http://vault.vault.svc.cluster.local:8200"
+    roleName: "longhorn"
+    objects: |
+      - objectName: "harbor-pull__dockerconfigjson"
+        secretPath: "kv/data/atlas/shared/harbor-pull"
+        secretKey: "dockerconfigjson"
+  secretObjects:
+    - secretName: longhorn-registry
+      type: kubernetes.io/dockerconfigjson
+      data:
+        - objectName: harbor-pull__dockerconfigjson
+          key: .dockerconfigjson

infrastructure/longhorn/core/vault-serviceaccount.yaml

@@ -0,0 +1,6 @@
+# infrastructure/longhorn/core/vault-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-vault-sync
+  namespace: longhorn-system

infrastructure/longhorn/core/vault-sync-deployment.yaml

@@ -0,0 +1,45 @@
+# infrastructure/longhorn/core/vault-sync-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: longhorn-vault-sync
+  namespace: longhorn-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: longhorn-vault-sync
+  template:
+    metadata:
+      labels:
+        app: longhorn-vault-sync
+    spec:
+      serviceAccountName: longhorn-vault-sync
+      nodeSelector:
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 80
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: ["rpi5", "rpi4"]
+      containers:
+        - name: sync
+          image: alpine:3.20
+          command: ["/bin/sh", "-c"]
+          args:
+            - "sleep infinity"
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: longhorn-vault

infrastructure/longhorn/ui-ingress/kustomization.yaml

@@ -2,6 +2,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - serviceaccount.yaml
+  - oauth2-proxy-longhorn.yaml
   - middleware.yaml
   - ingress.yaml
-  - oauth2-proxy-longhorn.yaml

infrastructure/longhorn/ui-ingress/oauth2-proxy-longhorn.yaml

@@ -32,7 +32,18 @@ spec:
     metadata:
       labels:
         app: oauth2-proxy-longhorn
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: "longhorn"
+        vault.hashicorp.com/agent-inject-secret-oidc-config: "kv/data/atlas/longhorn/oauth2-proxy"
+        vault.hashicorp.com/agent-inject-template-oidc-config: |
+          {{- with secret "kv/data/atlas/longhorn/oauth2-proxy" -}}
+          client_id = "{{ .Data.data.client_id }}"
+          client_secret = "{{ .Data.data.client_secret }}"
+          cookie_secret = "{{ .Data.data.cookie_secret }}"
+          {{- end -}}
     spec:
+      serviceAccountName: longhorn-vault
       nodeSelector:
         node-role.kubernetes.io/worker: "true"
       affinity:
@@ -50,6 +61,7 @@ spec:
           imagePullPolicy: IfNotPresent
           args:
             - --provider=oidc
+            - --config=/vault/secrets/oidc-config
             - --redirect-url=https://longhorn.bstein.dev/oauth2/callback
             - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas
             - --scope=openid profile email groups
@@ -69,22 +81,6 @@ spec:
             - --skip-jwt-bearer-tokens=true
             - --oidc-groups-claim=groups
             - --cookie-domain=longhorn.bstein.dev
-          env:
-            - name: OAUTH2_PROXY_CLIENT_ID
-              valueFrom:
-                secretKeyRef:
-                  name: oauth2-proxy-longhorn-oidc
-                  key: client_id
-            - name: OAUTH2_PROXY_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: oauth2-proxy-longhorn-oidc
-                  key: client_secret
-            - name: OAUTH2_PROXY_COOKIE_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: oauth2-proxy-longhorn-oidc
-                  key: cookie_secret
           ports:
             - containerPort: 4180
               name: http

infrastructure/longhorn/ui-ingress/serviceaccount.yaml

@@ -0,0 +1,6 @@
+# infrastructure/longhorn/ui-ingress/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-vault
+  namespace: longhorn-system

infrastructure/metallb/helmrelease.yaml

@@ -0,0 +1,47 @@
+# infrastructure/metallb/helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: metallb
+  namespace: metallb-system
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: metallb
+      version: 0.15.3
+      sourceRef:
+        kind: HelmRepository
+        name: metallb
+        namespace: flux-system
+  install:
+    crds: CreateReplace
+    remediation: { retries: 3 }
+    timeout: 10m
+  upgrade:
+    crds: CreateReplace
+    remediation:
+      retries: 3
+      remediateLastFailure: true
+    cleanupOnFail: true
+    timeout: 10m
+  values:
+    loadBalancerClass: metallb
+    prometheus:
+      metricsPort: 7472
+    controller:
+      logLevel: info
+      webhookMode: enabled
+      tlsMinVersion: VersionTLS12
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi4
+                      - rpi5
+    speaker:
+      logLevel: info

infrastructure/metallb/kustomization.yaml

@@ -3,8 +3,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - namespace.yaml
-  - metallb-rendered.yaml
+  - helmrelease.yaml
   - ippool.yaml
-patchesStrategicMerge:
-  - patches/node-placement.yaml
-  - patches/speaker-loglevel.yaml

infrastructure/metallb/patches/node-placement.yaml

@@ -1,27 +0,0 @@
-# infrastructure/metallb/patches/node-placement.yaml
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: metallb-controller
-  namespace: metallb-system
-spec:
-  template:
-    spec:
-      containers:
-        - name: controller
-          args:
-            - --port=7472
-            - --log-level=info
-            - --webhook-mode=enabled
-            - --tls-min-version=VersionTLS12
-            - --lb-class=metallb
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: hardware
-                    operator: In
-                    values:
-                      - rpi4
-                      - rpi5

infrastructure/metallb/patches/speaker-loglevel.yaml

@@ -1,15 +0,0 @@
-# infrastructure/metallb/patches/speaker-loglevel.yaml
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: metallb-speaker
-  namespace: metallb-system
-spec:
-  template:
-    spec:
-      containers:
-        - name: speaker
-          args:
-            - --port=7472
-            - --log-level=info
-            - --lb-class=metallb

infrastructure/modules/base/storageclass/asteria-encrypted.yaml

@@ -0,0 +1,24 @@
+# infrastructure/modules/base/storageclass/asteria-encrypted.yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: asteria-encrypted
+parameters:
+  diskSelector: asteria
+  fromBackup: ""
+  numberOfReplicas: "2"
+  staleReplicaTimeout: "30"
+  fsType: "ext4"
+  replicaAutoBalance: "least-effort"
+  dataLocality: "disabled"
+  encrypted: "true"
+  csi.storage.k8s.io/provisioner-secret-name: ${pvc.name}
+  csi.storage.k8s.io/provisioner-secret-namespace: ${pvc.namespace}
+  csi.storage.k8s.io/node-publish-secret-name: ${pvc.name}
+  csi.storage.k8s.io/node-publish-secret-namespace: ${pvc.namespace}
+  csi.storage.k8s.io/node-stage-secret-name: ${pvc.name}
+  csi.storage.k8s.io/node-stage-secret-namespace: ${pvc.namespace}
+provisioner: driver.longhorn.io
+reclaimPolicy: Retain
+allowVolumeExpansion: true
+volumeBindingMode: Immediate

infrastructure/modules/base/storageclass/kustomization.yaml

@@ -3,4 +3,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - asteria.yaml
+  - asteria-encrypted.yaml
   - astreae.yaml

infrastructure/postgres/kustomization.yaml

@@ -1,4 +1,4 @@
-# services/postgres/kustomization.yaml
+# infrastructure/postgres/kustomization.yaml
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: postgres

infrastructure/postgres/namespace.yaml

@@ -1,4 +1,4 @@
-# services/postgres/namespace.yaml
+# infrastructure/postgres/namespace.yaml
 apiVersion: v1
 kind: Namespace
 metadata:

infrastructure/postgres/secretproviderclass.yaml

@@ -1,4 +1,4 @@
-# services/postgres/secretproviderclass.yaml
+# infrastructure/postgres/secretproviderclass.yaml
 apiVersion: secrets-store.csi.x-k8s.io/v1
 kind: SecretProviderClass
 metadata:
@@ -11,5 +11,5 @@ spec:
     roleName: "postgres"
     objects: |
       - objectName: "postgres_password"
-        secretPath: "kv/data/postgres"
+        secretPath: "kv/data/atlas/postgres/postgres-db"
         secretKey: "POSTGRES_PASSWORD"

infrastructure/postgres/service.yaml

@@ -0,0 +1,23 @@
+# infrastructure/postgres/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: postgres-service
+  namespace: postgres
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "9187"
+    prometheus.io/path: "/metrics"
+spec:
+  clusterIP: None
+  ports:
+    - name: postgres
+      port: 5432
+      protocol: TCP
+      targetPort: 5432
+    - name: metrics
+      port: 9187
+      protocol: TCP
+      targetPort: 9187
+  selector:
+    app: postgres

infrastructure/postgres/serviceaccount.yaml

@@ -1,4 +1,4 @@
-# services/postgres/serviceaccount.yaml
+# infrastructure/postgres/serviceaccount.yaml
 apiVersion: v1
 kind: ServiceAccount
 metadata:

infrastructure/postgres/statefulset.yaml

@@ -1,4 +1,4 @@
-# services/postgres/statefulset.yaml
+# infrastructure/postgres/statefulset.yaml
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
@@ -58,6 +58,23 @@ spec:
             - name: vault-secrets
               mountPath: /mnt/vault
               readOnly: true
+        - name: postgres-exporter
+          image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0
+          ports:
+            - name: metrics
+              containerPort: 9187
+              protocol: TCP
+          env:
+            - name: DATA_SOURCE_URI
+              value: "localhost:5432/postgres?sslmode=disable"
+            - name: DATA_SOURCE_USER
+              value: postgres
+            - name: DATA_SOURCE_PASS_FILE
+              value: /mnt/vault/postgres_password
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /mnt/vault
+              readOnly: true
       volumes:
         - name: vault-secrets
           csi:

infrastructure/sources/cert-manager/letsencrypt-prod.yaml

@@ -1,10 +1,11 @@
+# infrastructure/sources/cert-manager/letsencrypt-prod.yaml
 apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
 metadata:
   name: letsencrypt-prod
 spec:
   acme:
-    email: brad.stein@gmail.com
+    email: brad@bstein.dev
     server: https://acme-v02.api.letsencrypt.org/directory
     privateKeySecretRef:
       name: letsencrypt-prod-account-key

infrastructure/sources/cert-manager/letsencrypt.yaml

@@ -1,10 +1,11 @@
+# infrastructure/sources/cert-manager/letsencrypt.yaml
 apiVersion: cert-manager.io/v1
 kind: ClusterIssuer
 metadata:
   name: letsencrypt
 spec:
   acme:
-    email: brad.stein@gmail.com
+    email: brad@bstein.dev
     server: https://acme-v02.api.letsencrypt.org/directory
     privateKeySecretRef:
       name: letsencrypt-account-key

infrastructure/sources/helm/ananace.yaml

@@ -0,0 +1,9 @@
+# infrastructure/sources/helm/ananace.yaml
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: ananace
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://ananace.gitlab.io/charts

infrastructure/sources/helm/kustomization.yaml

@@ -2,15 +2,18 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - ananace.yaml
   - fluent-bit.yaml
   - grafana.yaml
   - hashicorp.yaml
   - jetstack.yaml
   - jenkins.yaml
   - mailu.yaml
+  - metallb.yaml
   - opentelemetry.yaml
   - opensearch.yaml
   - harbor.yaml
+  - longhorn.yaml
   - prometheus.yaml
   - victoria-metrics.yaml
   - secrets-store-csi.yaml

infrastructure/sources/helm/longhorn.yaml

@@ -0,0 +1,9 @@
+# infrastructure/sources/helm/longhorn.yaml
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: longhorn
+  namespace: flux-system
+spec:
+  interval: 30m
+  url: https://charts.longhorn.io

infrastructure/sources/helm/metallb.yaml

@@ -0,0 +1,9 @@
+# infrastructure/sources/helm/metallb.yaml
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: metallb
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://metallb.github.io/metallb

infrastructure/traefik/deployment.yaml

@@ -27,6 +27,8 @@ items:
         creationTimestamp: null
         labels:
           app: traefik
+          app.kubernetes.io/instance: traefik-kube-system
+          app.kubernetes.io/name: traefik
       spec:
         containers:
         - args:

infrastructure/traefik/kustomization.yaml

@@ -5,6 +5,7 @@ metadata:
   name: traefik
   namespace: flux-system
 resources:
+  - crds.yaml
   - deployment.yaml
   - serviceaccount.yaml
   - clusterrole.yaml

infrastructure/traefik/traefik-service-lb.yaml

@@ -3,9 +3,10 @@ apiVersion: v1
 kind: Service
 metadata:
   name: traefik
-  namespace: kube-system
+  namespace: traefik
   annotations:
     metallb.universe.tf/address-pool: communication-pool
+    metallb.universe.tf/allow-shared-ip: traefik
 spec:
   type: LoadBalancer
   loadBalancerClass: metallb
@@ -20,5 +21,4 @@ spec:
       targetPort: websecure
       protocol: TCP
   selector:
-    app.kubernetes.io/instance: traefik-kube-system
-    app.kubernetes.io/name: traefik
+    app: traefik

infrastructure/vault-csi/secrets-store-csi-driver.yaml

@@ -17,4 +17,5 @@ spec:
   values:
     syncSecret:
       enabled: true
-    enableSecretRotation: false
+    enableSecretRotation: true
+    rotationPollInterval: 2m

infrastructure/vault-injector/helmrelease.yaml

@@ -0,0 +1,43 @@
+# infrastructure/vault-injector/helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: vault-injector
+  namespace: vault
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: vault
+      version: 0.31.0
+      sourceRef:
+        kind: HelmRepository
+        name: hashicorp
+        namespace: flux-system
+  install:
+    remediation: { retries: 3 }
+    timeout: 10m
+  upgrade:
+    remediation:
+      retries: 3
+      remediateLastFailure: true
+    cleanupOnFail: true
+    timeout: 10m
+  values:
+    global:
+      externalVaultAddr: http://vault.vault.svc.cluster.local:8200
+      tlsDisable: true
+    server:
+      enabled: false
+    csi:
+      enabled: false
+    injector:
+      enabled: true
+      replicas: 1
+      agentImage:
+        repository: hashicorp/vault
+        tag: "1.17.6"
+      webhook:
+        failurePolicy: Ignore
+      nodeSelector:
+        node-role.kubernetes.io/worker: "true"

infrastructure/vault-injector/kustomization.yaml

@@ -0,0 +1,5 @@
+# infrastructure/vault-injector/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - helmrelease.yaml

knowledge/catalog/atlas-summary.json

@@ -1,8 +1,8 @@
 {
   "counts": {
-    "helmrelease_host_hints": 7,
-    "http_endpoints": 35,
-    "services": 44,
-    "workloads": 49
+    "helmrelease_host_hints": 19,
+    "http_endpoints": 45,
+    "services": 47,
+    "workloads": 74
   }
 }

knowledge/diagrams/atlas-http.mmd

@@ -17,6 +17,11 @@ flowchart LR
   host_bstein_dev --> svc_bstein_dev_home_bstein_dev_home_backend
   wl_bstein_dev_home_bstein_dev_home_backend["bstein-dev-home/bstein-dev-home-backend (Deployment)"]
   svc_bstein_dev_home_bstein_dev_home_backend --> wl_bstein_dev_home_bstein_dev_home_backend
+  host_budget_bstein_dev["budget.bstein.dev"]
+  svc_finance_actual_budget["finance/actual-budget (Service)"]
+  host_budget_bstein_dev --> svc_finance_actual_budget
+  wl_finance_actual_budget["finance/actual-budget (Deployment)"]
+  svc_finance_actual_budget --> wl_finance_actual_budget
   host_call_live_bstein_dev["call.live.bstein.dev"]
   svc_comms_element_call["comms/element-call (Service)"]
   host_call_live_bstein_dev --> svc_comms_element_call
@@ -37,6 +42,11 @@ flowchart LR
   host_cloud_bstein_dev --> svc_nextcloud_nextcloud
   wl_nextcloud_nextcloud["nextcloud/nextcloud (Deployment)"]
   svc_nextcloud_nextcloud --> wl_nextcloud_nextcloud
+  host_health_bstein_dev["health.bstein.dev"]
+  svc_health_wger["health/wger (Service)"]
+  host_health_bstein_dev --> svc_health_wger
+  wl_health_wger["health/wger (Deployment)"]
+  svc_health_wger --> wl_health_wger
   host_kit_live_bstein_dev["kit.live.bstein.dev"]
   svc_comms_livekit_token_service["comms/livekit-token-service (Service)"]
   host_kit_live_bstein_dev --> svc_comms_livekit_token_service
@@ -47,15 +57,22 @@ flowchart LR
   wl_comms_livekit["comms/livekit (Deployment)"]
   svc_comms_livekit --> wl_comms_livekit
   host_live_bstein_dev["live.bstein.dev"]
-  svc_comms_othrys_element_element_web["comms/othrys-element-element-web (Service)"]
-  host_live_bstein_dev --> svc_comms_othrys_element_element_web
-  wl_comms_othrys_element_element_web["comms/othrys-element-element-web (Deployment)"]
-  svc_comms_othrys_element_element_web --> wl_comms_othrys_element_element_web
   host_live_bstein_dev --> svc_comms_matrix_wellknown
   svc_comms_othrys_synapse_matrix_synapse["comms/othrys-synapse-matrix-synapse (Service)"]
   host_live_bstein_dev --> svc_comms_othrys_synapse_matrix_synapse
-  wl_comms_othrys_synapse_matrix_synapse["comms/othrys-synapse-matrix-synapse (Deployment)"]
-  svc_comms_othrys_synapse_matrix_synapse --> wl_comms_othrys_synapse_matrix_synapse
+  svc_comms_matrix_guest_register["comms/matrix-guest-register (Service)"]
+  host_live_bstein_dev --> svc_comms_matrix_guest_register
+  wl_comms_matrix_guest_register["comms/matrix-guest-register (Deployment)"]
+  svc_comms_matrix_guest_register --> wl_comms_matrix_guest_register
+  svc_comms_matrix_authentication_service["comms/matrix-authentication-service (Service)"]
+  host_live_bstein_dev --> svc_comms_matrix_authentication_service
+  wl_comms_matrix_authentication_service["comms/matrix-authentication-service (Deployment)"]
+  svc_comms_matrix_authentication_service --> wl_comms_matrix_authentication_service
+  host_logs_bstein_dev["logs.bstein.dev"]
+  svc_logging_oauth2_proxy_logs["logging/oauth2-proxy-logs (Service)"]
+  host_logs_bstein_dev --> svc_logging_oauth2_proxy_logs
+  wl_logging_oauth2_proxy_logs["logging/oauth2-proxy-logs (Deployment)"]
+  svc_logging_oauth2_proxy_logs --> wl_logging_oauth2_proxy_logs
   host_longhorn_bstein_dev["longhorn.bstein.dev"]
   svc_longhorn_system_oauth2_proxy_longhorn["longhorn-system/oauth2-proxy-longhorn (Service)"]
   host_longhorn_bstein_dev --> svc_longhorn_system_oauth2_proxy_longhorn
@@ -65,21 +82,25 @@ flowchart LR
   svc_mailu_mailserver_mailu_front["mailu-mailserver/mailu-front (Service)"]
   host_mail_bstein_dev --> svc_mailu_mailserver_mailu_front
   host_matrix_live_bstein_dev["matrix.live.bstein.dev"]
-  svc_comms_matrix_authentication_service["comms/matrix-authentication-service (Service)"]
   host_matrix_live_bstein_dev --> svc_comms_matrix_authentication_service
-  wl_comms_matrix_authentication_service["comms/matrix-authentication-service (Deployment)"]
-  svc_comms_matrix_authentication_service --> wl_comms_matrix_authentication_service
   host_matrix_live_bstein_dev --> svc_comms_matrix_wellknown
   host_matrix_live_bstein_dev --> svc_comms_othrys_synapse_matrix_synapse
-  svc_comms_matrix_guest_register["comms/matrix-guest-register (Service)"]
   host_matrix_live_bstein_dev --> svc_comms_matrix_guest_register
-  wl_comms_matrix_guest_register["comms/matrix-guest-register (Deployment)"]
-  svc_comms_matrix_guest_register --> wl_comms_matrix_guest_register
   host_monero_bstein_dev["monero.bstein.dev"]
   svc_crypto_monerod["crypto/monerod (Service)"]
   host_monero_bstein_dev --> svc_crypto_monerod
   wl_crypto_monerod["crypto/monerod (Deployment)"]
   svc_crypto_monerod --> wl_crypto_monerod
+  host_money_bstein_dev["money.bstein.dev"]
+  svc_finance_firefly["finance/firefly (Service)"]
+  host_money_bstein_dev --> svc_finance_firefly
+  wl_finance_firefly["finance/firefly (Deployment)"]
+  svc_finance_firefly --> wl_finance_firefly
+  host_notes_bstein_dev["notes.bstein.dev"]
+  svc_outline_outline["outline/outline (Service)"]
+  host_notes_bstein_dev --> svc_outline_outline
+  wl_outline_outline["outline/outline (Deployment)"]
+  svc_outline_outline --> wl_outline_outline
   host_office_bstein_dev["office.bstein.dev"]
   svc_nextcloud_collabora["nextcloud/collabora (Service)"]
   host_office_bstein_dev --> svc_nextcloud_collabora
@@ -110,6 +131,11 @@ flowchart LR
   host_stream_bstein_dev --> svc_jellyfin_jellyfin
   wl_jellyfin_jellyfin["jellyfin/jellyfin (Deployment)"]
   svc_jellyfin_jellyfin --> wl_jellyfin_jellyfin
+  host_tasks_bstein_dev["tasks.bstein.dev"]
+  svc_planka_planka["planka/planka (Service)"]
+  host_tasks_bstein_dev --> svc_planka_planka
+  wl_planka_planka["planka/planka (Deployment)"]
+  svc_planka_planka --> wl_planka_planka
   host_vault_bstein_dev["vault.bstein.dev"]
   svc_vaultwarden_vaultwarden_service["vaultwarden/vaultwarden-service (Service)"]
   host_vault_bstein_dev --> svc_vaultwarden_vaultwarden_service
@@ -133,23 +159,30 @@ flowchart LR
     wl_comms_livekit_token_service
     svc_comms_livekit
     wl_comms_livekit
-    svc_comms_othrys_element_element_web
-    wl_comms_othrys_element_element_web
     svc_comms_othrys_synapse_matrix_synapse
-    wl_comms_othrys_synapse_matrix_synapse
-    svc_comms_matrix_authentication_service
-    wl_comms_matrix_authentication_service
     svc_comms_matrix_guest_register
     wl_comms_matrix_guest_register
+    svc_comms_matrix_authentication_service
+    wl_comms_matrix_authentication_service
   end
   subgraph crypto[crypto]
     svc_crypto_monerod
     wl_crypto_monerod
   end
+  subgraph finance[finance]
+    svc_finance_actual_budget
+    wl_finance_actual_budget
+    svc_finance_firefly
+    wl_finance_firefly
+  end
   subgraph gitea[gitea]
     svc_gitea_gitea
     wl_gitea_gitea
   end
+  subgraph health[health]
+    svc_health_wger
+    wl_health_wger
+  end
   subgraph jellyfin[jellyfin]
     svc_jellyfin_pegasus
     wl_jellyfin_pegasus
@@ -160,6 +193,10 @@ flowchart LR
     svc_jenkins_jenkins
     wl_jenkins_jenkins
   end
+  subgraph logging[logging]
+    svc_logging_oauth2_proxy_logs
+    wl_logging_oauth2_proxy_logs
+  end
   subgraph longhorn_system[longhorn-system]
     svc_longhorn_system_oauth2_proxy_longhorn
     wl_longhorn_system_oauth2_proxy_longhorn
@@ -173,6 +210,14 @@ flowchart LR
     svc_nextcloud_collabora
     wl_nextcloud_collabora
   end
+  subgraph outline[outline]
+    svc_outline_outline
+    wl_outline_outline
+  end
+  subgraph planka[planka]
+    svc_planka_planka
+    wl_planka_planka
+  end
   subgraph sso[sso]
     svc_sso_oauth2_proxy
     wl_sso_oauth2_proxy

scripts/dashboards_render_atlas.py

@@ -70,6 +70,7 @@ WORKER_NODES = [
     "titan-13",
     "titan-14",
     "titan-15",
+    "titan-16",
     "titan-17",
     "titan-18",
     "titan-19",
@@ -85,19 +86,17 @@ WORKER_TOTAL = len(WORKER_NODES)
 CONTROL_SUFFIX = f"/{CONTROL_TOTAL}"
 WORKER_SUFFIX = f"/{WORKER_TOTAL}"
 # Namespaces considered infrastructure (excluded from workload counts)
-INFRA_NAMESPACES = [
-    "kube-system",
-    "longhorn-system",
-    "metallb-system",
+INFRA_PATTERNS = [
+    "kube-.*",
+    ".*-system",
+    "traefik",
     "monitoring",
     "logging",
     "cert-manager",
-    "flux-system",
-    "traefik",
     "maintenance",
     "postgres",
 ]
-INFRA_REGEX = f"^({'|'.join(INFRA_NAMESPACES)})$"
+INFRA_REGEX = f"^({'|'.join(INFRA_PATTERNS)})$"
 # Namespaces allowed on control plane without counting as workloads
 CP_ALLOWED_NS = INFRA_REGEX
 LONGHORN_NODE_REGEX = "titan-1[2-9]|titan-2[24]"
@@ -209,7 +208,66 @@ def namespace_ram_raw(scope_var):
 
 
 def namespace_gpu_usage_instant(scope_var):
-    return f"sum(DCGM_FI_DEV_GPU_UTIL{{{namespace_gpu_selector(scope_var)}}}) by (namespace)"
+    return gpu_usage_by_namespace(scope_var)
+
+
+def jetson_gpu_util_by_node():
+    return 'max by (node) (jetson_gr3d_freq_percent{node!=""})'
+
+
+def dcgm_gpu_util_by_node():
+    dcgm_pod = 'label_replace(DCGM_FI_DEV_GPU_UTIL, "pod", "$1", "Hostname", "(.*)")'
+    dcgm_ns = 'label_replace(' + dcgm_pod + ', "namespace", "monitoring", "", "")'
+    return (
+        "avg by (node) ("
+        f"{dcgm_ns} * on(namespace,pod) group_left(node) "
+        'kube_pod_info{namespace="monitoring"}'
+        ")"
+    )
+
+
+def gpu_util_by_node():
+    return f"{dcgm_gpu_util_by_node()} or {jetson_gpu_util_by_node()}"
+
+
+def gpu_util_by_hostname():
+    return 'label_replace(' + gpu_util_by_node() + ', "Hostname", "$1", "node", "(.*)")'
+
+
+def gpu_node_labels():
+    return 'kube_node_labels{label_accelerator=~".+"} or kube_node_labels{label_jetson="true"}'
+
+
+def gpu_requests_by_namespace_node(scope_var):
+    return (
+        "sum by (namespace,node) ("
+        f'kube_pod_container_resource_requests{{resource=~"nvidia.com/gpu.*",{scope_var}}} '
+        "* on(namespace,pod) group_left(node) kube_pod_info "
+        f"* on(node) group_left() ({gpu_node_labels()})"
+        ")"
+    )
+
+
+def gpu_usage_by_namespace(scope_var):
+    requests_by_ns = gpu_requests_by_namespace_node(scope_var)
+    total_by_node = f"sum by (node) ({requests_by_ns})"
+    return (
+        "sum by (namespace) ("
+        f"({requests_by_ns}) / clamp_min({total_by_node}, 1) "
+        f"* on(node) group_left() ({gpu_util_by_node()})"
+        ")"
+    )
+
+
+def jetson_gpu_usage_by_namespace(scope_var):
+    requests_by_ns = jetson_gpu_requests(scope_var)
+    total_by_node = f"sum by (node) ({requests_by_ns})"
+    return (
+        "sum by (namespace) ("
+        f"({requests_by_ns}) / clamp_min({total_by_node}, 1) "
+        f"* on(node) group_left() {jetson_gpu_util_by_node()}"
+        ")"
+    )
 
 
 def namespace_share_expr(resource_expr):
@@ -229,7 +287,7 @@ def namespace_gpu_share_expr(scope_var):
     usage = namespace_gpu_usage_instant(scope_var)
     total = f"(sum({usage}) or on() vector(0))"
     share = f"100 * ({usage}) / clamp_min({total}, 1)"
-    idle = 'label_replace(vector(100), "namespace", "idle", "", "") and on() (' + total + " == 0)"
+    idle = 'label_replace(vector(100), "namespace", "idle", "", "") * scalar(' + total + " == bool 0)"
     return f"({share}) or ({idle})"
 
 
@@ -319,6 +377,76 @@ NAMESPACE_SCOPE_WORKLOAD = f'namespace!~"{INFRA_REGEX}"'
 NAMESPACE_SCOPE_ALL = 'namespace=~".*"'
 NAMESPACE_SCOPE_INFRA = f'namespace=~"{INFRA_REGEX}"'
 NAMESPACE_SCOPE_VARS = ["namespace_scope_cpu", "namespace_scope_gpu", "namespace_scope_ram"]
+GLUE_LABEL = 'label_atlas_bstein_dev_glue="true"'
+GLUE_JOBS = f"kube_cronjob_labels{{{GLUE_LABEL}}}"
+GLUE_FILTER = f"and on(namespace,cronjob) {GLUE_JOBS}"
+GLUE_LAST_SUCCESS = f"(kube_cronjob_status_last_successful_time {GLUE_FILTER})"
+GLUE_LAST_SCHEDULE = f"(kube_cronjob_status_last_schedule_time {GLUE_FILTER})"
+GLUE_SUSPENDED = f"(kube_cronjob_spec_suspend {GLUE_FILTER}) == 1"
+GLUE_ACTIVE = f"(kube_cronjob_status_active {GLUE_FILTER})"
+GLUE_LAST_SUCCESS_AGE = f"(time() - {GLUE_LAST_SUCCESS})"
+GLUE_LAST_SCHEDULE_AGE = f"(time() - {GLUE_LAST_SCHEDULE})"
+GLUE_LAST_SUCCESS_AGE_HOURS = f"({GLUE_LAST_SUCCESS_AGE}) / 3600"
+GLUE_LAST_SCHEDULE_AGE_HOURS = f"({GLUE_LAST_SCHEDULE_AGE}) / 3600"
+GLUE_STALE_WINDOW_SEC = 36 * 3600
+GLUE_STALE = f"({GLUE_LAST_SUCCESS_AGE} > bool {GLUE_STALE_WINDOW_SEC})"
+GLUE_MISSING = f"({GLUE_JOBS} unless on(namespace,cronjob) kube_cronjob_status_last_successful_time)"
+GLUE_STALE_ACTIVE = f"({GLUE_STALE} unless on(namespace,cronjob) {GLUE_SUSPENDED})"
+GLUE_MISSING_ACTIVE = f"({GLUE_MISSING} unless on(namespace,cronjob) {GLUE_SUSPENDED})"
+GLUE_STALE_COUNT = f"(sum({GLUE_STALE_ACTIVE}) + count({GLUE_MISSING_ACTIVE})) or on() vector(0)"
+GLUE_MISSING_COUNT = f"count({GLUE_MISSING_ACTIVE}) or on() vector(0)"
+GLUE_SUSPENDED_COUNT = f"sum({GLUE_SUSPENDED}) or on() vector(0)"
+ARIADNE_TASK_ERRORS_RANGE = 'sum by (task) (increase(ariadne_task_runs_total{status="error"}[$__range]))'
+ARIADNE_TASK_ERRORS_24H = 'sum by (task) (increase(ariadne_task_runs_total{status="error"}[24h]))'
+ARIADNE_TASK_ERRORS_1H = 'sum by (task) (increase(ariadne_task_runs_total{status="error"}[1h]))'
+ARIADNE_TASK_ERRORS_30D = 'sum by (task) (increase(ariadne_task_runs_total{status="error"}[30d]))'
+ARIADNE_TASK_SUCCESS_24H = 'sum by (task) (increase(ariadne_task_runs_total{status="ok"}[24h]))'
+ARIADNE_TASK_RUNS_BY_STATUS_1H = 'sum by (status) (increase(ariadne_task_runs_total[1h]))'
+ARIADNE_TASK_ERRORS_1H_TOTAL = 'sum(increase(ariadne_task_runs_total{status="error"}[1h]))'
+ARIADNE_TASK_ERRORS_24H_TOTAL = 'sum(increase(ariadne_task_runs_total{status="error"}[24h]))'
+ARIADNE_TASK_RUNS_1H_TOTAL = 'sum(increase(ariadne_task_runs_total[1h]))'
+ARIADNE_TASK_ATTEMPTS_SERIES = 'sum(increase(ariadne_task_runs_total[$__interval]))'
+ARIADNE_TASK_FAILURES_SERIES = 'sum(increase(ariadne_task_runs_total{status="error"}[$__interval]))'
+ARIADNE_TASK_WARNINGS_SERIES = (
+    'sum(increase(ariadne_task_runs_total{status!~"ok|error"}[$__interval])) or on() vector(0)'
+)
+ARIADNE_SCHEDULE_LAST_SUCCESS_HOURS = "(time() - ariadne_schedule_last_success_timestamp_seconds) / 3600"
+ARIADNE_SCHEDULE_LAST_ERROR_HOURS = "(time() - ariadne_schedule_last_error_timestamp_seconds) / 3600"
+ARIADNE_SCHEDULE_LAST_SUCCESS_RANGE_HOURS = (
+    "(time() - max_over_time(ariadne_schedule_last_success_timestamp_seconds[$__range])) / 3600"
+)
+ARIADNE_SCHEDULE_LAST_ERROR_RANGE_HOURS = (
+    "(time() - max_over_time(ariadne_schedule_last_error_timestamp_seconds[$__range])) / 3600"
+)
+ARIADNE_ACCESS_REQUESTS = "ariadne_access_requests_total"
+ARIADNE_CI_COVERAGE = 'ariadne_ci_coverage_percent{repo="ariadne"}'
+ARIADNE_CI_TESTS = 'ariadne_ci_tests_total{repo="ariadne"}'
+ARIADNE_TEST_SUCCESS_RATE = (
+    "100 * "
+    'sum(max_over_time(ariadne_ci_tests_total{repo="ariadne",result="passed"}[30d])) '
+    "/ clamp_min("
+    'sum(max_over_time(ariadne_ci_tests_total{repo="ariadne",result=~"passed|failed|error"}[30d])), 1)'
+)
+ARIADNE_TEST_FAILURES_24H = (
+    'sum by (result) (max_over_time(ariadne_ci_tests_total{repo="ariadne",result=~"failed|error"}[24h]))'
+)
+POSTGRES_CONN_USED = (
+    'label_replace(sum(pg_stat_activity_count), "conn", "used", "__name__", ".*") '
+    'or label_replace(max(pg_settings_max_connections), "conn", "max", "__name__", ".*")'
+)
+POSTGRES_CONN_HOTTEST = 'topk(1, sum by (datname) (pg_stat_activity_count))'
+ONEOFF_JOB_OWNER = (
+    'label_replace(kube_job_owner{owner_kind="CronJob"}, "owner_name", "$1", "job_name", "(.*)")'
+)
+ONEOFF_JOB_PODS = f'(kube_pod_owner{{owner_kind="Job"}} unless on(namespace, owner_name) {ONEOFF_JOB_OWNER})'
+ONEOFF_JOB_POD_AGE_HOURS = (
+    '((time() - kube_pod_start_time{pod!=""}) / 3600) '
+    f'* on(namespace,pod) group_left(owner_name) {ONEOFF_JOB_PODS} '
+    '* on(namespace,pod) group_left(phase) '
+    'max by (namespace,pod,phase) (kube_pod_status_phase{phase=~"Running|Succeeded"})'
+)
+GLUE_LAST_SUCCESS_RANGE_HOURS = f"(time() - max_over_time({GLUE_LAST_SUCCESS}[$__range])) / 3600"
+GLUE_LAST_SCHEDULE_RANGE_HOURS = f"(time() - max_over_time({GLUE_LAST_SCHEDULE}[$__range])) / 3600"
 GPU_NODES = ["titan-20", "titan-21", "titan-22", "titan-24"]
 GPU_NODE_REGEX = "|".join(GPU_NODES)
 TRAEFIK_ROUTER_EXPR = "sum by (router) (rate(traefik_router_requests_total[5m]))"
@@ -496,6 +624,7 @@ def timeseries_panel(
     grid,
     *,
     unit="none",
+    max_value=None,
     legend=None,
     legend_display="table",
     legend_placement="bottom",
@@ -520,6 +649,8 @@ def timeseries_panel(
             "tooltip": {"mode": "multi"},
         },
     }
+    if max_value is not None:
+        panel["fieldConfig"]["defaults"]["max"] = max_value
     if legend:
         panel["targets"][0]["legendFormat"] = legend
     if legend_calcs:
@@ -671,13 +802,22 @@ def bargauge_panel(
     grid,
     *,
     unit="none",
+    legend=None,
     links=None,
     limit=None,
+    sort_order="desc",
     thresholds=None,
     decimals=None,
     instant=False,
+    overrides=None,
 ):
     """Return a bar gauge panel with label-aware reduction."""
+    cleaned_expr = expr.strip()
+    if not cleaned_expr.startswith(("sort(", "sort_desc(")):
+        if sort_order == "desc":
+            expr = f"sort_desc({expr})"
+        elif sort_order == "asc":
+            expr = f"sort({expr})"
     panel = {
         "id": panel_id,
         "type": "bargauge",
@@ -685,7 +825,12 @@ def bargauge_panel(
         "datasource": PROM_DS,
         "gridPos": grid,
         "targets": [
-            {"expr": expr, "refId": "A", "legendFormat": "{{node}}", **({"instant": True} if instant else {})}
+            {
+                "expr": expr,
+                "refId": "A",
+                "legendFormat": legend or "{{node}}",
+                **({"instant": True} if instant else {}),
+            }
         ],
         "fieldConfig": {
             "defaults": {
@@ -715,6 +860,8 @@ def bargauge_panel(
             },
         },
     }
+    if overrides:
+        panel["fieldConfig"]["overrides"].extend(overrides)
     if decimals is not None:
         panel["fieldConfig"]["defaults"]["decimals"] = decimals
     if links:
@@ -723,7 +870,7 @@ def bargauge_panel(
     panel["transformations"] = [
         {
             "id": "sortBy",
-            "options": {"fields": ["Value"], "order": "desc"},
+            "options": {"fields": ["Value"], "order": sort_order},
         }
     ]
     if limit:
@@ -763,6 +910,15 @@ def build_overview():
             {"color": "red", "value": 3},
         ],
     }
+    age_thresholds = {
+        "mode": "absolute",
+        "steps": [
+            {"color": "green", "value": None},
+            {"color": "yellow", "value": 6},
+            {"color": "orange", "value": 24},
+            {"color": "red", "value": 48},
+        ],
+    }
 
     row1_stats = [
         {
@@ -965,7 +1121,7 @@ def build_overview():
             30,
             "Mail Sent (1d)",
             'max(postmark_outbound_sent{window="1d"})',
-            {"h": 2, "w": 6, "x": 0, "y": 8},
+            {"h": 3, "w": 4, "x": 0, "y": 8},
             unit="none",
             links=link_to("atlas-mail"),
         )
@@ -976,7 +1132,7 @@ def build_overview():
             "type": "stat",
             "title": "Mail Bounces (1d)",
             "datasource": PROM_DS,
-            "gridPos": {"h": 2, "w": 6, "x": 12, "y": 8},
+            "gridPos": {"h": 3, "w": 4, "x": 8, "y": 8},
             "targets": [
                 {
                     "expr": 'max(postmark_outbound_bounce_rate{window="1d"})',
@@ -1022,7 +1178,7 @@ def build_overview():
             32,
             "Mail Success Rate (1d)",
             'clamp_min(100 - max(postmark_outbound_bounce_rate{window="1d"}), 0)',
-            {"h": 2, "w": 6, "x": 6, "y": 8},
+            {"h": 3, "w": 4, "x": 4, "y": 8},
             unit="percent",
             thresholds=mail_success_thresholds,
             decimals=1,
@@ -1034,13 +1190,38 @@ def build_overview():
             33,
             "Mail Limit Used (30d)",
             "max(postmark_sending_limit_used_percent)",
-            {"h": 2, "w": 6, "x": 18, "y": 8},
+            {"h": 3, "w": 4, "x": 12, "y": 8},
             unit="percent",
             thresholds=mail_limit_thresholds,
             decimals=1,
             links=link_to("atlas-mail"),
         )
     )
+    panels.append(
+        stat_panel(
+            34,
+            "Postgres Connections Used",
+            POSTGRES_CONN_USED,
+            {"h": 3, "w": 4, "x": 16, "y": 8},
+            decimals=0,
+            text_mode="name_and_value",
+            legend="{{conn}}",
+            instant=True,
+        )
+    )
+    panels.append(
+        stat_panel(
+            35,
+            "Postgres Hottest Connections",
+            POSTGRES_CONN_HOTTEST,
+            {"h": 3, "w": 4, "x": 20, "y": 8},
+            unit="none",
+            decimals=0,
+            text_mode="name_and_value",
+            legend="{{datname}}",
+            instant=True,
+        )
+    )
 
     storage_panels = [
         (23, "Astreae Usage", astreae_usage_expr("/mnt/astreae"), "percent"),
@@ -1054,13 +1235,104 @@ def build_overview():
                 panel_id,
                 title,
                 expr,
-                {"h": 6, "w": 6, "x": 6 * idx, "y": 10},
+                {"h": 3, "w": 6, "x": 6 * idx, "y": 11},
                 unit=unit,
                 thresholds=PERCENT_THRESHOLDS if unit == "percent" else None,
                 links=link_to("atlas-storage"),
             )
         )
 
+    panels.append(
+        bargauge_panel(
+            40,
+            "One-off Job Pods (age hours)",
+            ONEOFF_JOB_POD_AGE_HOURS,
+            {"h": 6, "w": 6, "x": 0, "y": 14},
+            unit="h",
+            instant=True,
+            legend="{{namespace}}/{{pod}}",
+            thresholds=age_thresholds,
+            limit=8,
+            decimals=2,
+        )
+    )
+    panels.append(
+        {
+            "id": 41,
+            "type": "timeseries",
+            "title": "Ariadne Attempts / Failures",
+            "datasource": PROM_DS,
+            "gridPos": {"h": 6, "w": 6, "x": 6, "y": 14},
+            "targets": [
+                {"expr": ARIADNE_TASK_ATTEMPTS_SERIES, "refId": "A", "legendFormat": "Attempts"},
+                {"expr": ARIADNE_TASK_FAILURES_SERIES, "refId": "B", "legendFormat": "Failures"},
+            ],
+            "fieldConfig": {
+                "defaults": {"unit": "none"},
+                "overrides": [
+                    {
+                        "matcher": {"id": "byName", "options": "Attempts"},
+                        "properties": [
+                            {"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}
+                        ],
+                    },
+                    {
+                        "matcher": {"id": "byName", "options": "Failures"},
+                        "properties": [
+                            {"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}
+                        ],
+                    },
+                ],
+            },
+            "options": {
+                "legend": {"displayMode": "table", "placement": "right"},
+                "tooltip": {"mode": "multi"},
+            },
+        }
+    )
+    panels.append(
+        timeseries_panel(
+            42,
+            "Ariadne Test Success Rate",
+            ARIADNE_TEST_SUCCESS_RATE,
+            {"h": 6, "w": 6, "x": 12, "y": 14},
+            unit="percent",
+            max_value=100,
+            legend=None,
+            legend_display="list",
+        )
+    )
+    panels.append(
+        bargauge_panel(
+            43,
+            "Tests with Failures (24h)",
+            ARIADNE_TEST_FAILURES_24H,
+            {"h": 6, "w": 6, "x": 18, "y": 14},
+            unit="none",
+            instant=True,
+            legend="{{result}}",
+            overrides=[
+                {
+                    "matcher": {"id": "byName", "options": "error"},
+                    "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "yellow"}}],
+                },
+                {
+                    "matcher": {"id": "byName", "options": "failed"},
+                    "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}],
+                },
+            ],
+            thresholds={
+                "mode": "absolute",
+                "steps": [
+                    {"color": "green", "value": None},
+                    {"color": "yellow", "value": 1},
+                    {"color": "orange", "value": 5},
+                    {"color": "red", "value": 10},
+                ],
+            },
+        )
+    )
+
     cpu_scope = "$namespace_scope_cpu"
     gpu_scope = "$namespace_scope_gpu"
     ram_scope = "$namespace_scope_ram"
@@ -1070,9 +1342,9 @@ def build_overview():
             11,
             "Namespace CPU Share",
             namespace_cpu_share_expr(cpu_scope),
-            {"h": 9, "w": 8, "x": 0, "y": 16},
+            {"h": 9, "w": 8, "x": 0, "y": 20},
             links=namespace_scope_links("namespace_scope_cpu"),
-            description="Values are normalized within the selected scope; use panel links to switch scope.",
+            description="Shares are normalized within the selected filter. Switching scope changes the denominator.",
         )
     )
     panels.append(
@@ -1080,9 +1352,9 @@ def build_overview():
             12,
             "Namespace GPU Share",
             namespace_gpu_share_expr(gpu_scope),
-            {"h": 9, "w": 8, "x": 8, "y": 16},
+            {"h": 9, "w": 8, "x": 8, "y": 20},
             links=namespace_scope_links("namespace_scope_gpu"),
-            description="Values are normalized within the selected scope; use panel links to switch scope.",
+            description="Shares are normalized within the selected filter. Switching scope changes the denominator.",
         )
     )
     panels.append(
@@ -1090,9 +1362,9 @@ def build_overview():
             13,
             "Namespace RAM Share",
             namespace_ram_share_expr(ram_scope),
-            {"h": 9, "w": 8, "x": 16, "y": 16},
+            {"h": 9, "w": 8, "x": 16, "y": 20},
             links=namespace_scope_links("namespace_scope_ram"),
-            description="Values are normalized within the selected scope; use panel links to switch scope.",
+            description="Shares are normalized within the selected filter. Switching scope changes the denominator.",
         )
     )
 
@@ -1102,7 +1374,7 @@ def build_overview():
             14,
             "Worker Node CPU",
             node_cpu_expr(worker_filter),
-            {"h": 12, "w": 12, "x": 0, "y": 32},
+            {"h": 12, "w": 12, "x": 0, "y": 36},
             unit="percent",
             legend="{{node}}",
             legend_calcs=["last"],
@@ -1116,7 +1388,7 @@ def build_overview():
             15,
             "Worker Node RAM",
             node_mem_expr(worker_filter),
-            {"h": 12, "w": 12, "x": 12, "y": 32},
+            {"h": 12, "w": 12, "x": 12, "y": 36},
             unit="percent",
             legend="{{node}}",
             legend_calcs=["last"],
@@ -1131,7 +1403,7 @@ def build_overview():
             16,
             "Control plane CPU",
             node_cpu_expr(CONTROL_ALL_REGEX),
-            {"h": 10, "w": 12, "x": 0, "y": 44},
+            {"h": 10, "w": 12, "x": 0, "y": 48},
             unit="percent",
             legend="{{node}}",
             legend_display="table",
@@ -1143,7 +1415,7 @@ def build_overview():
             17,
             "Control plane RAM",
             node_mem_expr(CONTROL_ALL_REGEX),
-            {"h": 10, "w": 12, "x": 12, "y": 44},
+            {"h": 10, "w": 12, "x": 12, "y": 48},
             unit="percent",
             legend="{{node}}",
             legend_display="table",
@@ -1156,7 +1428,7 @@ def build_overview():
             28,
             "Node Pod Share",
             '(sum(kube_pod_info{pod!="" , node!=""}) by (node) / clamp_min(sum(kube_pod_info{pod!="" , node!=""}), 1)) * 100',
-            {"h": 10, "w": 12, "x": 0, "y": 54},
+            {"h": 10, "w": 12, "x": 0, "y": 58},
         )
     )
     panels.append(
@@ -1164,7 +1436,7 @@ def build_overview():
             29,
             "Top Nodes by Pod Count",
             'topk(12, sum(kube_pod_info{pod!="" , node!=""}) by (node))',
-            {"h": 10, "w": 12, "x": 12, "y": 54},
+            {"h": 10, "w": 12, "x": 12, "y": 58},
             unit="none",
             limit=12,
             decimals=0,
@@ -1186,7 +1458,7 @@ def build_overview():
             18,
             "Cluster Ingress Throughput",
             NET_INGRESS_EXPR,
-            {"h": 7, "w": 8, "x": 0, "y": 25},
+            {"h": 7, "w": 8, "x": 0, "y": 29},
             unit="Bps",
             legend="Ingress (Traefik)",
             legend_display="list",
@@ -1199,7 +1471,7 @@ def build_overview():
             19,
             "Cluster Egress Throughput",
             NET_EGRESS_EXPR,
-            {"h": 7, "w": 8, "x": 8, "y": 25},
+            {"h": 7, "w": 8, "x": 8, "y": 29},
             unit="Bps",
             legend="Egress (Traefik)",
             legend_display="list",
@@ -1212,7 +1484,7 @@ def build_overview():
             20,
             "Intra-Cluster Throughput",
             NET_INTERNAL_EXPR,
-            {"h": 7, "w": 8, "x": 16, "y": 25},
+            {"h": 7, "w": 8, "x": 16, "y": 29},
             unit="Bps",
             legend="Internal traffic",
             legend_display="list",
@@ -1226,7 +1498,7 @@ def build_overview():
             21,
             "Root Filesystem Usage",
             root_usage_expr(),
-            {"h": 16, "w": 12, "x": 0, "y": 64},
+            {"h": 16, "w": 12, "x": 0, "y": 68},
             unit="percent",
             legend="{{node}}",
             legend_calcs=["last"],
@@ -1241,7 +1513,7 @@ def build_overview():
             22,
             "Nodes Closest to Full Root Disks",
             f"topk(12, {root_usage_expr()})",
-            {"h": 16, "w": 12, "x": 12, "y": 64},
+            {"h": 16, "w": 12, "x": 12, "y": 68},
             unit="percent",
             thresholds=PERCENT_THRESHOLDS,
             links=link_to("atlas-storage"),
@@ -1727,7 +1999,7 @@ def build_storage_dashboard():
         stat_panel(
             31,
             "Maintenance Cron Freshness (s)",
-            'time() - max by (cronjob) (kube_cronjob_status_last_successful_time{namespace="maintenance",cronjob=~"image-sweeper|grafana-smtp-sync"})',
+            'time() - max by (cronjob) (kube_cronjob_status_last_successful_time{namespace="maintenance",cronjob="image-sweeper"})',
             {"h": 4, "w": 12, "x": 12, "y": 44},
             unit="s",
             thresholds={
@@ -2136,6 +2408,285 @@ def build_mail_dashboard():
     }
 
 
+def build_jobs_dashboard():
+    panels = []
+    age_thresholds = {
+        "mode": "absolute",
+        "steps": [
+            {"color": "green", "value": None},
+            {"color": "yellow", "value": 6},
+            {"color": "orange", "value": 24},
+            {"color": "red", "value": 48},
+        ],
+    }
+    recent_error_thresholds = {
+        "mode": "absolute",
+        "steps": [
+            {"color": "red", "value": None},
+            {"color": "orange", "value": 1},
+            {"color": "yellow", "value": 6},
+            {"color": "green", "value": 24},
+        ],
+    }
+
+    task_error_thresholds = {
+        "mode": "absolute",
+        "steps": [
+            {"color": "green", "value": None},
+            {"color": "yellow", "value": 1},
+            {"color": "orange", "value": 3},
+            {"color": "red", "value": 5},
+        ],
+    }
+
+    panels.append(
+        bargauge_panel(
+            1,
+            "Ariadne Task Errors (range)",
+            ARIADNE_TASK_ERRORS_RANGE,
+            {"h": 7, "w": 8, "x": 0, "y": 0},
+            unit="none",
+            instant=True,
+            legend="{{task}}",
+            thresholds=task_error_thresholds,
+        )
+    )
+    panels.append(
+        {
+            "id": 2,
+            "type": "timeseries",
+            "title": "Ariadne Attempts / Failures",
+            "datasource": PROM_DS,
+            "gridPos": {"h": 7, "w": 8, "x": 8, "y": 0},
+            "targets": [
+                {"expr": ARIADNE_TASK_ATTEMPTS_SERIES, "refId": "A", "legendFormat": "Attempts"},
+                {"expr": ARIADNE_TASK_FAILURES_SERIES, "refId": "B", "legendFormat": "Failures"},
+            ],
+            "fieldConfig": {
+                "defaults": {"unit": "none"},
+                "overrides": [
+                    {
+                        "matcher": {"id": "byName", "options": "Attempts"},
+                        "properties": [
+                            {"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}
+                        ],
+                    },
+                    {
+                        "matcher": {"id": "byName", "options": "Failures"},
+                        "properties": [
+                            {"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}
+                        ],
+                    },
+                ],
+            },
+            "options": {
+                "legend": {"displayMode": "table", "placement": "right"},
+                "tooltip": {"mode": "multi"},
+            },
+        }
+    )
+    panels.append(
+        bargauge_panel(
+            3,
+            "One-off Job Pods (age hours)",
+            ONEOFF_JOB_POD_AGE_HOURS,
+            {"h": 7, "w": 8, "x": 16, "y": 0},
+            unit="h",
+            instant=True,
+            legend="{{namespace}}/{{pod}}",
+            thresholds=age_thresholds,
+            limit=12,
+            decimals=2,
+        )
+    )
+    panels.append(
+        stat_panel(
+            4,
+            "Glue Jobs Stale (>36h)",
+            GLUE_STALE_COUNT,
+            {"h": 4, "w": 4, "x": 0, "y": 7},
+            unit="none",
+            thresholds={
+                "mode": "absolute",
+                "steps": [
+                    {"color": "green", "value": None},
+                    {"color": "yellow", "value": 1},
+                    {"color": "orange", "value": 2},
+                    {"color": "red", "value": 3},
+                ],
+            },
+        )
+    )
+    panels.append(
+        stat_panel(
+            5,
+            "Glue Jobs Missing Success",
+            GLUE_MISSING_COUNT,
+            {"h": 4, "w": 4, "x": 4, "y": 7},
+            unit="none",
+        )
+    )
+    panels.append(
+        stat_panel(
+            6,
+            "Glue Jobs Suspended",
+            GLUE_SUSPENDED_COUNT,
+            {"h": 4, "w": 4, "x": 8, "y": 7},
+            unit="none",
+        )
+    )
+    panels.append(
+        stat_panel(
+            7,
+            "Ariadne Task Errors (1h)",
+            ARIADNE_TASK_ERRORS_1H_TOTAL,
+            {"h": 4, "w": 4, "x": 12, "y": 7},
+            unit="none",
+        )
+    )
+    panels.append(
+        stat_panel(
+            8,
+            "Ariadne Task Errors (24h)",
+            ARIADNE_TASK_ERRORS_24H_TOTAL,
+            {"h": 4, "w": 4, "x": 16, "y": 7},
+            unit="none",
+        )
+    )
+    panels.append(
+        stat_panel(
+            9,
+            "Ariadne Task Runs (1h)",
+            ARIADNE_TASK_RUNS_1H_TOTAL,
+            {"h": 4, "w": 4, "x": 20, "y": 7},
+            unit="none",
+        )
+    )
+    panels.append(
+        bargauge_panel(
+            10,
+            "Ariadne Schedule Last Error (hours ago)",
+            ARIADNE_SCHEDULE_LAST_ERROR_RANGE_HOURS,
+            {"h": 6, "w": 12, "x": 0, "y": 17},
+            unit="h",
+            instant=True,
+            legend="{{task}}",
+            thresholds=recent_error_thresholds,
+            decimals=2,
+        )
+    )
+    panels.append(
+        bargauge_panel(
+            11,
+            "Ariadne Schedule Last Success (hours ago)",
+            ARIADNE_SCHEDULE_LAST_SUCCESS_RANGE_HOURS,
+            {"h": 6, "w": 12, "x": 12, "y": 17},
+            unit="h",
+            instant=True,
+            legend="{{task}}",
+            thresholds=age_thresholds,
+            decimals=2,
+        )
+    )
+    panels.append(
+        bargauge_panel(
+            12,
+            "Glue Jobs Last Success (hours ago)",
+            GLUE_LAST_SUCCESS_RANGE_HOURS,
+            {"h": 6, "w": 12, "x": 0, "y": 23},
+            unit="h",
+            instant=True,
+            legend="{{namespace}}/{{cronjob}}",
+            thresholds=age_thresholds,
+            decimals=2,
+        )
+    )
+    panels.append(
+        bargauge_panel(
+            13,
+            "Glue Jobs Last Schedule (hours ago)",
+            GLUE_LAST_SCHEDULE_RANGE_HOURS,
+            {"h": 6, "w": 12, "x": 12, "y": 23},
+            unit="h",
+            instant=True,
+            legend="{{namespace}}/{{cronjob}}",
+            thresholds=age_thresholds,
+            decimals=2,
+        )
+    )
+    panels.append(
+        bargauge_panel(
+            14,
+            "Ariadne Task Errors (1h)",
+            ARIADNE_TASK_ERRORS_1H,
+            {"h": 6, "w": 12, "x": 0, "y": 29},
+            unit="none",
+            instant=True,
+            legend="{{task}}",
+            thresholds=task_error_thresholds,
+        )
+    )
+    panels.append(
+        bargauge_panel(
+            15,
+            "Ariadne Task Errors (30d)",
+            ARIADNE_TASK_ERRORS_30D,
+            {"h": 6, "w": 12, "x": 12, "y": 29},
+            unit="none",
+            instant=True,
+            legend="{{task}}",
+            thresholds=task_error_thresholds,
+        )
+    )
+    panels.append(
+        bargauge_panel(
+            16,
+            "Ariadne Access Requests",
+            ARIADNE_ACCESS_REQUESTS,
+            {"h": 6, "w": 8, "x": 0, "y": 11},
+            unit="none",
+            instant=True,
+            legend="{{status}}",
+        )
+    )
+    panels.append(
+        stat_panel(
+            17,
+            "Ariadne CI Coverage (%)",
+            ARIADNE_CI_COVERAGE,
+            {"h": 6, "w": 4, "x": 8, "y": 11},
+            unit="percent",
+            decimals=1,
+            instant=True,
+            legend="{{branch}}",
+        )
+    )
+    panels.append(
+        table_panel(
+            18,
+            "Ariadne CI Tests (latest)",
+            ARIADNE_CI_TESTS,
+            {"h": 6, "w": 12, "x": 12, "y": 11},
+            unit="none",
+            transformations=[{"id": "labelsToFields", "options": {}}, {"id": "sortBy", "options": {"fields": ["Value"], "order": "desc"}}],
+            instant=True,
+        )
+    )
+
+    return {
+        "uid": "atlas-jobs",
+        "title": "Atlas Jobs",
+        "folderUid": PRIVATE_FOLDER,
+        "editable": True,
+        "panels": panels,
+        "time": {"from": "now-7d", "to": "now"},
+        "annotations": {"list": []},
+        "schemaVersion": 39,
+        "style": "dark",
+        "tags": ["atlas", "jobs", "glue"],
+    }
+
+
 def build_gpu_dashboard():
     panels = []
     gpu_scope = "$namespace_scope_gpu"
@@ -2146,7 +2697,7 @@ def build_gpu_dashboard():
             namespace_gpu_share_expr(gpu_scope),
             {"h": 8, "w": 12, "x": 0, "y": 0},
             links=namespace_scope_links("namespace_scope_gpu"),
-            description="Values are normalized within the selected scope; use panel links to switch scope.",
+            description="Shares are normalized within the selected filter. Switching scope changes the denominator.",
         )
     )
     panels.append(
@@ -2165,7 +2716,7 @@ def build_gpu_dashboard():
         timeseries_panel(
             3,
             "GPU Util by Node",
-            'sum by (Hostname) (DCGM_FI_DEV_GPU_UTIL{pod!=""})',
+            gpu_util_by_hostname(),
             {"h": 8, "w": 12, "x": 0, "y": 8},
             unit="percent",
             legend="{{Hostname}}",
@@ -2229,6 +2780,10 @@ DASHBOARDS = {
         "builder": build_mail_dashboard,
         "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-mail.yaml",
     },
+    "atlas-jobs": {
+        "builder": build_jobs_dashboard,
+        "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-jobs.yaml",
+    },
     "atlas-gpu": {
         "builder": build_gpu_dashboard,
         "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-gpu.yaml",