Merge main into deploy

Reviewed-on: #9

.gitignore

@@ -0,0 +1,9 @@
+*.md
+!README.md
+!knowledge/**/*.md
+!services/comms/knowledge/**/*.md
+__pycache__/
+*.py[cod]
+.pytest_cache
+.venv
+tmp/

README.md

@@ -0,0 +1,3 @@
+# titan-iac
+
+Flux-managed Kubernetes cluster for bstein.dev services.

ci/Jenkinsfile.titan-iac

@@ -0,0 +1,53 @@
+pipeline {
+  agent {
+    kubernetes {
+      defaultContainer 'python'
+      yaml """
+apiVersion: v1
+kind: Pod
+spec:
+  containers:
+    - name: python
+      image: python:3.12-slim
+      command:
+        - cat
+      tty: true
+"""
+    }
+  }
+  environment {
+    PIP_DISABLE_PIP_VERSION_CHECK = '1'
+    PYTHONUNBUFFERED = '1'
+    DEPLOY_BRANCH = 'deploy'
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        checkout scm
+      }
+    }
+    stage('Install deps') {
+      steps {
+        sh 'pip install --no-cache-dir -r ci/requirements.txt'
+      }
+    }
+    stage('Glue tests') {
+      steps {
+        sh 'pytest -q ci/tests/glue'
+      }
+    }
+    stage('Promote') {
+      steps {
+        withCredentials([usernamePassword(credentialsId: 'gitea-pat', usernameVariable: 'GIT_USER', passwordVariable: 'GIT_TOKEN')]) {
+          sh '''
+            set +x
+            git config user.email "jenkins@bstein.dev"
+            git config user.name "jenkins"
+            git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git
+            git push origin HEAD:${DEPLOY_BRANCH}
+          '''
+        }
+      }
+    }
+  }
+}

ci/requirements.txt

@@ -0,0 +1,4 @@
+pytest==8.3.4
+kubernetes==30.1.0
+PyYAML==6.0.2
+requests==2.32.3

ci/tests/glue/config.yaml

@@ -0,0 +1,7 @@
+max_success_age_hours: 48
+allow_suspended:
+  - comms/othrys-room-reset
+  - comms/pin-othrys-invite
+  - comms/seed-othrys-room
+  - finance/firefly-user-sync
+  - health/wger-user-sync

ci/tests/glue/test_glue_cronjobs.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+
+import yaml
+from kubernetes import client, config
+
+
+CONFIG_PATH = Path(__file__).with_name("config.yaml")
+
+
+def _load_config() -> dict:
+    with CONFIG_PATH.open("r", encoding="utf-8") as handle:
+        return yaml.safe_load(handle) or {}
+
+
+def _load_kube():
+    try:
+        config.load_incluster_config()
+    except config.ConfigException:
+        config.load_kube_config()
+
+
+def test_glue_cronjobs_recent_success():
+    cfg = _load_config()
+    max_age_hours = int(cfg.get("max_success_age_hours", 48))
+    allow_suspended = set(cfg.get("allow_suspended", []))
+
+    _load_kube()
+    batch = client.BatchV1Api()
+    cronjobs = batch.list_cron_job_for_all_namespaces(label_selector="atlas.bstein.dev/glue=true").items
+
+    assert cronjobs, "No glue cronjobs found with atlas.bstein.dev/glue=true"
+
+    now = datetime.now(timezone.utc)
+    for cronjob in cronjobs:
+        name = f"{cronjob.metadata.namespace}/{cronjob.metadata.name}"
+        if cronjob.spec.suspend:
+            assert name in allow_suspended, f"{name} is suspended but not in allow_suspended"
+            continue
+
+        last_success = cronjob.status.last_successful_time
+        assert last_success is not None, f"{name} has no lastSuccessfulTime"
+        age_hours = (now - last_success).total_seconds() / 3600
+        assert age_hours <= max_age_hours, f"{name} last success {age_hours:.1f}h ago"

ci/tests/glue/test_glue_metrics.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import os
+
+import requests
+
+
+VM_URL = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
+
+
+def _query(promql: str) -> list[dict]:
+    response = requests.get(f"{VM_URL}/api/v1/query", params={"query": promql}, timeout=10)
+    response.raise_for_status()
+    payload = response.json()
+    return payload.get("data", {}).get("result", [])
+
+
+def test_glue_metrics_present():
+    series = _query('kube_cronjob_labels{label_atlas_bstein_dev_glue="true"}')
+    assert series, "No glue cronjob label series found"
+
+
+def test_glue_metrics_success_join():
+    query = (
+        "kube_cronjob_status_last_successful_time "
+        'and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue="true"}'
+    )
+    series = _query(query)
+    assert series, "No glue cronjob last success series found"

clusters/atlas/flux-system/applications/ai-llm/kustomization.yaml

@@ -1,22 +1,23 @@
-# infrastructure/flux-system/kustomization-core.yaml
+# clusters/atlas/flux-system/applications/ai-llm/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: core
+  name: ai-llm
   namespace: flux-system
 spec:
   interval: 10m
-  path: ./infrastructure/core
+  path: ./services/ai-llm
+  targetNamespace: ai
   prune: true
   sourceRef:
     kind: GitRepository
     name: flux-system
     namespace: flux-system
   wait: true
-
-  # Only wait for the NVIDIA device-plugin DaemonSet on titan-22
   healthChecks:
     - apiVersion: apps/v1
-      kind: DaemonSet
-      name: nvidia-device-plugin-minipc
-      namespace: kube-system
+      kind: Deployment
+      name: ollama
+      namespace: ai
+  dependsOn:
+    - name: core

clusters/atlas/flux-system/applications/bstein-dev-home/image-automation.yaml

@@ -0,0 +1,26 @@
+# clusters/atlas/flux-system/applications/bstein-dev-home/image-automation.yaml
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageUpdateAutomation
+metadata:
+  name: bstein-dev-home
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  git:
+    checkout:
+      ref:
+        branch: feature/vault-consumption
+    commit:
+      author:
+        email: ops@bstein.dev
+        name: flux-bot
+      messageTemplate: "chore(bstein-dev-home): update images to {{range .Updated.Images}}{{.}}{{end}}"
+    push:
+      branch: feature/vault-consumption
+  update:
+    strategy: Setters
+    path: services/bstein-dev-home

clusters/atlas/flux-system/applications/bstein-dev-home/kustomization.yaml

@@ -0,0 +1,15 @@
+# clusters/atlas/flux-system/applications/bstein-dev-home/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: bstein-dev-home
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/bstein-dev-home
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: bstein-dev-home
+  wait: false

clusters/atlas/flux-system/applications/comms/kustomization.yaml

@@ -1,18 +1,17 @@
-# infrastructure/flux-system/kustomization-zot.yaml
+# clusters/atlas/flux-system/applications/comms/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
-  name: zot
+  name: comms
   namespace: flux-system
 spec:
   interval: 10m
-  path: ./services/zot
-  targetNamespace: zot
-  prune: false
+  prune: true
   sourceRef:
     kind: GitRepository
     name: flux-system
-    namespace: flux-system
-  wait: true
+  path: ./services/comms
+  targetNamespace: comms
+  timeout: 2m
   dependsOn:
-    - name: core 
+    - name: traefik

clusters/atlas/flux-system/applications/crypto/kustomization.yaml

@@ -1,4 +1,4 @@
-# infrastructure/flux-system/kustomization-crypto.yaml
+# clusters/atlas/flux-system/applications/crypto/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:

clusters/atlas/flux-system/applications/finance/kustomization.yaml

@@ -0,0 +1,24 @@
+# clusters/atlas/flux-system/applications/finance/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: finance
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/finance
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: finance
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: actual-budget
+      namespace: finance
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: firefly
+      namespace: finance
+  wait: false

clusters/atlas/flux-system/applications/gitea/kustomization.yaml

@@ -1,4 +1,4 @@
-# infrastructure/flux-system/kustomization-gitea.yaml
+# clusters/atlas/flux-system/applications/gitea/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
 kind: Kustomization
 metadata:

clusters/atlas/flux-system/applications/harbor/image-automation.yaml

@@ -0,0 +1,27 @@
+# clusters/atlas/flux-system/applications/harbor/image-automation.yaml
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageUpdateAutomation
+metadata:
+  name: harbor
+  namespace: harbor
+spec:
+  suspend: true
+  interval: 5m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  git:
+    checkout:
+      ref:
+        branch: feature/ci-gitops
+    commit:
+      author:
+        email: ops@bstein.dev
+        name: flux-bot
+      messageTemplate: "chore(harbor): apply image updates"
+    push:
+      branch: feature/ci-gitops
+  update:
+    strategy: Setters
+    path: ./services/harbor

clusters/atlas/flux-system/applications/harbor/kustomization.yaml

@@ -0,0 +1,23 @@
+# clusters/atlas/flux-system/applications/harbor/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: harbor
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/harbor
+  targetNamespace: harbor
+  prune: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  healthChecks:
+    - apiVersion: helm.toolkit.fluxcd.io/v2
+      kind: HelmRelease
+      name: harbor
+      namespace: harbor
+  wait: false
+  dependsOn:
+    - name: core

clusters/atlas/flux-system/applications/health/kustomization.yaml

@@ -0,0 +1,25 @@
+# clusters/atlas/flux-system/applications/health/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: health
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/health
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: health
+  dependsOn:
+    - name: keycloak
+    - name: postgres
+    - name: traefik
+    - name: vault
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: wger
+      namespace: health
+  wait: false

clusters/atlas/flux-system/applications/jellyfin/kustomization.yaml

@@ -1,4 +1,4 @@
-# infrastructure/flux-system/kustomization-jellyfin.yaml
+# clusters/atlas/flux-system/applications/jellyfin/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
@@ -15,5 +15,6 @@ spec:
     namespace: flux-system
   dependsOn:
     - name: core
+    - name: openldap
   wait: true
   timeout: 5m

clusters/atlas/flux-system/applications/jenkins/kustomization.yaml

@@ -0,0 +1,27 @@
+# clusters/atlas/flux-system/applications/jenkins/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: jenkins
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/jenkins
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: jenkins
+  dependsOn:
+    - name: helm
+    - name: traefik
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: jenkins
+      namespace: jenkins
+    - apiVersion: v1
+      kind: Service
+      name: jenkins
+      namespace: jenkins
+  wait: false

clusters/atlas/flux-system/applications/keycloak/kustomization.yaml

@@ -0,0 +1,15 @@
+# clusters/atlas/flux-system/applications/keycloak/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: keycloak
+  namespace: flux-system
+spec:
+  interval: 10m
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./services/keycloak
+  targetNamespace: sso
+  timeout: 2m

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -0,0 +1,32 @@
+# clusters/atlas/flux-system/applications/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - gitea/kustomization.yaml
+  - vault/kustomization.yaml
+  - vaultwarden/kustomization.yaml
+  - comms/kustomization.yaml
+  - crypto/kustomization.yaml
+  - monerod/kustomization.yaml
+  - pegasus/kustomization.yaml
+  - pegasus/image-automation.yaml
+  - bstein-dev-home/kustomization.yaml
+  - bstein-dev-home/image-automation.yaml
+  - harbor/kustomization.yaml
+  - harbor/image-automation.yaml
+  - jellyfin/kustomization.yaml
+  - xmr-miner/kustomization.yaml
+  - wallet-monero-temp/kustomization.yaml
+  - sui-metrics/kustomization.yaml
+  - openldap/kustomization.yaml
+  - keycloak/kustomization.yaml
+  - oauth2-proxy/kustomization.yaml
+  - mailu/kustomization.yaml
+  - jenkins/kustomization.yaml
+  - ai-llm/kustomization.yaml
+  - nextcloud/kustomization.yaml
+  - nextcloud-mail-sync/kustomization.yaml
+  - outline/kustomization.yaml
+  - planka/kustomization.yaml
+  - finance/kustomization.yaml
+  - health/kustomization.yaml

clusters/atlas/flux-system/applications/mailu/kustomization.yaml

@@ -0,0 +1,18 @@
+# clusters/atlas/flux-system/applications/mailu/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: mailu
+  namespace: flux-system
+spec:
+  interval: 10m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  path: ./services/mailu
+  targetNamespace: mailu-mailserver
+  prune: true
+  wait: true
+  dependsOn:
+    - name: helm

clusters/atlas/flux-system/applications/monerod/kustomization.yaml

@@ -1,4 +1,4 @@
-# infrastructure/flux-system/kustomization-monerod.yaml
+# clusters/atlas/flux-system/applications/monerod/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:

clusters/atlas/flux-system/applications/nextcloud-mail-sync/kustomization.yaml

@@ -0,0 +1,17 @@
+# clusters/atlas/flux-system/applications/nextcloud-mail-sync/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: nextcloud-mail-sync
+  namespace: flux-system
+spec:
+  interval: 10m
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./services/nextcloud-mail-sync
+  targetNamespace: nextcloud
+  timeout: 2m
+  dependsOn:
+    - name: keycloak

clusters/atlas/flux-system/applications/nextcloud/kustomization.yaml

@@ -0,0 +1,16 @@
+# clusters/atlas/flux-system/applications/nextcloud/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: nextcloud
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/nextcloud
+  targetNamespace: nextcloud
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true

clusters/atlas/flux-system/applications/oauth2-proxy/kustomization.yaml

@@ -0,0 +1,15 @@
+# clusters/atlas/flux-system/applications/oauth2-proxy/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: oauth2-proxy
+  namespace: flux-system
+spec:
+  interval: 10m
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  path: ./services/oauth2-proxy
+  targetNamespace: sso
+  timeout: 2m

clusters/atlas/flux-system/applications/openldap/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/applications/openldap/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: openldap
+  namespace: flux-system
+spec:
+  interval: 10m
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  path: ./services/openldap
+  targetNamespace: sso
+  dependsOn:
+    - name: core
+  wait: true
+  timeout: 5m

clusters/atlas/flux-system/applications/outline/kustomization.yaml

@@ -0,0 +1,28 @@
+# clusters/atlas/flux-system/applications/outline/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: outline
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/outline
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: outline
+  dependsOn:
+    - name: keycloak
+    - name: mailu
+    - name: traefik
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: outline
+      namespace: outline
+    - apiVersion: v1
+      kind: Service
+      name: outline
+      namespace: outline
+  wait: false

clusters/atlas/flux-system/applications/pegasus/image-automation.yaml

@@ -0,0 +1,26 @@
+# clusters/atlas/flux-system/applications/pegasus/image-automation.yaml
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageUpdateAutomation
+metadata:
+  name: pegasus
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  git:
+    checkout:
+      ref:
+        branch: feature/ci-gitops
+    commit:
+      author:
+        email: ops@bstein.dev
+        name: flux-bot
+      messageTemplate: "chore(pegasus): apply image updates"
+    push:
+      branch: feature/ci-gitops
+  update:
+    strategy: Setters
+    path: services/pegasus

clusters/atlas/flux-system/applications/pegasus/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/applications/pegasus/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: pegasus
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/pegasus
+  targetNamespace: jellyfin
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  dependsOn:
+    - name: core
+  wait: true
+  timeout: 5m

clusters/atlas/flux-system/applications/planka/kustomization.yaml

@@ -0,0 +1,28 @@
+# clusters/atlas/flux-system/applications/planka/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: planka
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/planka
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: planka
+  dependsOn:
+    - name: keycloak
+    - name: mailu
+    - name: traefik
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: planka
+      namespace: planka
+    - apiVersion: v1
+      kind: Service
+      name: planka
+      namespace: planka
+  wait: false

clusters/atlas/flux-system/applications/sui-metrics/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/applications/sui-metrics/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: sui-metrics
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/sui-metrics/overlays/atlas
+  prune: true
+  dependsOn:
+    - name: monitoring
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true
+  timeout: 5m
+  targetNamespace: sui-metrics

clusters/atlas/flux-system/applications/vault/kustomization.yaml

@@ -1,4 +1,4 @@
-# infrastructure/flux-system/kustomization-vault.yaml
+# clusters/atlas/flux-system/applications/vault/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:

clusters/atlas/flux-system/applications/vaultwarden/kustomization.yaml

@@ -0,0 +1,20 @@
+# clusters/atlas/flux-system/applications/vaultwarden/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: vaultwarden
+  namespace: flux-system
+spec:
+  interval: 10m
+  suspend: false
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  path: ./services/vaultwarden
+  targetNamespace: vaultwarden
+  prune: true
+  wait: true
+  dependsOn:
+    - name: helm
+    - name: traefik

clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: wallet-monero-temp
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/crypto/wallet-monero-temp
+  targetNamespace: crypto
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  dependsOn:
+    - name: crypto
+    - name: xmr-miner
+  wait: true

clusters/atlas/flux-system/applications/xmr-miner/kustomization.yaml

@@ -1,4 +1,4 @@
-# infrastructure/flux-system/kustomization-core.yaml
+# clusters/atlas/flux-system/applications/xmr-miner/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:

clusters/atlas/flux-system/gotk-sync.yaml

@@ -1,3 +1,4 @@
+# clusters/atlas/flux-system/gotk-sync.yaml
 # This manifest was generated by flux. DO NOT EDIT.
 ---
 apiVersion: source.toolkit.fluxcd.io/v1
@@ -8,7 +9,7 @@ metadata:
 spec:
   interval: 1m0s
   ref:
-    branch: main
+    branch: deploy
   secretRef:
     name: flux-system-gitea
   url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
@@ -20,7 +21,7 @@ metadata:
   namespace: flux-system
 spec:
   interval: 10m0s
-  path: ./
+  path: ./clusters/atlas/flux-system
   prune: true
   sourceRef:
     kind: GitRepository

clusters/atlas/flux-system/kustomization.yaml

@@ -0,0 +1,8 @@
+# clusters/atlas/flux-system/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - gotk-components.yaml
+  - gotk-sync.yaml
+  - platform
+  - applications

clusters/atlas/flux-system/platform/cert-manager-cleanup/kustomization.yaml

@@ -0,0 +1,17 @@
+# clusters/atlas/flux-system/platform/cert-manager-cleanup/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cert-manager-cleanup
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./infrastructure/cert-manager/cleanup
+  prune: true
+  force: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: cert-manager
+  wait: true

clusters/atlas/flux-system/platform/cert-manager/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/platform/cert-manager/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cert-manager
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./infrastructure/cert-manager
+  prune: true
+  force: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: cert-manager
+  dependsOn:
+    - name: helm
+  wait: true

clusters/atlas/flux-system/platform/core/kustomization.yaml

@@ -0,0 +1,15 @@
+# clusters/atlas/flux-system/platform/core/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: core
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./infrastructure/core
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: false

clusters/atlas/flux-system/platform/gitops-ui/kustomization.yaml

@@ -0,0 +1,20 @@
+# clusters/atlas/flux-system/platform/gitops-ui/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: gitops-ui
+  namespace: flux-system
+spec:
+  interval: 10m
+  timeout: 10m
+  path: ./services/gitops-ui
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: flux-system
+  dependsOn:
+    - name: helm
+    - name: traefik
+  wait: true

clusters/atlas/flux-system/platform/helm/kustomization.yaml

@@ -1,4 +1,4 @@
-# infrastructure/flux-system/kustomization-helm.yaml
+# clusters/atlas/flux-system/platform/helm/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:

clusters/atlas/flux-system/platform/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/platform/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - core/kustomization.yaml
+  - helm/kustomization.yaml
+  - cert-manager/kustomization.yaml
+  - metallb/kustomization.yaml
+  - traefik/kustomization.yaml
+  - gitops-ui/kustomization.yaml
+  - monitoring/kustomization.yaml
+  - logging/kustomization.yaml
+  - maintenance/kustomization.yaml
+  - longhorn-adopt/kustomization.yaml
+  - longhorn/kustomization.yaml
+  - longhorn-ui/kustomization.yaml
+  - postgres/kustomization.yaml
+  - ../platform/vault-csi/kustomization.yaml
+  - ../platform/vault-injector/kustomization.yaml

clusters/atlas/flux-system/platform/logging/kustomization.yaml

@@ -0,0 +1,14 @@
+# clusters/atlas/flux-system/platform/logging/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: logging
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/logging
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  wait: false

clusters/atlas/flux-system/platform/longhorn-adopt/kustomization.yaml

@@ -0,0 +1,17 @@
+# clusters/atlas/flux-system/platform/longhorn-adopt/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: longhorn-adopt
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./infrastructure/longhorn/adopt
+  prune: true
+  force: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: longhorn-system
+  wait: true

clusters/atlas/flux-system/platform/longhorn-ui/kustomization.yaml

@@ -1,3 +1,4 @@
+# clusters/atlas/flux-system/platform/longhorn-ui/kustomization.yaml
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
@@ -14,4 +15,5 @@ spec:
     namespace: flux-system
   dependsOn:
     - name: core
+    - name: longhorn
   wait: true

clusters/atlas/flux-system/platform/longhorn/kustomization.yaml

@@ -0,0 +1,20 @@
+# clusters/atlas/flux-system/platform/longhorn/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: longhorn
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./infrastructure/longhorn/core
+  prune: true
+  force: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  targetNamespace: longhorn-system
+  dependsOn:
+    - name: helm
+    - name: longhorn-adopt
+  wait: false

clusters/atlas/flux-system/platform/maintenance/kustomization.yaml

@@ -0,0 +1,14 @@
+# clusters/atlas/flux-system/platform/maintenance/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: maintenance
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/maintenance
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  wait: false

clusters/atlas/flux-system/platform/metallb/kustomization.yaml

@@ -0,0 +1,16 @@
+# clusters/atlas/flux-system/platform/metallb/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: metallb
+  namespace: flux-system
+spec:
+  interval: 30m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  path: ./infrastructure/metallb
+  prune: true
+  wait: true
+  targetNamespace: metallb-system

clusters/atlas/flux-system/platform/monitoring/kustomization.yaml

@@ -0,0 +1,14 @@
+# clusters/atlas/flux-system/platform/monitoring/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: monitoring
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/monitoring
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  wait: false

clusters/atlas/flux-system/platform/postgres/kustomization.yaml

@@ -0,0 +1,24 @@
+# clusters/atlas/flux-system/platform/postgres/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: postgres
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./infrastructure/postgres
+  prune: true
+  force: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: postgres
+  dependsOn:
+    - name: vault
+    - name: vault-csi
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: StatefulSet
+      name: postgres
+      namespace: postgres
+  wait: true

clusters/atlas/flux-system/platform/traefik/kustomization.yaml

@@ -0,0 +1,19 @@
+# clusters/atlas/flux-system/platform/traefik/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: traefik
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./infrastructure/traefik
+  targetNamespace: traefik
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  dependsOn:
+    - name: core
+    - name: metallb
+  wait: true

clusters/atlas/flux-system/platform/vault-csi/kustomization.yaml

@@ -0,0 +1,16 @@
+# clusters/atlas/flux-system/platform/vault-csi/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: vault-csi
+  namespace: flux-system
+spec:
+  interval: 30m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  path: ./infrastructure/vault-csi
+  prune: true
+  wait: true
+  targetNamespace: kube-system

clusters/atlas/flux-system/platform/vault-injector/kustomization.yaml

@@ -0,0 +1,16 @@
+# clusters/atlas/flux-system/platform/vault-injector/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: vault-injector
+  namespace: flux-system
+spec:
+  interval: 30m
+  path: ./infrastructure/vault-injector
+  targetNamespace: vault
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+    namespace: flux-system
+  wait: true

clusters/oceanus/applications/kustomization.yaml

@@ -0,0 +1,4 @@
+# clusters/oceanus/applications/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources: []

clusters/oceanus/flux-system/kustomization.yaml

@@ -0,0 +1,9 @@
+# clusters/oceanus/flux-system/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  # Populate when oceanus cluster is bootstrapped with Flux.
+  # - gotk-components.yaml
+  # - gotk-sync.yaml
+  - ../platform
+  - ../applications

clusters/oceanus/platform/kustomization.yaml

@@ -0,0 +1,6 @@
+# clusters/oceanus/platform/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../infrastructure/modules/base
+  - ../../infrastructure/modules/profiles/oceanus-validator

dockerfiles/Dockerfile.comms-guest-tools

@@ -0,0 +1,5 @@
+FROM python:3.11-slim
+
+ENV PIP_DISABLE_PIP_VERSION_CHECK=1
+
+RUN pip install --no-cache-dir requests psycopg2-binary

dockerfiles/Dockerfile.data-prepper

@@ -0,0 +1,16 @@
+FROM --platform=$BUILDPLATFORM opensearchproject/data-prepper:2.8.0 AS source
+
+FROM --platform=$TARGETPLATFORM eclipse-temurin:17-jre
+
+ENV DATA_PREPPER_PATH=/usr/share/data-prepper
+
+RUN useradd -u 10001 -M -U -d / -s /usr/sbin/nologin data_prepper \
+  && mkdir -p /var/log/data-prepper
+
+COPY --from=source /usr/share/data-prepper /usr/share/data-prepper
+
+RUN chown -R 10001:10001 /usr/share/data-prepper /var/log/data-prepper
+
+USER 10001
+WORKDIR /usr/share/data-prepper
+CMD ["bin/data-prepper"]

dockerfiles/Dockerfile.harbor-core-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-core:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/harbor/entrypoint.sh"]

dockerfiles/Dockerfile.harbor-jobservice-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-jobservice:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/harbor/entrypoint.sh"]

dockerfiles/Dockerfile.harbor-registry-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-registry:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/home/harbor/entrypoint.sh"]

dockerfiles/Dockerfile.harbor-registryctl-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-registryctl:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/home/harbor/start.sh"]

dockerfiles/Dockerfile.livekit-token-vault

@@ -0,0 +1,10 @@
+FROM ghcr.io/element-hq/lk-jwt-service:0.3.0 AS base
+
+FROM alpine:3.20
+RUN apk add --no-cache ca-certificates
+COPY --from=base /lk-jwt-service /lk-jwt-service
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/lk-jwt-service"]

dockerfiles/Dockerfile.oauth2-proxy-vault

@@ -0,0 +1,10 @@
+FROM quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 AS base
+
+FROM alpine:3.20
+RUN apk add --no-cache ca-certificates
+COPY --from=base /bin/oauth2-proxy /bin/oauth2-proxy
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/bin/oauth2-proxy"]

dockerfiles/Dockerfile.pegasus-vault

@@ -0,0 +1,10 @@
+FROM registry.bstein.dev/streaming/pegasus:1.2.32 AS base
+
+FROM alpine:3.20
+RUN apk add --no-cache ca-certificates
+COPY --from=base /pegasus /pegasus
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/pegasus"]

dockerfiles/vault-entrypoint.sh

@@ -0,0 +1,34 @@
+#!/bin/sh
+set -eu
+
+if [ -n "${VAULT_ENV_FILE:-}" ]; then
+  if [ -f "${VAULT_ENV_FILE}" ]; then
+    # shellcheck disable=SC1090
+    . "${VAULT_ENV_FILE}"
+  else
+    echo "Vault env file not found: ${VAULT_ENV_FILE}" >&2
+    exit 1
+  fi
+fi
+
+if [ -n "${VAULT_COPY_FILES:-}" ]; then
+  old_ifs="$IFS"
+  IFS=','
+  for pair in ${VAULT_COPY_FILES}; do
+    src="${pair%%:*}"
+    dest="${pair#*:}"
+    if [ -z "${src}" ] || [ -z "${dest}" ]; then
+      echo "Vault copy entry malformed: ${pair}" >&2
+      exit 1
+    fi
+    if [ ! -f "${src}" ]; then
+      echo "Vault file not found: ${src}" >&2
+      exit 1
+    fi
+    mkdir -p "$(dirname "${dest}")"
+    cp "${src}" "${dest}"
+  done
+  IFS="$old_ifs"
+fi
+
+exec "$@"

hosts/group_vars/all.yaml

@@ -0,0 +1,2 @@
+# hosts/group_vars/all.yaml
+validator_version: latest

hosts/host_vars/titan-24.yaml

@@ -0,0 +1,2 @@
+# hosts/host_vars/titan-24.yaml
+validator_compose_path: /opt/sui-validator

hosts/inventory/lab.yaml

@@ -0,0 +1,28 @@
+# hosts/inventory/lab.yaml
+# Replace ansible_host and ansible_user values with real connectivity details.
+all:
+  children:
+    atlas:
+      hosts:
+        titan-24:
+          ansible_host: REPLACE_ME
+          ansible_user: ubuntu
+          roleset: tethys_hybrid
+        titan-22:
+          ansible_host: REPLACE_ME
+          ansible_user: debian
+          roleset: minipc_gpu
+    baremetal:
+      hosts:
+        titan-db:
+          ansible_host: REPLACE_ME
+          ansible_user: postgres
+          roleset: database
+        titan-jh:
+          ansible_host: REPLACE_ME
+          ansible_user: jump
+          roleset: jumphost
+        oceanus:
+          ansible_host: REPLACE_ME
+          ansible_user: validator
+          roleset: validator

hosts/playbooks/site.yaml

@@ -0,0 +1,29 @@
+# hosts/playbooks/site.yaml
+---
+- name: Configure titan-db
+  hosts: titan-db
+  gather_facts: true
+  roles:
+    - common
+    - titan_db
+
+- name: Configure titan-jh
+  hosts: titan-jh
+  gather_facts: true
+  roles:
+    - common
+    - titan_jh
+
+- name: Configure oceanus validator host
+  hosts: oceanus
+  gather_facts: true
+  roles:
+    - common
+    - oceanus_base
+
+- name: Prepare hybrid tethys node
+  hosts: titan-24
+  gather_facts: true
+  roles:
+    - common
+    - tethys_canary

hosts/roles/common/tasks/main.yaml

@@ -0,0 +1,9 @@
+# hosts/roles/common/tasks/main.yaml
+---
+- name: Ensure base packages present
+  ansible.builtin.package:
+    name:
+      - curl
+      - vim
+    state: present
+  tags: ['common', 'packages']

hosts/roles/oceanus_base/tasks/main.yaml

@@ -0,0 +1,6 @@
+# hosts/roles/oceanus_base/tasks/main.yaml
+---
+- name: Placeholder for oceanus base configuration
+  ansible.builtin.debug:
+    msg: "Install validator prerequisites and monitoring exporters here."
+  tags: ['oceanus']

hosts/roles/tethys_canary/tasks/main.yaml

@@ -0,0 +1,6 @@
+# hosts/roles/tethys_canary/tasks/main.yaml
+---
+- name: Placeholder for SUI validator container runtime setup
+  ansible.builtin.debug:
+    msg: "Configure container runtime and validator compose stack here."
+  tags: ['tethys', 'validator']

hosts/roles/titan_db/tasks/main.yaml

@@ -0,0 +1,6 @@
+# hosts/roles/titan_db/tasks/main.yaml
+---
+- name: Placeholder for titan-db provisioning
+  ansible.builtin.debug:
+    msg: "Install database packages, configure backups, and manage users here."
+  tags: ['titan_db']

hosts/roles/titan_jh/tasks/main.yaml

@@ -0,0 +1,19 @@
+# hosts/roles/titan_jh/tasks/main.yaml
+---
+- name: Install node exporter
+  ansible.builtin.package:
+    name: prometheus-node-exporter
+    state: present
+  tags: ['jumphost', 'monitoring']
+
+- name: Enable node exporter
+  ansible.builtin.service:
+    name: prometheus-node-exporter
+    enabled: true
+    state: started
+  tags: ['jumphost', 'monitoring']
+
+- name: Placeholder for jumphost hardening
+  ansible.builtin.debug:
+    msg: "Harden SSH, manage bastion tooling, and configure audit logging here."
+  tags: ['jumphost']

infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml

@@ -0,0 +1,40 @@
+# infrastructure/cert-manager/cleanup/cert-manager-cleanup-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: cert-manager-cleanup-2
+  namespace: cert-manager
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      serviceAccountName: cert-manager-cleanup
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/worker
+                    operator: Exists
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: ["arm64"]
+      containers:
+        - name: cleanup
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command: ["/usr/bin/env", "bash"]
+          args: ["/scripts/cert_manager_cleanup.sh"]
+          volumeMounts:
+            - name: script
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: script
+          configMap:
+            name: cert-manager-cleanup-script
+            defaultMode: 0555

infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml

@@ -0,0 +1,58 @@
+# infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cert-manager-cleanup
+  namespace: cert-manager
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cert-manager-cleanup
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - services
+      - endpoints
+      - configmaps
+      - secrets
+      - serviceaccounts
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["apps"]
+    resources:
+      - deployments
+      - daemonsets
+      - statefulsets
+      - replicasets
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["batch"]
+    resources:
+      - jobs
+      - cronjobs
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["rbac.authorization.k8s.io"]
+    resources:
+      - roles
+      - rolebindings
+      - clusterroles
+      - clusterrolebindings
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["admissionregistration.k8s.io"]
+    resources:
+      - validatingwebhookconfigurations
+      - mutatingwebhookconfigurations
+    verbs: ["get", "list", "watch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cert-manager-cleanup
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cert-manager-cleanup
+subjects:
+  - kind: ServiceAccount
+    name: cert-manager-cleanup
+    namespace: cert-manager

infrastructure/cert-manager/cleanup/kustomization.yaml

@@ -0,0 +1,15 @@
+# infrastructure/cert-manager/cleanup/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - cert-manager-cleanup-rbac.yaml
+  - cert-manager-cleanup-job.yaml
+
+configMapGenerator:
+  - name: cert-manager-cleanup-script
+    namespace: cert-manager
+    files:
+      - cert_manager_cleanup.sh=scripts/cert_manager_cleanup.sh
+    options:
+      disableNameSuffixHash: true

infrastructure/cert-manager/cleanup/namespace.yaml

@@ -0,0 +1,5 @@
+# infrastructure/cert-manager/cleanup/namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager

infrastructure/cert-manager/cleanup/scripts/cert_manager_cleanup.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+namespace="cert-manager"
+selectors=(
+  "app.kubernetes.io/name=cert-manager"
+  "app.kubernetes.io/instance=cert-manager"
+  "app.kubernetes.io/instance=certmanager-prod"
+)
+
+delete_namespaced() {
+  local selector="$1"
+  kubectl -n "${namespace}" delete deployment,daemonset,statefulset,replicaset \
+    --selector "${selector}" --ignore-not-found --wait=false
+  kubectl -n "${namespace}" delete pod,service,endpoints,serviceaccount,configmap,secret \
+    --selector "${selector}" --ignore-not-found --wait=false
+  kubectl -n "${namespace}" delete role,rolebinding \
+    --selector "${selector}" --ignore-not-found --wait=false
+  kubectl -n "${namespace}" delete job,cronjob \
+    --selector "${selector}" --ignore-not-found --wait=false
+}
+
+delete_cluster_scoped() {
+  local selector="$1"
+  kubectl delete clusterrole,clusterrolebinding \
+    --selector "${selector}" --ignore-not-found --wait=false
+  kubectl delete mutatingwebhookconfiguration,validatingwebhookconfiguration \
+    --selector "${selector}" --ignore-not-found --wait=false
+}
+
+for selector in "${selectors[@]}"; do
+  delete_namespaced "${selector}"
+  delete_cluster_scoped "${selector}"
+done
+
+kubectl delete mutatingwebhookconfiguration cert-manager-webhook --ignore-not-found --wait=false
+kubectl delete validatingwebhookconfiguration cert-manager-webhook --ignore-not-found --wait=false

infrastructure/cert-manager/helmrelease.yaml

@@ -0,0 +1,67 @@
+# infrastructure/cert-manager/helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: cert-manager
+  namespace: cert-manager
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: cert-manager
+      version: v1.17.0
+      sourceRef:
+        kind: HelmRepository
+        name: jetstack
+        namespace: flux-system
+  install:
+    crds: CreateReplace
+    remediation: { retries: 3 }
+    timeout: 10m
+  upgrade:
+    crds: CreateReplace
+    remediation:
+      retries: 3
+      remediateLastFailure: true
+    cleanupOnFail: true
+    timeout: 10m
+  values:
+    installCRDs: true
+    nodeSelector:
+      node-role.kubernetes.io/worker: "true"
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                    - rpi5
+                    - rpi4
+    webhook:
+      nodeSelector:
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+                      - rpi4
+    cainjector:
+      nodeSelector:
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+                      - rpi4

infrastructure/cert-manager/kustomization.yaml

@@ -0,0 +1,6 @@
+# infrastructure/cert-manager/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helmrelease.yaml

infrastructure/cert-manager/namespace.yaml

@@ -0,0 +1,5 @@
+# infrastructure/cert-manager/namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: cert-manager

infrastructure/core/coredns-custom.yaml

@@ -0,0 +1,44 @@
+# infrastructure/core/coredns-custom.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns-custom
+  namespace: kube-system
+data:
+  bstein-dev.server: |
+    bstein.dev:53 {
+      errors
+      cache 30
+      hosts {
+        192.168.22.9 alerts.bstein.dev
+        192.168.22.9 auth.bstein.dev
+        192.168.22.9 bstein.dev
+        10.43.6.87 budget.bstein.dev
+        192.168.22.9 call.live.bstein.dev
+        192.168.22.9 cd.bstein.dev
+        192.168.22.9 chat.ai.bstein.dev
+        192.168.22.9 ci.bstein.dev
+        192.168.22.9 cloud.bstein.dev
+        192.168.22.9 health.bstein.dev
+        192.168.22.9 kit.live.bstein.dev
+        192.168.22.9 live.bstein.dev
+        192.168.22.9 logs.bstein.dev
+        192.168.22.9 longhorn.bstein.dev
+        192.168.22.4 mail.bstein.dev
+        192.168.22.9 matrix.live.bstein.dev
+        192.168.22.9 metrics.bstein.dev
+        192.168.22.9 monero.bstein.dev
+        10.43.6.87 money.bstein.dev
+        192.168.22.9 notes.bstein.dev
+        192.168.22.9 office.bstein.dev
+        192.168.22.9 pegasus.bstein.dev
+        192.168.22.9 registry.bstein.dev
+        192.168.22.9 scm.bstein.dev
+        192.168.22.9 secret.bstein.dev
+        192.168.22.9 sso.bstein.dev
+        192.168.22.9 stream.bstein.dev
+        192.168.22.9 tasks.bstein.dev
+        192.168.22.9 vault.bstein.dev
+        fallthrough
+      }
+    }

infrastructure/core/coredns-deployment.yaml

@@ -0,0 +1,141 @@
+# infrastructure/core/coredns-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/name: CoreDNS
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 2
+  revisionHistoryLimit: 0
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 25%
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-dns
+    spec:
+      containers:
+        - name: coredns
+          image: registry.bstein.dev/infra/coredns:1.12.1
+          imagePullPolicy: IfNotPresent
+          args:
+            - -conf
+            - /etc/coredns/Corefile
+          ports:
+            - containerPort: 53
+              name: dns
+              protocol: UDP
+            - containerPort: 53
+              name: dns-tcp
+              protocol: TCP
+            - containerPort: 9153
+              name: metrics
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+              scheme: HTTP
+            initialDelaySeconds: 60
+            periodSeconds: 10
+            timeoutSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: 8181
+              scheme: HTTP
+            periodSeconds: 2
+            timeoutSeconds: 1
+            successThreshold: 1
+            failureThreshold: 3
+          resources:
+            limits:
+              memory: 170Mi
+            requests:
+              cpu: 100m
+              memory: 70Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              add:
+                - NET_BIND_SERVICE
+              drop:
+                - all
+            readOnlyRootFilesystem: true
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/coredns
+              readOnly: true
+            - name: custom-config-volume
+              mountPath: /etc/coredns/custom
+              readOnly: true
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
+                      - rpi4
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values:
+                      - "true"
+      dnsPolicy: Default
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      serviceAccountName: coredns
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+        - key: node-role.kubernetes.io/control-plane
+          operator: Exists
+          effect: NoSchedule
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              k8s-app: kube-dns
+        - maxSkew: 1
+          topologyKey: topology.kubernetes.io/zone
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              k8s-app: kube-dns
+      volumes:
+        - name: config-volume
+          configMap:
+            name: coredns
+            defaultMode: 420
+            items:
+              - key: Corefile
+                path: Corefile
+              - key: NodeHosts
+                path: NodeHosts
+        - name: custom-config-volume
+          configMap:
+            name: coredns-custom
+            optional: true
+            defaultMode: 420

infrastructure/core/gpu/profiles/jetson-only/kustomization.yaml

@@ -1,5 +0,0 @@
-# infrastructure/core/gpu/daemonsets/profiles/jetson-only/kustomization.yaml
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - ../../device-plugin-jetson

infrastructure/core/gpu/profiles/minipc-and-jetson/kustomization.yaml

@@ -1,6 +0,0 @@
-# infrastructure/core/gpu/daemonsets/profiles/minipc-and-jetson/kustomization.yaml
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - ../../device-plugin-minipc
-  - ../../device-plugin-jetson

infrastructure/core/kustomization.yaml

@@ -2,7 +2,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - base
-  # - gpu/profiles/jetson-only 
-  # - gpu/profiles/minipc-and-jetson
-  - gpu/profiles/minipc-only 
+  - ../modules/base
+  - ../modules/profiles/atlas-ha
+  - coredns-custom.yaml
+  - coredns-deployment.yaml
+  - ../sources/cert-manager/letsencrypt.yaml
+  - ../sources/cert-manager/letsencrypt-prod.yaml

infrastructure/flux-system/kustomization.yaml

@@ -2,15 +2,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- gotk-components.yaml
-- gotk-sync.yaml
-- kustomization-zot.yaml
-- kustomization-core.yaml
-- kustomization-helm.yaml
-- kustomization-gitea.yaml
-- kustomization-vault.yaml
-- kustomization-crypto.yaml
-- kustomization-monerod.yaml
-- kustomization-jellyfin.yaml
-- kustomization-xmr-miner.yaml
-- kustomization-longhorn-ui.yaml
+  - ../clusters/atlas/flux-system

infrastructure/longhorn/adopt/kustomization.yaml

@@ -0,0 +1,15 @@
+# infrastructure/longhorn/adopt/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - longhorn-adopt-rbac.yaml
+  - longhorn-helm-adopt-job.yaml
+
+configMapGenerator:
+  - name: longhorn-helm-adopt-script
+    namespace: longhorn-system
+    files:
+      - longhorn_helm_adopt.sh=scripts/longhorn_helm_adopt.sh
+    options:
+      disableNameSuffixHash: true

infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml

@@ -0,0 +1,56 @@
+# infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: longhorn-helm-adopt
+  namespace: longhorn-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: longhorn-helm-adopt
+rules:
+  - apiGroups: [""]
+    resources:
+      - configmaps
+      - services
+      - serviceaccounts
+      - secrets
+    verbs: ["get", "list", "watch", "patch", "update"]
+  - apiGroups: ["apps"]
+    resources:
+      - deployments
+      - daemonsets
+    verbs: ["get", "list", "watch", "patch", "update"]
+  - apiGroups: ["batch"]
+    resources:
+      - jobs
+    verbs: ["get", "list", "watch", "patch", "update"]
+  - apiGroups: ["rbac.authorization.k8s.io"]
+    resources:
+      - roles
+      - rolebindings
+      - clusterroles
+      - clusterrolebindings
+    verbs: ["get", "list", "watch", "patch", "update"]
+  - apiGroups: ["apiextensions.k8s.io"]
+    resources:
+      - customresourcedefinitions
+    verbs: ["get", "list", "watch", "patch", "update"]
+  - apiGroups: ["scheduling.k8s.io"]
+    resources:
+      - priorityclasses
+    verbs: ["get", "list", "watch", "patch", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: longhorn-helm-adopt
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: longhorn-helm-adopt
+subjects:
+  - kind: ServiceAccount
+    name: longhorn-helm-adopt
+    namespace: longhorn-system

infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml

@@ -0,0 +1,40 @@
+# infrastructure/longhorn/adopt/longhorn-helm-adopt-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: longhorn-helm-adopt-2
+  namespace: longhorn-system
+spec:
+  backoffLimit: 1
+  template:
+    spec:
+      serviceAccountName: longhorn-helm-adopt
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/worker
+                    operator: Exists
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: ["arm64"]
+      containers:
+        - name: adopt
+          image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
+          command: ["/usr/bin/env", "bash"]
+          args: ["/scripts/longhorn_helm_adopt.sh"]
+          volumeMounts:
+            - name: script
+              mountPath: /scripts
+              readOnly: true
+      volumes:
+        - name: script
+          configMap:
+            name: longhorn-helm-adopt-script
+            defaultMode: 0555

infrastructure/longhorn/adopt/namespace.yaml

@@ -0,0 +1,5 @@
+# infrastructure/longhorn/adopt/namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system

infrastructure/longhorn/adopt/scripts/longhorn_helm_adopt.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+release_name="longhorn"
+release_namespace="longhorn-system"
+selector="app.kubernetes.io/instance=${release_name}"
+
+annotate_and_label() {
+  local scope="$1"
+  local kind="$2"
+  if [ "${scope}" = "namespaced" ]; then
+    kubectl -n "${release_namespace}" annotate "${kind}" -l "${selector}" \
+      meta.helm.sh/release-name="${release_name}" \
+      meta.helm.sh/release-namespace="${release_namespace}" \
+      --overwrite >/dev/null 2>&1 || true
+    kubectl -n "${release_namespace}" label "${kind}" -l "${selector}" \
+      app.kubernetes.io/managed-by=Helm --overwrite >/dev/null 2>&1 || true
+  else
+    kubectl annotate "${kind}" -l "${selector}" \
+      meta.helm.sh/release-name="${release_name}" \
+      meta.helm.sh/release-namespace="${release_namespace}" \
+      --overwrite >/dev/null 2>&1 || true
+    kubectl label "${kind}" -l "${selector}" \
+      app.kubernetes.io/managed-by=Helm --overwrite >/dev/null 2>&1 || true
+  fi
+}
+
+namespaced_kinds=(
+  configmap
+  service
+  serviceaccount
+  deployment
+  daemonset
+  job
+  role
+  rolebinding
+)
+
+cluster_kinds=(
+  clusterrole
+  clusterrolebinding
+  customresourcedefinition
+  priorityclass
+)
+
+for kind in "${namespaced_kinds[@]}"; do
+  annotate_and_label "namespaced" "${kind}"
+done
+
+for kind in "${cluster_kinds[@]}"; do
+  annotate_and_label "cluster" "${kind}"
+done

infrastructure/longhorn/core/helmrelease.yaml

@@ -0,0 +1,80 @@
+# infrastructure/longhorn/core/helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: longhorn
+      version: 1.8.2
+      sourceRef:
+        kind: HelmRepository
+        name: longhorn
+        namespace: flux-system
+  install:
+    crds: Skip
+    remediation: { retries: 3 }
+    timeout: 15m
+  upgrade:
+    crds: Skip
+    remediation:
+      retries: 3
+      remediateLastFailure: true
+    cleanupOnFail: true
+    timeout: 15m
+  values:
+    service:
+      ui:
+        type: NodePort
+        nodePort: 30824
+    privateRegistry:
+      createSecret: false
+      registrySecret: longhorn-registry
+    image:
+      pullPolicy: Always
+      longhorn:
+        engine:
+          repository: registry.bstein.dev/infra/longhorn-engine
+          tag: v1.8.2
+        manager:
+          repository: registry.bstein.dev/infra/longhorn-manager
+          tag: v1.8.2
+        ui:
+          repository: registry.bstein.dev/infra/longhorn-ui
+          tag: v1.8.2
+        instanceManager:
+          repository: registry.bstein.dev/infra/longhorn-instance-manager
+          tag: v1.8.2
+        shareManager:
+          repository: registry.bstein.dev/infra/longhorn-share-manager
+          tag: v1.8.2
+        backingImageManager:
+          repository: registry.bstein.dev/infra/longhorn-backing-image-manager
+          tag: v1.8.2
+        supportBundleKit:
+          repository: registry.bstein.dev/infra/longhorn-support-bundle-kit
+          tag: v0.0.56
+      csi:
+        attacher:
+          repository: registry.bstein.dev/infra/longhorn-csi-attacher
+          tag: v4.9.0
+        provisioner:
+          repository: registry.bstein.dev/infra/longhorn-csi-provisioner
+          tag: v5.3.0
+        nodeDriverRegistrar:
+          repository: registry.bstein.dev/infra/longhorn-csi-node-driver-registrar
+          tag: v2.14.0
+        resizer:
+          repository: registry.bstein.dev/infra/longhorn-csi-resizer
+          tag: v1.13.2
+        snapshotter:
+          repository: registry.bstein.dev/infra/longhorn-csi-snapshotter
+          tag: v8.2.0
+        livenessProbe:
+          repository: registry.bstein.dev/infra/longhorn-livenessprobe
+          tag: v2.16.0
+    defaultSettings:
+      systemManagedPodsImagePullPolicy: Always

infrastructure/longhorn/core/kustomization.yaml

@@ -0,0 +1,18 @@
+# infrastructure/longhorn/core/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - vault-serviceaccount.yaml
+  - secretproviderclass.yaml
+  - vault-sync-deployment.yaml
+  - helmrelease.yaml
+  - longhorn-settings-ensure-job.yaml
+
+configMapGenerator:
+  - name: longhorn-settings-ensure-script
+    files:
+      - longhorn_settings_ensure.sh=scripts/longhorn_settings_ensure.sh
+
+generatorOptions:
+  disableNameSuffixHash: true