From bb05ec9f7a6304521b4c2297dc86dc40dd64eece Mon Sep 17 00:00:00 2001
From: jenkins <brad@bstein.dev>
Date: Sun, 4 Oct 2026 03:47:23 -0500
Subject: [PATCH] maintenance: fit log guard and retain completed storage setup
 jobs

---
 .../core/longhorn-csi-toleration-ensure-job.yaml  | 15 ++++++++++-----
 .../core/longhorn-disk-tags-ensure-job.yaml       | 12 ++++++++++--
 .../core/longhorn-settings-ensure-job.yaml        | 13 +++++++++----
 services/logging/node-log-rotation-daemonset.yaml | 11 +++++++++++
 4 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/infrastructure/longhorn/core/longhorn-csi-toleration-ensure-job.yaml b/infrastructure/longhorn/core/longhorn-csi-toleration-ensure-job.yaml
index 8b084dac..e5b36ca1 100644
--- a/infrastructure/longhorn/core/longhorn-csi-toleration-ensure-job.yaml
+++ b/infrastructure/longhorn/core/longhorn-csi-toleration-ensure-job.yaml
@@ -2,18 +2,16 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: longhorn-csi-toleration-ensure-4
+  name: longhorn-csi-toleration-ensure-5
   namespace: longhorn-system
 spec:
   backoffLimit: 0
   activeDeadlineSeconds: 240
-  ttlSecondsAfterFinished: 3600
+  # Retain completion; Flux would recreate a TTL-deleted bootstrap job.
   template:
     spec:
       serviceAccountName: longhorn-service-account
       restartPolicy: Never
-      nodeSelector:
-        kubernetes.io/hostname: titan-11
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -27,7 +25,14 @@ spec:
       containers:
         - name: patch
           image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
-          command: ["/bin/sh", "-c"]
+          command: ["/bin/bash", "-c"]
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
           args:
             - |
               set -euo pipefail
diff --git a/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml b/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml
index 4d64d9a0..2724b443 100644
--- a/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml
+++ b/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml
@@ -2,11 +2,12 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: longhorn-disk-tags-ensure-4
+  name: longhorn-disk-tags-ensure-5
   namespace: longhorn-system
 spec:
   backoffLimit: 0
-  ttlSecondsAfterFinished: 3600
+  activeDeadlineSeconds: 240
+  # Retain completion; Flux would recreate a TTL-deleted bootstrap job.
   template:
     spec:
       serviceAccountName: longhorn-service-account
@@ -30,6 +31,13 @@ spec:
         - name: apply
           image: python:3.12.9-alpine3.20
           command: ["python", "/scripts/longhorn_disk_tags_ensure.py"]
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
           volumeMounts:
             - name: longhorn-disk-tags-ensure-script
               mountPath: /scripts
diff --git a/infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml b/infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml
index 5a98d6ca..ddce0c38 100644
--- a/infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml
+++ b/infrastructure/longhorn/core/longhorn-settings-ensure-job.yaml
@@ -2,18 +2,16 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: longhorn-settings-ensure-11
+  name: longhorn-settings-ensure-12
   namespace: longhorn-system
 spec:
   backoffLimit: 0
   activeDeadlineSeconds: 240
-  ttlSecondsAfterFinished: 3600
+  # Retain completion; Flux would recreate a TTL-deleted bootstrap job.
   template:
     spec:
       serviceAccountName: longhorn-service-account
       restartPolicy: Never
-      nodeSelector:
-        kubernetes.io/hostname: titan-11
       volumes:
         - name: longhorn-settings-ensure-script
           configMap:
@@ -33,6 +31,13 @@ spec:
         - name: apply
           image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
           command: ["/scripts/longhorn_settings_ensure.sh"]
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
           volumeMounts:
             - name: longhorn-settings-ensure-script
               mountPath: /scripts
diff --git a/services/logging/node-log-rotation-daemonset.yaml b/services/logging/node-log-rotation-daemonset.yaml
index 99216abf..1adcd94f 100644
--- a/services/logging/node-log-rotation-daemonset.yaml
+++ b/services/logging/node-log-rotation-daemonset.yaml
@@ -10,6 +10,9 @@ spec:
       app: node-log-rotation
   updateStrategy:
     type: RollingUpdate
+    rollingUpdate:
+      # Allow one healthy-node update while two nodes remain offline.
+      maxUnavailable: 3
   template:
     metadata:
       annotations:
@@ -42,6 +45,14 @@ spec:
           image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
           command: ["/usr/bin/env", "bash"]
           args: ["/scripts/node_log_rotation.sh"]
+          resources:
+            # The 24-hour working-set peak was 8 MiB; the loop normally sleeps.
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
           securityContext:
             privileged: true
             runAsUser: 0