logging: keep maintenance from preempting running work

infrastructure/modules/base/priorityclass/kustomization.yaml

@@ -5,3 +5,4 @@ resources:
   - scavenger.yaml
   - cassandra.yaml
   - media.yaml
+  - maintenance-batch.yaml

infrastructure/modules/base/priorityclass/maintenance-batch.yaml

@@ -0,0 +1,9 @@
+# infrastructure/modules/base/priorityclass/maintenance-batch.yaml
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: maintenance-batch
+value: 0
+globalDefault: false
+preemptionPolicy: Never
+description: "Routine maintenance waits for capacity instead of evicting running work"

services/logging/kustomization.yaml

@@ -18,6 +18,7 @@ resources:
   - bootstrap-jobs/opensearch-dashboards-setup-job.yaml
   - bootstrap-jobs/opensearch-observability-setup-job.yaml
   - opensearch-single-node-tune-cronjob.yaml
+  - opensearch-maintenance-placement-check-job.yaml
   - opensearch-prune-cronjob.yaml
   - fluent-bit-helmrelease.yaml
   - node-log-rotation-daemonset.yaml

services/logging/opensearch-maintenance-placement-check-job.yaml

@@ -0,0 +1,48 @@
+# services/logging/opensearch-maintenance-placement-check-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: opensearch-maintenance-placement-check-v1
+  namespace: logging
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 300
+  template:
+    spec:
+      restartPolicy: OnFailure
+      priorityClassName: maintenance-batch
+      automountServiceAccountToken: false
+      nodeSelector:
+        node-role.kubernetes.io/worker: 'true'
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: hardware
+                operator: In
+                values:
+                - rpi5
+                - rpi4
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: hardware
+                operator: In
+                values:
+                - rpi5
+      containers:
+      - name: check
+        image: python@sha256:0495f5559318affa673172ec7e35cd0a5213e4aaf4c76d0a66554c0af97b157e
+        command:
+        - python
+        - -c
+        - import urllib.request; r=urllib.request.urlopen("http://opensearch-master.logging.svc.cluster.local:9200/_cluster/health", timeout=20); assert r.status == 200; r.close(); print("OpenSearch health HTTP 200")
+        resources:
+          requests:
+            cpu: 10m
+            memory: 32Mi
+          limits:
+            cpu: 200m
+            memory: 128Mi

services/logging/opensearch-single-node-tune-cronjob.yaml

@@ -16,9 +16,10 @@ spec:
       template:
         spec:
           restartPolicy: OnFailure
+          priorityClassName: maintenance-batch
+          automountServiceAccountToken: false
           nodeSelector:
             node-role.kubernetes.io/worker: "true"
-            hardware: rpi5
           affinity:
             nodeAffinity:
               requiredDuringSchedulingIgnoredDuringExecution:
@@ -28,9 +29,17 @@ spec:
                         operator: In
                         values:
                           - rpi5
+                          - rpi4
+              preferredDuringSchedulingIgnoredDuringExecution:
+                - weight: 100
+                  preference:
+                    matchExpressions:
+                      - key: hardware
+                        operator: In
+                        values: [rpi5]
           containers:
             - name: tune
-              image: python:3.11-alpine
+              image: python@sha256:0495f5559318affa673172ec7e35cd0a5213e4aaf4c76d0a66554c0af97b157e
               command: ["python", "/scripts/tune.py"]
               env:
                 - name: OPENSEARCH_URL