logging: retain safe diagnostics for the failing maintenance job

services/logging/kustomization.yaml

@@ -18,6 +18,7 @@ resources:
   - bootstrap-jobs/opensearch-dashboards-setup-job.yaml
   - bootstrap-jobs/opensearch-observability-setup-job.yaml
   - opensearch-single-node-tune-cronjob.yaml
+  - opensearch-single-node-tune-check-job.yaml
   - opensearch-prune-cronjob.yaml
   - fluent-bit-helmrelease.yaml
   - node-log-rotation-daemonset.yaml

services/logging/opensearch-single-node-tune-check-job.yaml

@@ -0,0 +1,47 @@
+# services/logging/opensearch-single-node-tune-check-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: opensearch-single-node-tune-check-1
+  namespace: logging
+spec:
+  backoffLimit: 0
+  activeDeadlineSeconds: 300
+  template:
+    spec:
+      restartPolicy: Never
+      nodeSelector:
+        node-role.kubernetes.io/worker: 'true'
+        hardware: rpi5
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: hardware
+                operator: In
+                values:
+                - rpi5
+      containers:
+      - name: tune
+        image: python:3.11-alpine
+        command:
+        - python
+        - /scripts/tune.py
+        env:
+        - name: OPENSEARCH_URL
+          value: http://opensearch-master.logging.svc.cluster.local:9200
+        resources:
+          requests:
+            cpu: 10m
+            memory: 32Mi
+          limits:
+            cpu: 200m
+            memory: 128Mi
+        volumeMounts:
+        - name: scripts
+          mountPath: /scripts
+      volumes:
+      - name: scripts
+        configMap:
+          name: opensearch-single-node-tune-script

services/logging/opensearch-single-node-tune-cronjob.yaml

@@ -12,6 +12,7 @@ spec:
   jobTemplate:
     spec:
       backoffLimit: 2
+      activeDeadlineSeconds: 300
       template:
         spec:
           restartPolicy: OnFailure

services/logging/scripts/opensearch_single_node_tune.py

@@ -14,8 +14,15 @@ def request_json(path, method="GET", payload=None):
         data = json.dumps(payload).encode("utf-8")
         headers["Content-Type"] = "application/json"
     request = urllib.request.Request(f"{OS_URL}{path}", data=data, headers=headers, method=method)
+    try:
         with urllib.request.urlopen(request, timeout=30) as response:
             body = response.read().decode("utf-8")
+    except urllib.error.HTTPError as error:
+        # Settings paths and status codes identify the failed operation without
+        # printing log documents or server response bodies.
+        print(json.dumps({"operation": method, "path": path,
+                          "http_status": error.code}), flush=True)
+        raise
     return json.loads(body) if body else {}