logging: verify recovered capacity before clearing the index block

services/logging/opensearch-single-node-tune-check-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: opensearch-single-node-tune-check-1
+  name: opensearch-single-node-tune-check-2
   namespace: logging
 spec:
   backoffLimit: 0
@@ -27,7 +27,7 @@ spec:
         image: python:3.11-alpine
         command:
         - python
-        - /scripts/tune.py
+        - -c
         env:
         - name: OPENSEARCH_URL
           value: http://opensearch-master.logging.svc.cluster.local:9200
@@ -41,6 +41,17 @@ spec:
         volumeMounts:
         - name: scripts
           mountPath: /scripts
+        args:
+        - "import runpy\nscope = runpy.run_path(\"/scripts/tune.py\")\nrequest = scope[\"\
+          request_json\"]\nallocation = request(\"/_cat/allocation?format=json&bytes=b\"\
+          )\nassert allocation and all(int(row[\"disk.percent\"]) < 80 for row in\
+          \ allocation), \"Capacity guard failed\"\nsettings = request(\"/_cluster/settings?include_defaults=true&flat_settings=true\"\
+          )\nassert settings.get(\"defaults\", {}).get(\"cluster.blocks.create_index.auto_release\"\
+          ) == \"true\", \"Automatic release guard failed\"\nif settings.get(\"persistent\"\
+          , {}).get(\"cluster.blocks.create_index\") == \"true\":\n    scope[\"put\"\
+          ](\"/_cluster/settings\", {\"persistent\": {\"cluster.blocks.create_index\"\
+          : None}})\n    print(\"capacity_block_released_after_verified_expansion\"\
+          , flush=True)\nscope[\"main\"]()\n"
       volumes:
       - name: scripts
         configMap: