titan-iac/services/logging/oneoffs/opensearch-ism-job.yaml

# services/logging/oneoffs/opensearch-ism-job.yaml
# One-off job for logging/opensearch-ism-setup-5.
# Purpose: opensearch ism setup 5 (see container args/env in this file).
# Run by setting spec.suspend to false, reconcile, then set it back to true.
# Safe to delete the finished Job/pod; it should not run continuously.
apiVersion: batch/v1
kind: Job
metadata:
  name: opensearch-ism-setup-5
  namespace: logging
spec:
  suspend: true
  backoffLimit: 3
  ttlSecondsAfterFinished: 3600
  template:
    spec:
      restartPolicy: OnFailure
      nodeSelector:
        node-role.kubernetes.io/worker: "true"
        hardware: rpi5
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                - key: hardware
                  operator: In
                  values:
                    - rpi5
      containers:
        - name: apply
          image: alpine:3.20
          command: ["/bin/sh", "-c"]
          args:
            - |
              set -euo pipefail
              apk add --no-cache curl >/dev/null

              OS_URL="http://opensearch-master.logging.svc.cluster.local:9200"
              for attempt in $(seq 1 60); do
                if curl -s -o /dev/null -w "%{http_code}" "${OS_URL}" | grep -q "200"; then
                  break
                fi
                sleep 5
              done
              if ! curl -s -o /dev/null -w "%{http_code}" "${OS_URL}" | grep -q "200"; then
                echo "OpenSearch did not become ready in time" >&2
                exit 1
              fi

              policy='{"policy":{"description":"Delete logs after 180 days","schema_version":1,"default_state":"hot","states":[{"name":"hot","actions":[],"transitions":[{"state_name":"delete","conditions":{"min_index_age":"180d"}}]},{"name":"delete","actions":[{"delete":{}}],"transitions":[]}]}}'
              curl -sS -X PUT "${OS_URL}/_plugins/_ism/policies/logging-180d" \
                -H 'Content-Type: application/json' \
                -d "${policy}" >/dev/null

              trace_policy='{"policy":{"description":"Delete trace analytics after 30 days","schema_version":1,"default_state":"hot","states":[{"name":"hot","actions":[],"transitions":[{"state_name":"delete","conditions":{"min_index_age":"30d"}}]},{"name":"delete","actions":[{"delete":{}}],"transitions":[]}]}}'
              curl -sS -X PUT "${OS_URL}/_plugins/_ism/policies/trace-analytics-30d" \
                -H 'Content-Type: application/json' \
                -d "${trace_policy}" >/dev/null

              kube_template='{"index_patterns":["kube-*"],"priority":200,"template":{"settings":{"index.number_of_shards":1,"index.number_of_replicas":0,"index.refresh_interval":"30s","plugins.index_state_management.policy_id":"logging-180d"},"mappings":{"properties":{"@timestamp":{"type":"date"}}}}}'
              curl -sS -X PUT "${OS_URL}/_index_template/kube-logs" \
                -H 'Content-Type: application/json' \
                -d "${kube_template}" >/dev/null

              journal_template='{"index_patterns":["journald-*"],"priority":200,"template":{"settings":{"index.number_of_shards":1,"index.number_of_replicas":0,"index.refresh_interval":"30s","plugins.index_state_management.policy_id":"logging-180d"},"mappings":{"properties":{"@timestamp":{"type":"date"}}}}}'
              curl -sS -X PUT "${OS_URL}/_index_template/journald-logs" \
                -H 'Content-Type: application/json' \
                -d "${journal_template}" >/dev/null

              trace_template='{"index_patterns":["trace-analytics-*"],"priority":200,"template":{"settings":{"index.number_of_shards":1,"index.number_of_replicas":0,"index.refresh_interval":"30s","plugins.index_state_management.policy_id":"trace-analytics-30d"}}}'
              curl -sS -X PUT "${OS_URL}/_index_template/trace-analytics" \
                -H 'Content-Type: application/json' \
                -d "${trace_template}" >/dev/null

              curl -sS -X PUT "${OS_URL}/_all/_settings" \
                -H 'Content-Type: application/json' \
                -d '{"index":{"number_of_replicas":0}}' >/dev/null