diff --git a/services/logging/kustomization.yaml b/services/logging/kustomization.yaml index 3f3a25e..3c40da2 100644 --- a/services/logging/kustomization.yaml +++ b/services/logging/kustomization.yaml @@ -7,6 +7,10 @@ resources: - opensearch-observability-objects.yaml - node-log-rotation-serviceaccount.yaml - node-log-rotation-script.yaml + - node-image-gc-rpi4-serviceaccount.yaml + - node-image-gc-rpi4-script.yaml + - node-image-prune-rpi5-serviceaccount.yaml + - node-image-prune-rpi5-script.yaml - opensearch-pvc.yaml - opensearch-helmrelease.yaml - opensearch-dashboards-helmrelease.yaml @@ -18,5 +22,7 @@ resources: - opensearch-prune-cronjob.yaml - fluent-bit-helmrelease.yaml - node-log-rotation-daemonset.yaml + - node-image-gc-rpi4-daemonset.yaml + - node-image-prune-rpi5-daemonset.yaml - oauth2-proxy.yaml - ingress.yaml diff --git a/services/logging/node-image-gc-rpi4-daemonset.yaml b/services/logging/node-image-gc-rpi4-daemonset.yaml new file mode 100644 index 0000000..70bace5 --- /dev/null +++ b/services/logging/node-image-gc-rpi4-daemonset.yaml @@ -0,0 +1,49 @@ +# services/logging/node-image-gc-rpi4-daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-image-gc-rpi4 + namespace: logging +spec: + selector: + matchLabels: + app: node-image-gc-rpi4 + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: node-image-gc-rpi4 + spec: + serviceAccountName: node-image-gc-rpi4 + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + nodeSelector: + hardware: rpi4 + containers: + - name: node-image-gc-rpi4 + image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 + command: ["/usr/bin/env", "bash"] + args: ["/scripts/node_image_gc_rpi4.sh"] + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - name: host-root + mountPath: /host + - name: script + mountPath: /scripts + readOnly: true + volumes: + - name: host-root + hostPath: + path: / + - name: script + configMap: + name: node-image-gc-rpi4-script + defaultMode: 0555 diff --git a/services/logging/node-image-gc-rpi4-script.yaml b/services/logging/node-image-gc-rpi4-script.yaml new file mode 100644 index 0000000..44c4c16 --- /dev/null +++ b/services/logging/node-image-gc-rpi4-script.yaml @@ -0,0 +1,44 @@ +# services/logging/node-image-gc-rpi4-script.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-image-gc-rpi4-script + namespace: logging +data: + node_image_gc_rpi4.sh: | + #!/usr/bin/env bash + set -euo pipefail + + changed=0 + k3s_changed=0 + k3s_agent_changed=0 + + k3s_dropin="/host/etc/systemd/system/k3s.service.d/98-image-gc.conf" + k3s_agent_dropin="/host/etc/systemd/system/k3s-agent.service.d/98-image-gc.conf" + + if [ -f "/host/etc/systemd/system/k3s.service" ] && [ ! -f "${k3s_dropin}" ]; then + mkdir -p "$(dirname "${k3s_dropin}")" + printf "[Service]\nEnvironment=\"K3S_KUBELET_ARG=image-gc-high-threshold=70\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-low-threshold=60\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-minimum-available=5Gi\"\n" > "${k3s_dropin}" + changed=1 + k3s_changed=1 + fi + + if [ -f "/host/etc/systemd/system/k3s-agent.service" ] && [ ! -f "${k3s_agent_dropin}" ]; then + mkdir -p "$(dirname "${k3s_agent_dropin}")" + printf "[Service]\nEnvironment=\"K3S_KUBELET_ARG=image-gc-high-threshold=70\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-low-threshold=60\"\nEnvironment=\"K3S_KUBELET_ARG=image-gc-minimum-available=5Gi\"\n" > "${k3s_agent_dropin}" + changed=1 + k3s_agent_changed=1 + fi + + if [ "${changed}" -eq 1 ]; then + sleep "$(( (RANDOM % 300) + 10 ))" + chroot /host /bin/systemctl daemon-reload + if [ "${k3s_changed}" -eq 1 ]; then + chroot /host /bin/systemctl restart k3s + fi + if [ "${k3s_agent_changed}" -eq 1 ]; then + chroot /host /bin/systemctl restart k3s-agent + fi + fi + + sleep infinity diff --git a/services/logging/node-image-gc-rpi4-serviceaccount.yaml b/services/logging/node-image-gc-rpi4-serviceaccount.yaml new file mode 100644 index 0000000..ec1eb41 --- /dev/null +++ b/services/logging/node-image-gc-rpi4-serviceaccount.yaml @@ -0,0 +1,6 @@ +# services/logging/node-image-gc-rpi4-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-image-gc-rpi4 + namespace: logging diff --git a/services/logging/node-image-prune-rpi5-daemonset.yaml b/services/logging/node-image-prune-rpi5-daemonset.yaml new file mode 100644 index 0000000..63fa471 --- /dev/null +++ b/services/logging/node-image-prune-rpi5-daemonset.yaml @@ -0,0 +1,49 @@ +# services/logging/node-image-prune-rpi5-daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-image-prune-rpi5 + namespace: logging +spec: + selector: + matchLabels: + app: node-image-prune-rpi5 + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: node-image-prune-rpi5 + spec: + serviceAccountName: node-image-prune-rpi5 + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + nodeSelector: + hardware: rpi5 + containers: + - name: node-image-prune-rpi5 + image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 + command: ["/usr/bin/env", "bash"] + args: ["/scripts/node_image_prune_rpi5.sh"] + securityContext: + privileged: true + runAsUser: 0 + volumeMounts: + - name: host-root + mountPath: /host + - name: script + mountPath: /scripts + readOnly: true + volumes: + - name: host-root + hostPath: + path: / + - name: script + configMap: + name: node-image-prune-rpi5-script + defaultMode: 0555 diff --git a/services/logging/node-image-prune-rpi5-script.yaml b/services/logging/node-image-prune-rpi5-script.yaml new file mode 100644 index 0000000..ae79ce3 --- /dev/null +++ b/services/logging/node-image-prune-rpi5-script.yaml @@ -0,0 +1,34 @@ +# services/logging/node-image-prune-rpi5-script.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-image-prune-rpi5-script + namespace: logging +data: + node_image_prune_rpi5.sh: | + #!/usr/bin/env bash + set -euo pipefail + + threshold=70 + + sleep "$(( (RANDOM % 300) + 10 ))" + + while true; do + usage=$(df -P /host | awk 'NR==2 {gsub(/%/,"",$5); print $5}') + if [ -z "${usage}" ]; then + sleep 1800 + continue + fi + + if [ "${usage}" -ge "${threshold}" ]; then + chroot /host /bin/sh -c ' + if command -v crictl >/dev/null 2>&1; then + crictl --runtime-endpoint=unix:///run/k3s/containerd/containerd.sock rmi --prune || true + elif [ -x /usr/local/bin/crictl ]; then + /usr/local/bin/crictl --runtime-endpoint=unix:///run/k3s/containerd/containerd.sock rmi --prune || true + fi + ' + fi + + sleep 21600 + done diff --git a/services/logging/node-image-prune-rpi5-serviceaccount.yaml b/services/logging/node-image-prune-rpi5-serviceaccount.yaml new file mode 100644 index 0000000..938c78a --- /dev/null +++ b/services/logging/node-image-prune-rpi5-serviceaccount.yaml @@ -0,0 +1,6 @@ +# services/logging/node-image-prune-rpi5-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-image-prune-rpi5 + namespace: logging