maintenance: harden k3s traefik disable cleanup

services/maintenance/disable-k3s-traefik-rbac.yaml

@@ -0,0 +1,39 @@
+# services/maintenance/disable-k3s-traefik-rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: disable-k3s-traefik
+rules:
+  - apiGroups: ["k3s.cattle.io"]
+    resources: ["addons"]
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["helm.cattle.io"]
+    resources: ["helmcharts", "helmchartconfigs"]
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: [""]
+    resources: ["services", "serviceaccounts"]
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["apps"]
+    resources: ["deployments"]
+    verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["rbac.authorization.k8s.io"]
+    resources: ["clusterroles", "clusterrolebindings"]
+    verbs: ["get", "list", "watch", "delete"]
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: disable-k3s-traefik
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: disable-k3s-traefik
+subjects:
+  - kind: ServiceAccount
+    name: disable-k3s-traefik
+    namespace: maintenance

services/maintenance/k3s-traefik-cleanup-rbac.yaml

@@ -12,9 +12,15 @@ kind: ClusterRole
 metadata:
   name: k3s-traefik-cleanup
 rules:
+  - apiGroups: ["k3s.cattle.io"]
+    resources: ["addons"]
+    verbs: ["get", "list", "watch", "delete"]
   - apiGroups: ["helm.cattle.io"]
     resources: ["helmcharts", "helmchartconfigs"]
     verbs: ["get", "list", "watch", "delete"]
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "delete"]
   - apiGroups: [""]
     resources: ["services", "serviceaccounts"]
     verbs: ["get", "list", "watch", "delete"]

services/maintenance/kustomization.yaml

@@ -12,6 +12,7 @@ resources:
   - ariadne-serviceaccount.yaml
   - ariadne-rbac.yaml
   - disable-k3s-traefik-serviceaccount.yaml
+  - disable-k3s-traefik-rbac.yaml
   - k3s-traefik-cleanup-rbac.yaml
   - metis-serviceaccount.yaml
   - metis-rbac.yaml

services/maintenance/scripts/disable_k3s_traefik.sh

@@ -54,11 +54,28 @@ restart_k3s() {
   chroot "${host_root}" /bin/systemctl restart k3s
 }
 
-ensure_disable_flag
-remove_manifest
+cleanup_cluster_objects() {
+  kubectl -n kube-system delete addon traefik --ignore-not-found --wait=false >/dev/null 2>&1 || true
+  kubectl -n kube-system delete helmchart traefik traefik-crd --ignore-not-found --wait=false >/dev/null 2>&1 || true
+  kubectl -n kube-system delete job helm-install-traefik helm-install-traefik-crd --ignore-not-found --wait=false >/dev/null 2>&1 || true
+  kubectl -n kube-system delete deployment traefik --ignore-not-found --wait=false >/dev/null 2>&1 || true
+  kubectl -n kube-system delete service traefik --ignore-not-found --wait=false >/dev/null 2>&1 || true
+  kubectl -n kube-system delete serviceaccount traefik helm-traefik helm-traefik-crd --ignore-not-found --wait=false >/dev/null 2>&1 || true
+  kubectl delete clusterrole traefik-ingress-controller traefik-kube-system --ignore-not-found --wait=false >/dev/null 2>&1 || true
+  kubectl delete clusterrolebinding helm-kube-system-traefik helm-kube-system-traefik-crd traefik-ingress-controller traefik-kube-system --ignore-not-found --wait=false >/dev/null 2>&1 || true
+}
 
-if [ "${changed}" -eq 1 ]; then
-  restart_k3s
-fi
+while true; do
+  changed=0
+  ensure_disable_flag
+  remove_manifest
 
-sleep infinity
+  if [ "${changed}" -eq 1 ]; then
+    restart_k3s
+    sleep 15
+    remove_manifest
+  fi
+
+  cleanup_cluster_objects
+  sleep 300
+done

services/maintenance/scripts/k3s_traefik_cleanup.sh

@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+kubectl -n kube-system delete addon traefik --ignore-not-found --wait=false
 kubectl -n kube-system delete helmchart traefik traefik-crd --ignore-not-found --wait=false
+kubectl -n kube-system delete job helm-install-traefik helm-install-traefik-crd --ignore-not-found --wait=false
 kubectl -n kube-system delete deployment traefik --ignore-not-found --wait=false
 kubectl -n kube-system delete service traefik --ignore-not-found --wait=false
 kubectl -n kube-system delete serviceaccount traefik helm-traefik helm-traefik-crd --ignore-not-found --wait=false