From 225793facd27ea0dc89a21d27e54332f3b3d64aa Mon Sep 17 00:00:00 2001
From: jenkins <brad@bstein.dev>
Date: Sun, 23 Aug 2026 14:50:02 -0300
Subject: [PATCH] fix(mailu): self-heal mailbox lock failures

---
 services/mailu/helmrelease.yaml      |  33 ++++++-
 services/mailu/kustomization.yaml    |   1 +
 services/mailu/mailbox-watchdog.yaml | 133 +++++++++++++++++++++++++++
 3 files changed, 165 insertions(+), 2 deletions(-)
 create mode 100644 services/mailu/mailbox-watchdog.yaml

diff --git a/services/mailu/helmrelease.yaml b/services/mailu/helmrelease.yaml
index 078925d3..2de943d0 100644
--- a/services/mailu/helmrelease.yaml
+++ b/services/mailu/helmrelease.yaml
@@ -208,9 +208,10 @@ spec:
     dovecot:
       logLevel: DEBUG
       nodeSelector:
-        hardware: rpi4
+        hardware: rpi5
+        node-role.kubernetes.io/worker: "true"
       podAnnotations:
-        bstein.dev/storage-remount-at: "2026-08-22T17:49:52Z"
+        bstein.dev/storage-remount-at: "2026-08-23T17:55:00Z"
     oletools:
       logLevel: DEBUG
       nodeSelector:
@@ -575,6 +576,8 @@ spec:
               metadata:
                 name: mailu-dovecot
               spec:
+                strategy:
+                  type: Recreate
                 template:
                   metadata:
                     annotations:
@@ -634,6 +637,32 @@ spec:
                             $patch: delete
                           - name: VAULT_ENV_FILE
                             value: /vault/secrets/mailu-env.sh
+                        livenessProbe:
+                          exec:
+                            command:
+                              - sh
+                              - -ec
+                              - >-
+                                kill -0 "$(cat /run/dovecot/master.pid)" &&
+                                doveadm mailbox status -u brad@bstein.dev uidnext INBOX >/dev/null
+                          initialDelaySeconds: 30
+                          periodSeconds: 30
+                          timeoutSeconds: 10
+                          failureThreshold: 2
+                          successThreshold: 1
+                        readinessProbe:
+                          exec:
+                            command:
+                              - sh
+                              - -ec
+                              - >-
+                                kill -0 "$(cat /run/dovecot/master.pid)" &&
+                                doveadm mailbox status -u brad@bstein.dev uidnext INBOX >/dev/null
+                          initialDelaySeconds: 20
+                          periodSeconds: 15
+                          timeoutSeconds: 10
+                          failureThreshold: 1
+                          successThreshold: 1
                         volumeMounts:
                           - name: mailu-vault-entrypoint
                             mountPath: /entrypoint.sh
diff --git a/services/mailu/kustomization.yaml b/services/mailu/kustomization.yaml
index 285ada23..074ff2f6 100644
--- a/services/mailu/kustomization.yaml
+++ b/services/mailu/kustomization.yaml
@@ -15,6 +15,7 @@ resources:
   - ingressroute.yaml
   - mailu-sync-cronjob.yaml
   - front-lb.yaml
+  - mailbox-watchdog.yaml
 
 configMapGenerator:
   - name: mailu-vault-env
diff --git a/services/mailu/mailbox-watchdog.yaml b/services/mailu/mailbox-watchdog.yaml
new file mode 100644
index 00000000..2f8db0bc
--- /dev/null
+++ b/services/mailu/mailbox-watchdog.yaml
@@ -0,0 +1,133 @@
+# services/mailu/mailbox-watchdog.yaml
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mailu-mailbox-watchdog
+  namespace: mailu-mailserver
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mailu-mailbox-watchdog
+  namespace: mailu-mailserver
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "delete"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mailu-mailbox-watchdog
+  namespace: mailu-mailserver
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: mailu-mailbox-watchdog
+subjects:
+  - kind: ServiceAccount
+    name: mailu-mailbox-watchdog
+    namespace: mailu-mailserver
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mailu-mailbox-watchdog
+  namespace: mailu-mailserver
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mailu-mailbox-watchdog
+  template:
+    metadata:
+      labels:
+        app: mailu-mailbox-watchdog
+    spec:
+      serviceAccountName: mailu-mailbox-watchdog
+      automountServiceAccountToken: true
+      nodeSelector:
+        hardware: rpi5
+        node-role.kubernetes.io/worker: "true"
+      containers:
+        - name: watchdog
+          image: registry.bstein.dev/bstein/kubectl:1.35.0
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sh", "-ec"]
+          env:
+            - name: HOME
+              value: /tmp
+          args:
+            - |
+              failures=0
+              last_pod=""
+              sleep 60
+              while true; do
+                pod="$(kubectl get pods \
+                  -l app.kubernetes.io/component=dovecot,app.kubernetes.io/instance=mailu \
+                  --field-selector=status.phase=Running \
+                  -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || true)"
+
+                if [ -z "${pod}" ]; then
+                  failures=0
+                  last_pod=""
+                  sleep 30
+                  continue
+                fi
+
+                if [ "${pod}" != "${last_pod}" ]; then
+                  failures=0
+                  last_pod="${pod}"
+                  echo "watching ${pod}; allowing startup grace"
+                  sleep 60
+                  continue
+                fi
+
+                if timeout 15s kubectl exec "${pod}" -c dovecot -- \
+                  doveadm mailbox status -u brad@bstein.dev uidnext INBOX >/dev/null 2>&1; then
+                  if [ "${failures}" -gt 0 ]; then
+                    echo "mailbox health recovered after ${failures} failed checks"
+                  fi
+                  failures=0
+                else
+                  failures=$((failures + 1))
+                  echo "mailbox health check failed (${failures}/4) for ${pod}" >&2
+                fi
+
+                if [ "${failures}" -ge 4 ]; then
+                  echo "recreating ${pod} to remount mailbox storage after persistent failures" >&2
+                  kubectl delete pod "${pod}" --wait=false
+                  failures=0
+                  last_pod=""
+                  sleep 180
+                  continue
+                fi
+
+                sleep 30
+              done
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              cpu: 100m
+              memory: 128Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 65532
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+      imagePullSecrets:
+        - name: harbor-regcred
+      volumes:
+        - name: tmp
+          emptyDir: {}