From 56c959da95ad8ce56318c888df51ba5b3b210e20 Mon Sep 17 00:00:00 2001
From: jenkins <brad@bstein.dev>
Date: Sun, 4 Oct 2026 03:52:08 -0500
Subject: [PATCH] longhorn: scope recovery access to the stranded engine pod

---
 .../longhorn/core/kustomization.yaml          |  1 +
 .../scripts/longhorn_stale_engine_recovery.sh |  5 +-
 .../longhorn/core/tenant2-recovery-rbac.yaml  | 46 +++++++++++++++++++
 .../tenant2-stale-engine-recovery-job.yaml    |  4 +-
 4 files changed, 52 insertions(+), 4 deletions(-)
 create mode 100644 infrastructure/longhorn/core/tenant2-recovery-rbac.yaml

diff --git a/infrastructure/longhorn/core/kustomization.yaml b/infrastructure/longhorn/core/kustomization.yaml
index 846e04a7..d212bc61 100644
--- a/infrastructure/longhorn/core/kustomization.yaml
+++ b/infrastructure/longhorn/core/kustomization.yaml
@@ -9,6 +9,7 @@ resources:
   - helmrelease.yaml
   - cassandra-recurring-jobs.yaml
   - monerod-recovery-snapshot.yaml
+  - tenant2-recovery-rbac.yaml
   - tenant2-stale-engine-recovery-job.yaml
   - longhorn-settings-ensure-job.yaml
   - longhorn-csi-toleration-ensure-job.yaml
diff --git a/infrastructure/longhorn/core/scripts/longhorn_stale_engine_recovery.sh b/infrastructure/longhorn/core/scripts/longhorn_stale_engine_recovery.sh
index 7ba80eed..88cf5332 100644
--- a/infrastructure/longhorn/core/scripts/longhorn_stale_engine_recovery.sh
+++ b/infrastructure/longhorn/core/scripts/longhorn_stale_engine_recovery.sh
@@ -34,7 +34,8 @@ main() {
     .volume == $id and .all_copies_completed == true and
     (.nodes | sort == ["titan-13", "titan-15", "titan-17"])
   ' /recovery/copy-receipt.json >/dev/null
-  if [ -e /recovery/engine-reset-attempted ]; then
+  # The first Job's exec was forbidden before action; retain its earlier marker.
+  if [ -e /recovery/engine-reset-authorized-attempt ]; then
     echo "Recovery attempt already recorded; no action."
     return 0
   fi
@@ -48,7 +49,7 @@ main() {
     return 0
   fi
   # Record before acting so neither Flux nor a failed client can repeat the reset.
-  (set -o noclobber; date -u +%FT%TZ > /recovery/engine-reset-attempted)
+  (set -o noclobber; date -u +%FT%TZ > /recovery/engine-reset-authorized-attempt)
   kubectl -n longhorn-system exec "$manager" -- \
     longhorn-instance-manager process delete --name "$engine" >/dev/null
   echo "Requested one native reset of the stranded engine; controllers own recovery."
diff --git a/infrastructure/longhorn/core/tenant2-recovery-rbac.yaml b/infrastructure/longhorn/core/tenant2-recovery-rbac.yaml
new file mode 100644
index 00000000..fec69134
--- /dev/null
+++ b/infrastructure/longhorn/core/tenant2-recovery-rbac.yaml
@@ -0,0 +1,46 @@
+# infrastructure/longhorn/core/tenant2-recovery-rbac.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: tenant2-engine-recovery
+  namespace: longhorn-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: tenant2-engine-recovery
+  namespace: longhorn-system
+rules:
+  - apiGroups: ["longhorn.io"]
+    resources: ["volumes"]
+    resourceNames: ["pvc-02d99a30-3757-4a01-abfb-5b30aef793d6"]
+    verbs: ["get"]
+  - apiGroups: ["longhorn.io"]
+    resources: ["engines"]
+    resourceNames: ["pvc-02d99a30-3757-4a01-abfb-5b30aef793d6-e-0"]
+    verbs: ["get"]
+  - apiGroups: ["longhorn.io"]
+    resources: ["replicas"]
+    verbs: ["list"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    resourceNames: ["instance-manager-60fcef8ab4aad7407ce6d857ad2340f6"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    resourceNames: ["instance-manager-60fcef8ab4aad7407ce6d857ad2340f6"]
+    verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: tenant2-engine-recovery
+  namespace: longhorn-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: tenant2-engine-recovery
+subjects:
+  - kind: ServiceAccount
+    name: tenant2-engine-recovery
+    namespace: longhorn-system
diff --git a/infrastructure/longhorn/core/tenant2-stale-engine-recovery-job.yaml b/infrastructure/longhorn/core/tenant2-stale-engine-recovery-job.yaml
index 89a16a7e..8cd55f23 100644
--- a/infrastructure/longhorn/core/tenant2-stale-engine-recovery-job.yaml
+++ b/infrastructure/longhorn/core/tenant2-stale-engine-recovery-job.yaml
@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: tenant2-stale-engine-recovery-1
+  name: tenant2-stale-engine-recovery-2
   namespace: longhorn-system
 spec:
   backoffLimit: 0
@@ -10,7 +10,7 @@ spec:
   # Keep the completed Job; the host receipt also prevents repeated execution.
   template:
     spec:
-      serviceAccountName: longhorn-service-account
+      serviceAccountName: tenant2-engine-recovery
       restartPolicy: Never
       nodeSelector:
         kubernetes.io/hostname: titan-13