longhorn: scope recovery access to the stranded engine pod

infrastructure/longhorn/core/kustomization.yaml

@@ -9,6 +9,7 @@ resources:
   - helmrelease.yaml
   - cassandra-recurring-jobs.yaml
   - monerod-recovery-snapshot.yaml
+  - tenant2-recovery-rbac.yaml
   - tenant2-stale-engine-recovery-job.yaml
   - longhorn-settings-ensure-job.yaml
   - longhorn-csi-toleration-ensure-job.yaml

infrastructure/longhorn/core/scripts/longhorn_stale_engine_recovery.sh

@@ -34,7 +34,8 @@ main() {
     .volume == $id and .all_copies_completed == true and
     (.nodes | sort == ["titan-13", "titan-15", "titan-17"])
   ' /recovery/copy-receipt.json >/dev/null
-  if [ -e /recovery/engine-reset-attempted ]; then
+  # The first Job's exec was forbidden before action; retain its earlier marker.
+  if [ -e /recovery/engine-reset-authorized-attempt ]; then
     echo "Recovery attempt already recorded; no action."
     return 0
   fi
@@ -48,7 +49,7 @@ main() {
     return 0
   fi
   # Record before acting so neither Flux nor a failed client can repeat the reset.
-  (set -o noclobber; date -u +%FT%TZ > /recovery/engine-reset-attempted)
+  (set -o noclobber; date -u +%FT%TZ > /recovery/engine-reset-authorized-attempt)
   kubectl -n longhorn-system exec "$manager" -- \
     longhorn-instance-manager process delete --name "$engine" >/dev/null
   echo "Requested one native reset of the stranded engine; controllers own recovery."

infrastructure/longhorn/core/tenant2-recovery-rbac.yaml

@@ -0,0 +1,46 @@
+# infrastructure/longhorn/core/tenant2-recovery-rbac.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: tenant2-engine-recovery
+  namespace: longhorn-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: tenant2-engine-recovery
+  namespace: longhorn-system
+rules:
+  - apiGroups: ["longhorn.io"]
+    resources: ["volumes"]
+    resourceNames: ["pvc-02d99a30-3757-4a01-abfb-5b30aef793d6"]
+    verbs: ["get"]
+  - apiGroups: ["longhorn.io"]
+    resources: ["engines"]
+    resourceNames: ["pvc-02d99a30-3757-4a01-abfb-5b30aef793d6-e-0"]
+    verbs: ["get"]
+  - apiGroups: ["longhorn.io"]
+    resources: ["replicas"]
+    verbs: ["list"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    resourceNames: ["instance-manager-60fcef8ab4aad7407ce6d857ad2340f6"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["pods/exec"]
+    resourceNames: ["instance-manager-60fcef8ab4aad7407ce6d857ad2340f6"]
+    verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: tenant2-engine-recovery
+  namespace: longhorn-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: tenant2-engine-recovery
+subjects:
+  - kind: ServiceAccount
+    name: tenant2-engine-recovery
+    namespace: longhorn-system

infrastructure/longhorn/core/tenant2-stale-engine-recovery-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: tenant2-stale-engine-recovery-1
+  name: tenant2-stale-engine-recovery-2
   namespace: longhorn-system
 spec:
   backoffLimit: 0
@@ -10,7 +10,7 @@ spec:
   # Keep the completed Job; the host receipt also prevents repeated execution.
   template:
     spec:
-      serviceAccountName: longhorn-service-account
+      serviceAccountName: tenant2-engine-recovery
       restartPolicy: Never
       nodeSelector:
         kubernetes.io/hostname: titan-13