package cluster import ( "context" "encoding/json" "fmt" "sort" "strings" "time" ) type staleRWOOwnerDecision struct { Reason string ForceDelete bool Unsafe bool } type persistentVolumeClaimList struct { Items []persistentVolumeClaimResource `json:"items"` } type persistentVolumeClaimResource struct { Metadata struct { Namespace string `json:"namespace"` Name string `json:"name"` } `json:"metadata"` Spec struct { AccessModes []string `json:"accessModes"` VolumeName string `json:"volumeName"` } `json:"spec"` } // staleRWOPVCOwnerDecisions finds stale terminating single-writer pods safely. // Signature: (o *Orchestrator) staleRWOPVCOwnerDecisions(ctx context.Context, pods podList, grace time.Duration) (map[string]staleRWOOwnerDecision, error). // Why: RWO volume handoff needs container/mount-level safety, not blind repeated // pod deletion, so sidecar-only stale owners can clear while live writers block. func (o *Orchestrator) staleRWOPVCOwnerDecisions(ctx context.Context, pods podList, grace time.Duration) (map[string]staleRWOOwnerDecision, error) { eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json") if err != nil { return nil, fmt.Errorf("query events for stale RWO owner scan: %w", err) } var events eventList if strings.TrimSpace(eventsOut) != "" { if err := json.Unmarshal([]byte(eventsOut), &events); err != nil { return nil, fmt.Errorf("decode events for stale RWO owner scan: %w", err) } } pvcs, err := o.queryPVCs(ctx) if err != nil { return nil, err } now := time.Now() podsByController := map[string][]podResource{} for _, pod := range pods.Items { controllerKey := podControllerKey(pod) if controllerKey == "" { continue } podsByController[controllerKey] = append(podsByController[controllerKey], pod) } decisions := map[string]staleRWOOwnerDecision{} for _, oldPod := range pods.Items { oldKey := podKey(oldPod) if oldKey == "" || oldPod.Metadata.DeletionTimestamp == nil || !podControllerOwned(oldPod) { continue } if now.Sub(*oldPod.Metadata.DeletionTimestamp) < grace { continue } oldNode := strings.TrimSpace(oldPod.Spec.NodeName) if oldNode == "" { continue } blockedClaims := podRWOPVCVolumeNames(oldPod, pvcs) if len(blockedClaims) == 0 { continue } controllerKey := podControllerKey(oldPod) replacements := podsByController[controllerKey] replacementKey := "" for _, replacement := range replacements { if podKey(replacement) == oldKey { continue } if strings.TrimSpace(replacement.Spec.NodeName) == "" || strings.TrimSpace(replacement.Spec.NodeName) == oldNode { continue } if !replacementPodPendingForAttach(replacement) { continue } if !podHasBlockedAttachEvent(replacement, events) { continue } if !podsShareAnyClaim(oldPod, replacement, blockedClaims) { continue } replacementKey = podKey(replacement) break } if replacementKey == "" { continue } unsafe := runningContainersMountAnyVolume(oldPod, blockedClaims) claims := make([]string, 0, len(blockedClaims)) for _, claim := range blockedClaims { claims = append(claims, claim) } sort.Strings(claims) if unsafe { decisions[oldKey] = staleRWOOwnerDecision{ Reason: fmt.Sprintf("UnsafeStaleRWOPVCOwner:%s:%s", oldNode, strings.Join(claims, ",")), Unsafe: true, } continue } decisions[oldKey] = staleRWOOwnerDecision{ Reason: fmt.Sprintf("SidecarOnlyStaleRWOPVCOwner:%s:%s->%s", oldNode, strings.Join(claims, ","), replacementKey), ForceDelete: true, } } return decisions, nil } // queryPVCs runs one orchestration or CLI step. // Signature: (o *Orchestrator) queryPVCs(ctx context.Context) (map[string]persistentVolumeClaimResource, error). // Why: stale-owner recovery must confirm single-writer PVC semantics before it // considers force-deleting a terminating pod object. func (o *Orchestrator) queryPVCs(ctx context.Context) (map[string]persistentVolumeClaimResource, error) { out, err := o.kubectl(ctx, 30*time.Second, "get", "pvc", "-A", "-o", "json") if err != nil { return nil, fmt.Errorf("query pvcs for stale RWO owner scan: %w", err) } pvcs := map[string]persistentVolumeClaimResource{} if strings.TrimSpace(out) == "" { return pvcs, nil } var list persistentVolumeClaimList if err := json.Unmarshal([]byte(out), &list); err != nil { return nil, fmt.Errorf("decode pvcs for stale RWO owner scan: %w", err) } for _, pvc := range list.Items { key := strings.TrimSpace(pvc.Metadata.Namespace) + "/" + strings.TrimSpace(pvc.Metadata.Name) if strings.TrimSpace(pvc.Metadata.Namespace) != "" && strings.TrimSpace(pvc.Metadata.Name) != "" { pvcs[key] = pvc } } return pvcs, nil } // podRWOPVCVolumeNames runs one orchestration or CLI step. // Signature: podRWOPVCVolumeNames(pod podResource, pvcs map[string]persistentVolumeClaimResource) map[string]string. // Why: container mount safety checks need the pod volume names that correspond // to ReadWriteOnce PVC claims. func podRWOPVCVolumeNames(pod podResource, pvcs map[string]persistentVolumeClaimResource) map[string]string { volumes := map[string]string{} for _, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim == nil { continue } claim := strings.TrimSpace(volume.PersistentVolumeClaim.ClaimName) if claim == "" { continue } pvc, ok := pvcs[strings.TrimSpace(pod.Metadata.Namespace)+"/"+claim] if !ok || !pvcSingleWriter(pvc) { continue } volumes[strings.TrimSpace(volume.Name)] = claim } return volumes } // pvcSingleWriter runs one orchestration or CLI step. // Signature: pvcSingleWriter(pvc persistentVolumeClaimResource) bool. // Why: the stale-owner path is limited to exclusive-writer PVC modes and should // leave shared volumes to normal Kubernetes cleanup. func pvcSingleWriter(pvc persistentVolumeClaimResource) bool { for _, mode := range pvc.Spec.AccessModes { normalized := strings.ToLower(strings.TrimSpace(mode)) if normalized == "readwriteonce" || normalized == "readwriteoncepod" { return true } } return false } // podControllerKey runs one orchestration or CLI step. // Signature: podControllerKey(pod podResource) string. // Why: stale owner and replacement pods should be grouped by controller without // hard-coding ReplicaSet, StatefulSet, or application names. func podControllerKey(pod podResource) string { ns := strings.TrimSpace(pod.Metadata.Namespace) for _, owner := range pod.Metadata.OwnerReferences { kind := strings.TrimSpace(owner.Kind) name := strings.TrimSpace(owner.Name) if ns != "" && kind != "" && name != "" { return ns + "/" + kind + "/" + name } } return "" } // podKey runs one orchestration or CLI step. // Signature: podKey(pod podResource) string. // Why: recovery incidents need stable namespace/name keys for pod maps and // operator summaries. func podKey(pod podResource) string { ns := strings.TrimSpace(pod.Metadata.Namespace) name := strings.TrimSpace(pod.Metadata.Name) if ns == "" || name == "" { return "" } return ns + "/" + name } // replacementPodPendingForAttach runs one orchestration or CLI step. // Signature: replacementPodPendingForAttach(pod podResource) bool. // Why: stale RWO recovery should only act when a replacement is actually blocked // during scheduling or initialization. func replacementPodPendingForAttach(pod podResource) bool { phase := strings.TrimSpace(pod.Status.Phase) if strings.EqualFold(phase, "Pending") { return true } for _, st := range append(append([]podContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) { if st.State.Waiting == nil { continue } reason := strings.TrimSpace(st.State.Waiting.Reason) if strings.EqualFold(reason, "ContainerCreating") || strings.EqualFold(reason, "PodInitializing") { return true } } return false } // podHasBlockedAttachEvent runs one orchestration or CLI step. // Signature: podHasBlockedAttachEvent(pod podResource, events eventList) bool. // Why: force deletion should be tied to concrete Multi-Attach or exclusive-use // evidence rather than any old terminating PVC pod. func podHasBlockedAttachEvent(pod podResource, events eventList) bool { key := podKey(pod) for _, event := range events.Items { if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") { continue } eventKey := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name) if eventKey != key { continue } if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") { continue } message := strings.ToLower(strings.TrimSpace(event.Message)) reason := strings.ToLower(strings.TrimSpace(event.Reason)) if reason == "failedattachvolume" || strings.Contains(message, "multi-attach") || strings.Contains(message, "volume is already used by pod") || strings.Contains(message, "already exclusively attached") { return true } } return false } // podsShareAnyClaim runs one orchestration or CLI step. // Signature: podsShareAnyClaim(oldPod podResource, replacement podResource, oldClaims map[string]string) bool. // Why: the old stale pod and replacement must be competing for the same PVC // before Ananke treats them as one storage handoff incident. func podsShareAnyClaim(oldPod podResource, replacement podResource, oldClaims map[string]string) bool { replacementClaims := map[string]struct{}{} for _, volume := range replacement.Spec.Volumes { if volume.PersistentVolumeClaim == nil { continue } claim := strings.TrimSpace(volume.PersistentVolumeClaim.ClaimName) if claim != "" { replacementClaims[claim] = struct{}{} } } for _, claim := range oldClaims { if _, ok := replacementClaims[claim]; ok { return true } } return false } // runningContainersMountAnyVolume runs one orchestration or CLI step. // Signature: runningContainersMountAnyVolume(pod podResource, volumeNames map[string]string) bool. // Why: a live app container mounting the blocked PVC is unsafe to clear, while // sidecars without that mount can be handled as stale API ownership. func runningContainersMountAnyVolume(pod podResource, volumeNames map[string]string) bool { if len(volumeNames) == 0 { return false } specByName := map[string]podContainer{} for _, c := range pod.Spec.Containers { specByName[strings.TrimSpace(c.Name)] = c } for _, status := range pod.Status.ContainerStatuses { if status.State.Running == nil { continue } spec, ok := specByName[strings.TrimSpace(status.Name)] if !ok { return true } for _, mount := range spec.VolumeMounts { if _, ok := volumeNames[strings.TrimSpace(mount.Name)]; ok { return true } } } return false }