ananke/internal/cluster/orchestrator_stale_rwo_owner.go

package cluster

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"
	"strings"
	"time"
)

type staleRWOOwnerDecision struct {
	Reason      string
	ForceDelete bool
	Unsafe      bool
}

type persistentVolumeClaimList struct {
	Items []persistentVolumeClaimResource `json:"items"`
}

type persistentVolumeClaimResource struct {
	Metadata struct {
		Namespace string `json:"namespace"`
		Name      string `json:"name"`
	} `json:"metadata"`
	Spec struct {
		AccessModes []string `json:"accessModes"`
		VolumeName  string   `json:"volumeName"`
	} `json:"spec"`
}

// staleRWOPVCOwnerDecisions finds stale terminating single-writer pods safely.
// Signature: (o *Orchestrator) staleRWOPVCOwnerDecisions(ctx context.Context, pods podList, grace time.Duration) (map[string]staleRWOOwnerDecision, error).
// Why: RWO volume handoff needs container/mount-level safety, not blind repeated
// pod deletion, so sidecar-only stale owners can clear while live writers block.
func (o *Orchestrator) staleRWOPVCOwnerDecisions(ctx context.Context, pods podList, grace time.Duration) (map[string]staleRWOOwnerDecision, error) {
	eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
	if err != nil {
		return nil, fmt.Errorf("query events for stale RWO owner scan: %w", err)
	}
	var events eventList
	if strings.TrimSpace(eventsOut) != "" {
		if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
			return nil, fmt.Errorf("decode events for stale RWO owner scan: %w", err)
		}
	}
	pvcs, err := o.queryPVCs(ctx)
	if err != nil {
		return nil, err
	}

	now := time.Now()
	podsByController := map[string][]podResource{}
	for _, pod := range pods.Items {
		controllerKey := podControllerKey(pod)
		if controllerKey == "" {
			continue
		}
		podsByController[controllerKey] = append(podsByController[controllerKey], pod)
	}

	decisions := map[string]staleRWOOwnerDecision{}
	for _, oldPod := range pods.Items {
		oldKey := podKey(oldPod)
		if oldKey == "" || oldPod.Metadata.DeletionTimestamp == nil || !podControllerOwned(oldPod) {
			continue
		}
		if now.Sub(*oldPod.Metadata.DeletionTimestamp) < grace {
			continue
		}
		oldNode := strings.TrimSpace(oldPod.Spec.NodeName)
		if oldNode == "" {
			continue
		}
		blockedClaims := podRWOPVCVolumeNames(oldPod, pvcs)
		if len(blockedClaims) == 0 {
			continue
		}
		controllerKey := podControllerKey(oldPod)
		replacements := podsByController[controllerKey]
		replacementKey := ""
		for _, replacement := range replacements {
			if podKey(replacement) == oldKey {
				continue
			}
			if strings.TrimSpace(replacement.Spec.NodeName) == "" || strings.TrimSpace(replacement.Spec.NodeName) == oldNode {
				continue
			}
			if !replacementPodPendingForAttach(replacement) {
				continue
			}
			if !podHasBlockedAttachEvent(replacement, events) {
				continue
			}
			if !podsShareAnyClaim(oldPod, replacement, blockedClaims) {
				continue
			}
			replacementKey = podKey(replacement)
			break
		}
		if replacementKey == "" {
			continue
		}

		unsafe := runningContainersMountAnyVolume(oldPod, blockedClaims)
		claims := make([]string, 0, len(blockedClaims))
		for _, claim := range blockedClaims {
			claims = append(claims, claim)
		}
		sort.Strings(claims)
		if unsafe {
			decisions[oldKey] = staleRWOOwnerDecision{
				Reason: fmt.Sprintf("UnsafeStaleRWOPVCOwner:%s:%s", oldNode, strings.Join(claims, ",")),
				Unsafe: true,
			}
			continue
		}
		decisions[oldKey] = staleRWOOwnerDecision{
			Reason:      fmt.Sprintf("SidecarOnlyStaleRWOPVCOwner:%s:%s->%s", oldNode, strings.Join(claims, ","), replacementKey),
			ForceDelete: true,
		}
	}
	return decisions, nil
}

// queryPVCs runs one orchestration or CLI step.
// Signature: (o *Orchestrator) queryPVCs(ctx context.Context) (map[string]persistentVolumeClaimResource, error).
// Why: stale-owner recovery must confirm single-writer PVC semantics before it
// considers force-deleting a terminating pod object.
func (o *Orchestrator) queryPVCs(ctx context.Context) (map[string]persistentVolumeClaimResource, error) {
	out, err := o.kubectl(ctx, 30*time.Second, "get", "pvc", "-A", "-o", "json")
	if err != nil {
		return nil, fmt.Errorf("query pvcs for stale RWO owner scan: %w", err)
	}
	pvcs := map[string]persistentVolumeClaimResource{}
	if strings.TrimSpace(out) == "" {
		return pvcs, nil
	}
	var list persistentVolumeClaimList
	if err := json.Unmarshal([]byte(out), &list); err != nil {
		return nil, fmt.Errorf("decode pvcs for stale RWO owner scan: %w", err)
	}
	for _, pvc := range list.Items {
		key := strings.TrimSpace(pvc.Metadata.Namespace) + "/" + strings.TrimSpace(pvc.Metadata.Name)
		if strings.TrimSpace(pvc.Metadata.Namespace) != "" && strings.TrimSpace(pvc.Metadata.Name) != "" {
			pvcs[key] = pvc
		}
	}
	return pvcs, nil
}

// podRWOPVCVolumeNames runs one orchestration or CLI step.
// Signature: podRWOPVCVolumeNames(pod podResource, pvcs map[string]persistentVolumeClaimResource) map[string]string.
// Why: container mount safety checks need the pod volume names that correspond
// to ReadWriteOnce PVC claims.
func podRWOPVCVolumeNames(pod podResource, pvcs map[string]persistentVolumeClaimResource) map[string]string {
	volumes := map[string]string{}
	for _, volume := range pod.Spec.Volumes {
		if volume.PersistentVolumeClaim == nil {
			continue
		}
		claim := strings.TrimSpace(volume.PersistentVolumeClaim.ClaimName)
		if claim == "" {
			continue
		}
		pvc, ok := pvcs[strings.TrimSpace(pod.Metadata.Namespace)+"/"+claim]
		if !ok || !pvcSingleWriter(pvc) {
			continue
		}
		volumes[strings.TrimSpace(volume.Name)] = claim
	}
	return volumes
}

// pvcSingleWriter runs one orchestration or CLI step.
// Signature: pvcSingleWriter(pvc persistentVolumeClaimResource) bool.
// Why: the stale-owner path is limited to exclusive-writer PVC modes and should
// leave shared volumes to normal Kubernetes cleanup.
func pvcSingleWriter(pvc persistentVolumeClaimResource) bool {
	for _, mode := range pvc.Spec.AccessModes {
		normalized := strings.ToLower(strings.TrimSpace(mode))
		if normalized == "readwriteonce" || normalized == "readwriteoncepod" {
			return true
		}
	}
	return false
}

// podControllerKey runs one orchestration or CLI step.
// Signature: podControllerKey(pod podResource) string.
// Why: stale owner and replacement pods should be grouped by controller without
// hard-coding ReplicaSet, StatefulSet, or application names.
func podControllerKey(pod podResource) string {
	ns := strings.TrimSpace(pod.Metadata.Namespace)
	for _, owner := range pod.Metadata.OwnerReferences {
		kind := strings.TrimSpace(owner.Kind)
		name := strings.TrimSpace(owner.Name)
		if ns != "" && kind != "" && name != "" {
			return ns + "/" + kind + "/" + name
		}
	}
	return ""
}

// podKey runs one orchestration or CLI step.
// Signature: podKey(pod podResource) string.
// Why: recovery incidents need stable namespace/name keys for pod maps and
// operator summaries.
func podKey(pod podResource) string {
	ns := strings.TrimSpace(pod.Metadata.Namespace)
	name := strings.TrimSpace(pod.Metadata.Name)
	if ns == "" || name == "" {
		return ""
	}
	return ns + "/" + name
}

// replacementPodPendingForAttach runs one orchestration or CLI step.
// Signature: replacementPodPendingForAttach(pod podResource) bool.
// Why: stale RWO recovery should only act when a replacement is actually blocked
// during scheduling or initialization.
func replacementPodPendingForAttach(pod podResource) bool {
	phase := strings.TrimSpace(pod.Status.Phase)
	if strings.EqualFold(phase, "Pending") {
		return true
	}
	for _, st := range append(append([]podContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) {
		if st.State.Waiting == nil {
			continue
		}
		reason := strings.TrimSpace(st.State.Waiting.Reason)
		if strings.EqualFold(reason, "ContainerCreating") || strings.EqualFold(reason, "PodInitializing") {
			return true
		}
	}
	return false
}

// podHasBlockedAttachEvent runs one orchestration or CLI step.
// Signature: podHasBlockedAttachEvent(pod podResource, events eventList) bool.
// Why: force deletion should be tied to concrete Multi-Attach or exclusive-use
// evidence rather than any old terminating PVC pod.
func podHasBlockedAttachEvent(pod podResource, events eventList) bool {
	key := podKey(pod)
	for _, event := range events.Items {
		if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
			continue
		}
		eventKey := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name)
		if eventKey != key {
			continue
		}
		if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
			continue
		}
		message := strings.ToLower(strings.TrimSpace(event.Message))
		reason := strings.ToLower(strings.TrimSpace(event.Reason))
		if reason == "failedattachvolume" ||
			strings.Contains(message, "multi-attach") ||
			strings.Contains(message, "volume is already used by pod") ||
			strings.Contains(message, "already exclusively attached") {
			return true
		}
	}
	return false
}

// podsShareAnyClaim runs one orchestration or CLI step.
// Signature: podsShareAnyClaim(oldPod podResource, replacement podResource, oldClaims map[string]string) bool.
// Why: the old stale pod and replacement must be competing for the same PVC
// before Ananke treats them as one storage handoff incident.
func podsShareAnyClaim(oldPod podResource, replacement podResource, oldClaims map[string]string) bool {
	replacementClaims := map[string]struct{}{}
	for _, volume := range replacement.Spec.Volumes {
		if volume.PersistentVolumeClaim == nil {
			continue
		}
		claim := strings.TrimSpace(volume.PersistentVolumeClaim.ClaimName)
		if claim != "" {
			replacementClaims[claim] = struct{}{}
		}
	}
	for _, claim := range oldClaims {
		if _, ok := replacementClaims[claim]; ok {
			return true
		}
	}
	return false
}

// runningContainersMountAnyVolume runs one orchestration or CLI step.
// Signature: runningContainersMountAnyVolume(pod podResource, volumeNames map[string]string) bool.
// Why: a live app container mounting the blocked PVC is unsafe to clear, while
// sidecars without that mount can be handled as stale API ownership.
func runningContainersMountAnyVolume(pod podResource, volumeNames map[string]string) bool {
	if len(volumeNames) == 0 {
		return false
	}
	specByName := map[string]podContainer{}
	for _, c := range pod.Spec.Containers {
		specByName[strings.TrimSpace(c.Name)] = c
	}
	for _, status := range pod.Status.ContainerStatuses {
		if status.State.Running == nil {
			continue
		}
		spec, ok := specByName[strings.TrimSpace(status.Name)]
		if !ok {
			return true
		}
		for _, mount := range spec.VolumeMounts {
			if _, ok := volumeNames[strings.TrimSpace(mount.Name)]; ok {
				return true
			}
		}
	}
	return false
}