package cluster import ( "context" "encoding/json" "fmt" "strings" "time" ) // cleanupTerminatingPodsOnUnavailableNodes runs one orchestration or CLI step. // Signature: (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error). // Why: dead nodes can strand terminating pods indefinitely, so the daemon should // clear only that narrow failure class instead of leaving garbage behind forever. func (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error) { if o.runner.DryRun { return 0, nil } unavailable, err := o.unavailableNodeSet(ctx) if err != nil { return 0, err } if len(unavailable) == 0 { return 0, nil } out, err := o.kubectl(ctx, 30*time.Second, "get", "pods", "-A", "-o", "json") if err != nil { return 0, fmt.Errorf("query pods: %w", err) } var pods podDeleteList if err := json.Unmarshal([]byte(out), &pods); err != nil { return 0, fmt.Errorf("decode pods: %w", err) } grace := time.Duration(o.cfg.Startup.DeadNodeCleanupGraceSeconds) * time.Second now := time.Now() count := 0 for _, item := range pods.Items { if item.Metadata.DeletionTimestamp == nil || item.Spec.NodeName == "" { continue } if _, badNode := unavailable[item.Spec.NodeName]; !badNode { continue } if now.Sub(*item.Metadata.DeletionTimestamp) < grace { continue } o.log.Printf("warning: force deleting terminating pod %s/%s on unavailable node %s", item.Metadata.Namespace, item.Metadata.Name, item.Spec.NodeName) if _, err := o.kubectl( ctx, 20*time.Second, "-n", item.Metadata.Namespace, "delete", "pod", item.Metadata.Name, "--grace-period=0", "--force", "--wait=false", ); err != nil && !isNotFoundErr(err) { return count, fmt.Errorf("delete pod %s/%s: %w", item.Metadata.Namespace, item.Metadata.Name, err) } count++ } if count > 0 { o.log.Printf("post-start auto-heal cleaned %d terminating pod(s) from unavailable nodes", count) } return count, nil } // unavailableNodeSet runs one orchestration or CLI step. // Signature: (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error). // Why: isolates Ready-condition parsing so dead-node cleanup stays targeted. func (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error) { out, err := o.kubectl(ctx, 20*time.Second, "get", "nodes", "-o", "json") if err != nil { return nil, fmt.Errorf("query nodes: %w", err) } var nodes nodeReadyList if err := json.Unmarshal([]byte(out), &nodes); err != nil { return nil, fmt.Errorf("decode nodes: %w", err) } unavailable := map[string]struct{}{} for _, item := range nodes.Items { ready := "" for _, cond := range item.Status.Conditions { if strings.EqualFold(strings.TrimSpace(cond.Type), "Ready") { ready = strings.TrimSpace(cond.Status) break } } if ready != "True" { unavailable[item.Metadata.Name] = struct{}{} } } return unavailable, nil }