98 lines
2.9 KiB
Go
98 lines
2.9 KiB
Go
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// cleanupTerminatingPodsOnUnavailableNodes runs one orchestration or CLI step.
|
|
// Signature: (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error).
|
|
// Why: dead nodes can strand terminating pods indefinitely, so the daemon should
|
|
// clear only that narrow failure class instead of leaving garbage behind forever.
|
|
func (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error) {
|
|
if o.runner.DryRun {
|
|
return 0, nil
|
|
}
|
|
|
|
unavailable, err := o.unavailableNodeSet(ctx)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if len(unavailable) == 0 {
|
|
return 0, nil
|
|
}
|
|
|
|
out, err := o.kubectl(ctx, 30*time.Second, "get", "pods", "-A", "-o", "json")
|
|
if err != nil {
|
|
return 0, fmt.Errorf("query pods: %w", err)
|
|
}
|
|
var pods podDeleteList
|
|
if err := json.Unmarshal([]byte(out), &pods); err != nil {
|
|
return 0, fmt.Errorf("decode pods: %w", err)
|
|
}
|
|
|
|
grace := time.Duration(o.cfg.Startup.DeadNodeCleanupGraceSeconds) * time.Second
|
|
now := time.Now()
|
|
count := 0
|
|
for _, item := range pods.Items {
|
|
if item.Metadata.DeletionTimestamp == nil || item.Spec.NodeName == "" {
|
|
continue
|
|
}
|
|
if _, badNode := unavailable[item.Spec.NodeName]; !badNode {
|
|
continue
|
|
}
|
|
if now.Sub(*item.Metadata.DeletionTimestamp) < grace {
|
|
continue
|
|
}
|
|
o.log.Printf("warning: force deleting terminating pod %s/%s on unavailable node %s", item.Metadata.Namespace, item.Metadata.Name, item.Spec.NodeName)
|
|
if _, err := o.kubectl(
|
|
ctx,
|
|
20*time.Second,
|
|
"-n", item.Metadata.Namespace,
|
|
"delete", "pod", item.Metadata.Name,
|
|
"--grace-period=0",
|
|
"--force",
|
|
"--wait=false",
|
|
); err != nil && !isNotFoundErr(err) {
|
|
return count, fmt.Errorf("delete pod %s/%s: %w", item.Metadata.Namespace, item.Metadata.Name, err)
|
|
}
|
|
count++
|
|
}
|
|
if count > 0 {
|
|
o.log.Printf("post-start auto-heal cleaned %d terminating pod(s) from unavailable nodes", count)
|
|
}
|
|
return count, nil
|
|
}
|
|
|
|
// unavailableNodeSet runs one orchestration or CLI step.
|
|
// Signature: (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error).
|
|
// Why: isolates Ready-condition parsing so dead-node cleanup stays targeted.
|
|
func (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error) {
|
|
out, err := o.kubectl(ctx, 20*time.Second, "get", "nodes", "-o", "json")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("query nodes: %w", err)
|
|
}
|
|
var nodes nodeReadyList
|
|
if err := json.Unmarshal([]byte(out), &nodes); err != nil {
|
|
return nil, fmt.Errorf("decode nodes: %w", err)
|
|
}
|
|
|
|
unavailable := map[string]struct{}{}
|
|
for _, item := range nodes.Items {
|
|
ready := ""
|
|
for _, cond := range item.Status.Conditions {
|
|
if strings.EqualFold(strings.TrimSpace(cond.Type), "Ready") {
|
|
ready = strings.TrimSpace(cond.Status)
|
|
break
|
|
}
|
|
}
|
|
if ready != "True" {
|
|
unavailable[item.Metadata.Name] = struct{}{}
|
|
}
|
|
}
|
|
return unavailable, nil
|
|
}
|