package cluster import ( "context" "fmt" "strings" "time" ) // recoverManagedNodeRuntime runs one orchestration or CLI step. // Signature: (o *Orchestrator) recoverManagedNodeRuntime(ctx context.Context, node string, alreadyUnschedulable bool, reason string) error. // Why: kubelet/containerd wedges need one bounded escalation ladder: cordon, // non-blocking k3s-agent restart, health proof, and optional controlled reboot. func (o *Orchestrator) recoverManagedNodeRuntime(ctx context.Context, node string, alreadyUnschedulable bool, reason string) error { node = strings.TrimSpace(node) if node == "" { return fmt.Errorf("node is empty") } if !alreadyUnschedulable { if err := o.cordonNodeWithLease(ctx, node, cordonReasonKubeletProxy, reason); err != nil { return fmt.Errorf("cordon before kubelet restart: %w", err) } } o.log.Printf("warning: detected node runtime repair signal on %s; issuing non-blocking k3s-agent restart reason=%s", node, sanitizeCordonAnnotationValue(reason)) if _, err := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentRestart, 30*time.Second); err != nil { if !alreadyUnschedulable { o.bestEffort("uncordon node after failed kubelet proxy repair", func() error { return o.uncordonAndClearCordonLease(ctx, node, cordonReasonKubeletProxy) }) } return fmt.Errorf("restart k3s-agent: %w", err) } readyErr := o.waitForManagedNodeRuntimeReady(ctx, node, o.nodeRuntimeRestartWait()) if readyErr == nil { return o.releaseRuntimeRecoveryCordon(ctx, node, alreadyUnschedulable) } showOut, showErr := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentShow, 20*time.Second) if showErr != nil { return fmt.Errorf("%v; k3s-agent state check failed: %w", readyErr, showErr) } if !k3sAgentStateRequiresReboot(showOut) { return readyErr } if !o.cfg.Startup.HostRepairAllowReboot { return fmt.Errorf("node runtime requires controlled reboot but host_repair_allow_reboot is false; k3s-agent state=%s", summarizeSystemctlShow(showOut)) } o.log.Printf("warning: escalating node runtime recovery to controlled reboot node=%s state=%s", node, summarizeSystemctlShow(showOut)) if _, err := o.runHostPrivilegedAction(ctx, node, hostActionHostReboot, 30*time.Second); err != nil { return fmt.Errorf("controlled reboot command failed: %w", err) } if err := o.waitForManagedNodeRuntimeReady(ctx, node, o.nodeRuntimeRebootWait()); err != nil { return fmt.Errorf("node did not recover after controlled reboot: %w", err) } return o.releaseRuntimeRecoveryCordon(ctx, node, alreadyUnschedulable) } // releaseRuntimeRecoveryCordon runs one orchestration or CLI step. // Signature: (o *Orchestrator) releaseRuntimeRecoveryCordon(ctx context.Context, node string, alreadyUnschedulable bool) error. // Why: nodes Ananke cordoned for repair should return to service after proof of // recovery, while pre-existing cordons remain under their original owner. func (o *Orchestrator) releaseRuntimeRecoveryCordon(ctx context.Context, node string, alreadyUnschedulable bool) error { if alreadyUnschedulable { return nil } if err := o.uncordonAndClearCordonLease(ctx, node, cordonReasonKubeletProxy); err != nil { return fmt.Errorf("uncordon after kubelet proxy repair: %w", err) } return nil } // nodeRuntimeRestartWait runs one orchestration or CLI step. // Signature: (o *Orchestrator) nodeRuntimeRestartWait() time.Duration. // Why: k3s-agent restart recovery needs a bounded wait even when tests build a // partial startup config. func (o *Orchestrator) nodeRuntimeRestartWait() time.Duration { seconds := o.cfg.Startup.NodeRuntimeRestartWaitSeconds if seconds <= 0 { seconds = 140 } return time.Duration(seconds) * time.Second } // nodeRuntimeRebootWait runs one orchestration or CLI step. // Signature: (o *Orchestrator) nodeRuntimeRebootWait() time.Duration. // Why: controlled reboot recovery has a longer but still finite postcondition // window for SSH, systemd, kubelet, and Kubernetes readiness. func (o *Orchestrator) nodeRuntimeRebootWait() time.Duration { seconds := o.cfg.Startup.NodeRuntimeRebootWaitSeconds if seconds <= 0 { seconds = 420 } return time.Duration(seconds) * time.Second } // waitForManagedNodeRuntimeReady runs one orchestration or CLI step. // Signature: (o *Orchestrator) waitForManagedNodeRuntimeReady(ctx context.Context, node string, wait time.Duration) error. // Why: runtime repair is not complete until Kubernetes Ready, kubelet proxy, and // k3s-agent service state all agree the node is usable. func (o *Orchestrator) waitForManagedNodeRuntimeReady(ctx context.Context, node string, wait time.Duration) error { if wait <= 0 { wait = 140 * time.Second } kubeWait := wait - 20*time.Second if kubeWait < 10*time.Second { kubeWait = 10 * time.Second } if _, err := o.kubectl(ctx, wait, "wait", "node/"+node, "--for=condition=Ready", "--timeout="+fmt.Sprintf("%.0fs", kubeWait.Seconds())); err != nil { return fmt.Errorf("wait Ready after k3s-agent restart: %w", err) } healthy, err := o.kubeletProxyHealthy(ctx, node) if !healthy { return fmt.Errorf("proxy still broken after k3s-agent restart: %v", err) } out, err := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentIsActive, 12*time.Second) if err != nil { return fmt.Errorf("k3s-agent active check: %w", err) } if strings.TrimSpace(out) != "active" { return fmt.Errorf("k3s-agent is-active returned %q", strings.TrimSpace(out)) } return nil } // k3sAgentStateRequiresReboot runs one orchestration or CLI step. // Signature: k3sAgentStateRequiresReboot(out string) bool. // Why: only the stuck systemd states observed during hard runtime wedges should // escalate from non-blocking restart to controlled reboot. func k3sAgentStateRequiresReboot(out string) bool { lower := strings.ToLower(out) needles := []string{ "activestate=deactivating", "activestate=activating", "substate=stop-sigterm", "substate=stop-sigkill", "substate=final-sigterm", "substate=final-sigkill", "substate=start", "result=timeout", } for _, needle := range needles { if strings.Contains(lower, needle) { return true } } return false } // summarizeSystemctlShow runs one orchestration or CLI step. // Signature: summarizeSystemctlShow(out string) string. // Why: operator-facing runtime incidents need compact systemd state summaries // rather than full multi-line command output. func summarizeSystemctlShow(out string) string { lines := lines(out) if len(lines) == 0 { return "unknown" } return sanitizeCordonAnnotationValue(joinLimited(lines, 4)) }