ananke/internal/cluster/orchestrator_node_runtime_recovery.go

package cluster

import (
	"context"
	"fmt"
	"strings"
	"time"
)

// recoverManagedNodeRuntime runs one orchestration or CLI step.
// Signature: (o *Orchestrator) recoverManagedNodeRuntime(ctx context.Context, node string, alreadyUnschedulable bool, reason string) error.
// Why: kubelet/containerd wedges need one bounded escalation ladder: cordon,
// non-blocking k3s-agent restart, health proof, and optional controlled reboot.
func (o *Orchestrator) recoverManagedNodeRuntime(ctx context.Context, node string, alreadyUnschedulable bool, reason string) error {
	node = strings.TrimSpace(node)
	if node == "" {
		return fmt.Errorf("node is empty")
	}
	if !alreadyUnschedulable {
		if err := o.cordonNodeWithLease(ctx, node, cordonReasonKubeletProxy, reason); err != nil {
			return fmt.Errorf("cordon before kubelet restart: %w", err)
		}
	}

	o.log.Printf("warning: detected node runtime repair signal on %s; issuing non-blocking k3s-agent restart reason=%s", node, sanitizeCordonAnnotationValue(reason))
	if _, err := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentRestart, 30*time.Second); err != nil {
		if !alreadyUnschedulable {
			o.bestEffort("uncordon node after failed kubelet proxy repair", func() error {
				return o.uncordonAndClearCordonLease(ctx, node, cordonReasonKubeletProxy)
			})
		}
		return fmt.Errorf("restart k3s-agent: %w", err)
	}

	readyErr := o.waitForManagedNodeRuntimeReady(ctx, node, o.nodeRuntimeRestartWait())
	if readyErr == nil {
		return o.releaseRuntimeRecoveryCordon(ctx, node, alreadyUnschedulable)
	}

	showOut, showErr := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentShow, 20*time.Second)
	if showErr != nil {
		return fmt.Errorf("%v; k3s-agent state check failed: %w", readyErr, showErr)
	}
	if !k3sAgentStateRequiresReboot(showOut) {
		return readyErr
	}
	if !o.cfg.Startup.HostRepairAllowReboot {
		return fmt.Errorf("node runtime requires controlled reboot but host_repair_allow_reboot is false; k3s-agent state=%s", summarizeSystemctlShow(showOut))
	}

	o.log.Printf("warning: escalating node runtime recovery to controlled reboot node=%s state=%s", node, summarizeSystemctlShow(showOut))
	if _, err := o.runHostPrivilegedAction(ctx, node, hostActionHostReboot, 30*time.Second); err != nil {
		return fmt.Errorf("controlled reboot command failed: %w", err)
	}
	if err := o.waitForManagedNodeRuntimeReady(ctx, node, o.nodeRuntimeRebootWait()); err != nil {
		return fmt.Errorf("node did not recover after controlled reboot: %w", err)
	}
	return o.releaseRuntimeRecoveryCordon(ctx, node, alreadyUnschedulable)
}

// releaseRuntimeRecoveryCordon runs one orchestration or CLI step.
// Signature: (o *Orchestrator) releaseRuntimeRecoveryCordon(ctx context.Context, node string, alreadyUnschedulable bool) error.
// Why: nodes Ananke cordoned for repair should return to service after proof of
// recovery, while pre-existing cordons remain under their original owner.
func (o *Orchestrator) releaseRuntimeRecoveryCordon(ctx context.Context, node string, alreadyUnschedulable bool) error {
	if alreadyUnschedulable {
		return nil
	}
	if err := o.uncordonAndClearCordonLease(ctx, node, cordonReasonKubeletProxy); err != nil {
		return fmt.Errorf("uncordon after kubelet proxy repair: %w", err)
	}
	return nil
}

// nodeRuntimeRestartWait runs one orchestration or CLI step.
// Signature: (o *Orchestrator) nodeRuntimeRestartWait() time.Duration.
// Why: k3s-agent restart recovery needs a bounded wait even when tests build a
// partial startup config.
func (o *Orchestrator) nodeRuntimeRestartWait() time.Duration {
	seconds := o.cfg.Startup.NodeRuntimeRestartWaitSeconds
	if seconds <= 0 {
		seconds = 140
	}
	return time.Duration(seconds) * time.Second
}

// nodeRuntimeRebootWait runs one orchestration or CLI step.
// Signature: (o *Orchestrator) nodeRuntimeRebootWait() time.Duration.
// Why: controlled reboot recovery has a longer but still finite postcondition
// window for SSH, systemd, kubelet, and Kubernetes readiness.
func (o *Orchestrator) nodeRuntimeRebootWait() time.Duration {
	seconds := o.cfg.Startup.NodeRuntimeRebootWaitSeconds
	if seconds <= 0 {
		seconds = 420
	}
	return time.Duration(seconds) * time.Second
}

// waitForManagedNodeRuntimeReady runs one orchestration or CLI step.
// Signature: (o *Orchestrator) waitForManagedNodeRuntimeReady(ctx context.Context, node string, wait time.Duration) error.
// Why: runtime repair is not complete until Kubernetes Ready, kubelet proxy, and
// k3s-agent service state all agree the node is usable.
func (o *Orchestrator) waitForManagedNodeRuntimeReady(ctx context.Context, node string, wait time.Duration) error {
	if wait <= 0 {
		wait = 140 * time.Second
	}
	kubeWait := wait - 20*time.Second
	if kubeWait < 10*time.Second {
		kubeWait = 10 * time.Second
	}
	if _, err := o.kubectl(ctx, wait, "wait", "node/"+node, "--for=condition=Ready", "--timeout="+fmt.Sprintf("%.0fs", kubeWait.Seconds())); err != nil {
		return fmt.Errorf("wait Ready after k3s-agent restart: %w", err)
	}
	healthy, err := o.kubeletProxyHealthy(ctx, node)
	if !healthy {
		return fmt.Errorf("proxy still broken after k3s-agent restart: %v", err)
	}
	out, err := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentIsActive, 12*time.Second)
	if err != nil {
		return fmt.Errorf("k3s-agent active check: %w", err)
	}
	if strings.TrimSpace(out) != "active" {
		return fmt.Errorf("k3s-agent is-active returned %q", strings.TrimSpace(out))
	}
	return nil
}

// k3sAgentStateRequiresReboot runs one orchestration or CLI step.
// Signature: k3sAgentStateRequiresReboot(out string) bool.
// Why: only the stuck systemd states observed during hard runtime wedges should
// escalate from non-blocking restart to controlled reboot.
func k3sAgentStateRequiresReboot(out string) bool {
	lower := strings.ToLower(out)
	needles := []string{
		"activestate=deactivating",
		"activestate=activating",
		"substate=stop-sigterm",
		"substate=stop-sigkill",
		"substate=final-sigterm",
		"substate=final-sigkill",
		"substate=start",
		"result=timeout",
	}
	for _, needle := range needles {
		if strings.Contains(lower, needle) {
			return true
		}
	}
	return false
}

// summarizeSystemctlShow runs one orchestration or CLI step.
// Signature: summarizeSystemctlShow(out string) string.
// Why: operator-facing runtime incidents need compact systemd state summaries
// rather than full multi-line command output.
func summarizeSystemctlShow(out string) string {
	lines := lines(out)
	if len(lines) == 0 {
		return "unknown"
	}
	return sanitizeCordonAnnotationValue(joinLimited(lines, 4))
}