From cecd82cff31a912eb912e2470a50602897b468d6 Mon Sep 17 00:00:00 2001
From: codex <codex@bstein.dev>
Date: Tue, 7 Jul 2026 21:57:13 -0300
Subject: [PATCH] recovery: retry transient kubelet proxy probes

---
 internal/cluster/orchestrator_autorepair.go   | 45 ++++++++++++++++++-
 .../orchestrator_autorepair_proxy_test.go     | 34 ++++++++++++++
 2 files changed, 78 insertions(+), 1 deletion(-)

diff --git a/internal/cluster/orchestrator_autorepair.go b/internal/cluster/orchestrator_autorepair.go
index 6e77e69..1920ce2 100644
--- a/internal/cluster/orchestrator_autorepair.go
+++ b/internal/cluster/orchestrator_autorepair.go
@@ -345,7 +345,24 @@ func (o *Orchestrator) readyNodeCandidates(ctx context.Context) ([]readyNodeCand
 // Why: the apiserver node proxy is the path Jenkins uses for pod exec; checking
 // it catches Ready-but-unusable nodes before agents start failing websockets.
 func (o *Orchestrator) kubeletProxyHealthy(ctx context.Context, node string) (bool, error) {
-	out, err := o.kubectl(ctx, 10*time.Second, "get", "--raw", fmt.Sprintf("/api/v1/nodes/%s/proxy/healthz", node))
+	var lastErr error
+	for attempt := 0; attempt < 2; attempt++ {
+		healthy, err := o.kubeletProxyHealthyOnce(ctx, node)
+		if err == nil || !isTransientKubeletProxyCheckErr(err) {
+			return healthy, err
+		}
+		lastErr = err
+		o.log.Printf("warning: transient kubelet proxy health check failure for %s: %v; retrying", node, err)
+	}
+	return false, lastErr
+}
+
+// kubeletProxyHealthyOnce runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) kubeletProxyHealthyOnce(ctx context.Context, node string) (bool, error).
+// Why: transient apiserver/kubectl cancellations should be retried by the caller
+// without losing the exact kubelet proxy error when the node is really broken.
+func (o *Orchestrator) kubeletProxyHealthyOnce(ctx context.Context, node string) (bool, error) {
+	out, err := o.kubectl(ctx, 25*time.Second, "get", "--raw", fmt.Sprintf("/api/v1/nodes/%s/proxy/healthz", node), "--request-timeout=20s")
 	if err != nil {
 		if strings.TrimSpace(out) != "" {
 			return false, fmt.Errorf("%w: %s", err, strings.TrimSpace(out))
@@ -355,6 +372,32 @@ func (o *Orchestrator) kubeletProxyHealthy(ctx context.Context, node string) (bo
 	return true, nil
 }
 
+// isTransientKubeletProxyCheckErr runs one orchestration or CLI step.
+// Signature: isTransientKubeletProxyCheckErr(err error) bool.
+// Why: a killed or timed-out kubectl health probe is not proof the node proxy is
+// broken; retry before reporting or repairing a Ready node.
+func isTransientKubeletProxyCheckErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := strings.ToLower(err.Error())
+	transient := []string{
+		"signal: killed",
+		"context deadline exceeded",
+		"deadline exceeded",
+		"i/o timeout",
+		"request canceled",
+		"client.timeout",
+		"timeout awaiting",
+	}
+	for _, needle := range transient {
+		if strings.Contains(msg, needle) {
+			return true
+		}
+	}
+	return false
+}
+
 // isRepairableKubeletProxyErr runs one orchestration or CLI step.
 // Signature: isRepairableKubeletProxyErr(err error) bool.
 // Why: keep this repair narrow so Ananke restarts k3s-agent for the known
diff --git a/internal/cluster/orchestrator_autorepair_proxy_test.go b/internal/cluster/orchestrator_autorepair_proxy_test.go
index 06973e7..dbf5aa5 100644
--- a/internal/cluster/orchestrator_autorepair_proxy_test.go
+++ b/internal/cluster/orchestrator_autorepair_proxy_test.go
@@ -329,6 +329,36 @@ func TestKubeletProxyHealthAndRepairableErrorHelpers(t *testing.T) {
 		}
 	})
 
+	t.Run("transient killed health check is retried", func(t *testing.T) {
+		checks := 0
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: func(name string, args []string) bool {
+					if checks > 0 {
+						return false
+					}
+					if !matchContains("kubectl", "get --raw /api/v1/nodes/titan-07/proxy/healthz")(name, args) {
+						return false
+					}
+					checks++
+					return true
+				},
+				err: errors.New("signal: killed"),
+			},
+			{
+				match: matchContains("kubectl", "get --raw /api/v1/nodes/titan-07/proxy/healthz"),
+				out:   "ok",
+			},
+		})
+		healthy, err := orch.kubeletProxyHealthy(context.Background(), "titan-07")
+		if !healthy || err != nil {
+			t.Fatalf("expected retry to pass, healthy=%v err=%v", healthy, err)
+		}
+		if checks != 1 {
+			t.Fatalf("expected one transient failure before retry, got %d", checks)
+		}
+	})
+
 	for _, tc := range []struct {
 		name string
 		err  error
@@ -344,4 +374,8 @@ func TestKubeletProxyHealthAndRepairableErrorHelpers(t *testing.T) {
 			}
 		})
 	}
+
+	if !isTransientKubeletProxyCheckErr(errors.New("signal: killed")) {
+		t.Fatalf("expected signal killed to be transient kubelet proxy check error")
+	}
 }