From 390f1da114b0cff1953b65ca8b62f860ffc0d30a Mon Sep 17 00:00:00 2001 From: codex Date: Tue, 7 Jul 2026 21:51:43 -0300 Subject: [PATCH] recovery: repair image pull credential blockers --- internal/cluster/orchestrator_autorepair.go | 10 + .../orchestrator_image_pull_credentials.go | 237 ++++++++++++++++++ ...rchestrator_image_pull_credentials_test.go | 87 +++++++ .../orchestrator_workload_convergence.go | 10 + testing/hygiene/in_tree_test_allowlist.txt | 1 + 5 files changed, 345 insertions(+) create mode 100644 internal/cluster/orchestrator_image_pull_credentials.go create mode 100644 internal/cluster/orchestrator_image_pull_credentials_test.go diff --git a/internal/cluster/orchestrator_autorepair.go b/internal/cluster/orchestrator_autorepair.go index cb0111f..6e77e69 100644 --- a/internal/cluster/orchestrator_autorepair.go +++ b/internal/cluster/orchestrator_autorepair.go @@ -118,6 +118,16 @@ func (o *Orchestrator) postStartAutoHeal(ctx context.Context) error { o.log.Printf("post-start auto-heal repaired configured service backend(s): %s", joinLimited(serviceRepairs, 8)) } + imagePullCredentialRepairs, err := o.healImagePullCredentialSync(ctx) + if err != nil { + errs = append(errs, fmt.Sprintf("image-pull credential sync repair: %v", err)) + } else if len(imagePullCredentialRepairs) > 0 { + requestReconcile = true + sort.Strings(imagePullCredentialRepairs) + o.log.Printf("post-start auto-heal refreshed image-pull credential sync deployment(s): %s", joinLimited(imagePullCredentialRepairs, 8)) + o.noteStartupAutoHeal(fmt.Sprintf("refreshed image-pull credential sync deployment(s): %s", joinLimited(imagePullCredentialRepairs, 8))) + } + repairedProxies, err := o.repairBrokenKubeletProxies(ctx) if err != nil { errs = append(errs, fmt.Sprintf("kubelet proxy auto-repair: %v", err)) diff --git a/internal/cluster/orchestrator_image_pull_credentials.go b/internal/cluster/orchestrator_image_pull_credentials.go new file mode 100644 index 0000000..1d5d1d5 --- /dev/null +++ b/internal/cluster/orchestrator_image_pull_credentials.go @@ -0,0 +1,237 @@ +package cluster + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" +) + +type imagePullCredentialDeploymentList struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + Labels map[string]string `json:"labels"` + } `json:"metadata"` + } `json:"items"` +} + +// imagePullCredentialBlockerReasons classifies image pull failures caused by +// registry auth or missing imagePullSecret material. +// Signature: (o *Orchestrator) imagePullCredentialBlockerReasons(ctx context.Context) (map[string]string, error). +// Why: deleting a pod with bad registry credentials does not refresh Vault/CSI +// secret material; the secret sync path needs a direct repair signal. +func (o *Orchestrator) imagePullCredentialBlockerReasons(ctx context.Context) (map[string]string, error) { + eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json") + if err != nil { + return nil, fmt.Errorf("query events for image-pull credential scan: %w", err) + } + reasons := map[string]string{} + if strings.TrimSpace(eventsOut) == "" { + return reasons, nil + } + var events eventList + if err := json.Unmarshal([]byte(eventsOut), &events); err != nil { + return nil, fmt.Errorf("decode events for image-pull credential scan: %w", err) + } + for _, event := range events.Items { + if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") { + continue + } + if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") { + continue + } + reason := strings.TrimSpace(event.Reason) + message := strings.TrimSpace(event.Message) + if !imagePullCredentialEvent(reason, message) { + continue + } + namespace := strings.TrimSpace(event.InvolvedObject.Namespace) + if namespace == "" { + namespace = strings.TrimSpace(event.Metadata.Namespace) + } + name := strings.TrimSpace(event.InvolvedObject.Name) + if namespace == "" || name == "" { + continue + } + reasons[namespace+"/"+name] = "ImagePullCredentialBlocker:" + imagePullCredentialFailureClass(reason, message) + } + return reasons, nil +} + +// healImagePullCredentialSync restarts namespace-local Vault sync deployments +// when pods are blocked on registry credentials. +// Signature: (o *Orchestrator) healImagePullCredentialSync(ctx context.Context) ([]string, error). +// Why: Secrets Store CSI secretObjects are materialized by mounted workloads; a +// restarted vault-sync deployment is the bounded repair that refreshes pull +// secrets such as harbor-regcred without mutating application deployments. +func (o *Orchestrator) healImagePullCredentialSync(ctx context.Context) ([]string, error) { + if o.runner.DryRun { + return nil, nil + } + blockers, err := o.imagePullCredentialBlockerReasons(ctx) + if err != nil { + return nil, err + } + if len(blockers) == 0 { + return nil, nil + } + + namespaces := map[string]struct{}{} + for key := range blockers { + namespace, _, ok := strings.Cut(key, "/") + if ok && strings.TrimSpace(namespace) != "" { + namespaces[namespace] = struct{}{} + } + } + orderedNamespaces := make([]string, 0, len(namespaces)) + for namespace := range namespaces { + orderedNamespaces = append(orderedNamespaces, namespace) + } + sort.Strings(orderedNamespaces) + + repaired := []string{} + errs := []string{} + for _, namespace := range orderedNamespaces { + namespaceRepairs, restartErr := o.restartVaultSyncDeployments(ctx, namespace) + repaired = append(repaired, namespaceRepairs...) + if restartErr != nil { + errs = append(errs, restartErr.Error()) + } + } + sort.Strings(repaired) + if len(errs) > 0 { + return repaired, errors.New(strings.Join(errs, "; ")) + } + return repaired, nil +} + +// restartVaultSyncDeployments runs one orchestration or CLI step. +// Signature: (o *Orchestrator) restartVaultSyncDeployments(ctx context.Context, namespace string) ([]string, error). +// Why: namespaces using Vault/CSI secret material have tiny sync deployments +// named or labeled vault-sync; restarting those nudges secretObject rotation. +func (o *Orchestrator) restartVaultSyncDeployments(ctx context.Context, namespace string) ([]string, error) { + out, err := o.kubectl(ctx, 20*time.Second, "-n", namespace, "get", "deployment", "-o", "json") + if err != nil { + return nil, fmt.Errorf("query deployments in %s for image-pull credential repair: %w", namespace, err) + } + var deployments imagePullCredentialDeploymentList + if err := json.Unmarshal([]byte(out), &deployments); err != nil { + return nil, fmt.Errorf("decode deployments in %s for image-pull credential repair: %w", namespace, err) + } + + repaired := []string{} + for _, deployment := range deployments.Items { + name := strings.TrimSpace(deployment.Metadata.Name) + if name == "" || !vaultSyncDeployment(name, deployment.Metadata.Labels) { + continue + } + if _, err := o.kubectl(ctx, 25*time.Second, "-n", namespace, "rollout", "restart", "deployment", name); err != nil { + return repaired, fmt.Errorf("restart %s/deployment/%s for image-pull credential repair: %w", namespace, name, err) + } + if _, err := o.kubectl(ctx, 75*time.Second, "-n", namespace, "rollout", "status", "deployment/"+name, "--timeout=60s"); err != nil { + return repaired, fmt.Errorf("wait for %s/deployment/%s after image-pull credential repair: %w", namespace, name, err) + } + repaired = append(repaired, namespace+"/deployment/"+name) + } + if len(repaired) == 0 { + return nil, fmt.Errorf("image-pull credential blocker in namespace %s but no vault-sync deployment was found", namespace) + } + return repaired, nil +} + +// imagePullCredentialEvent runs one orchestration or CLI step. +// Signature: imagePullCredentialEvent(reason string, message string) bool. +// Why: image-pull auth detection must include kubelet pull-secret events while +// staying separate from DNS and ordinary transient pull backoff. +func imagePullCredentialEvent(reason string, message string) bool { + normalizedReason := strings.ToLower(strings.TrimSpace(reason)) + if normalizedReason == "failedtoretrieveimagepullsecret" { + return true + } + if normalizedReason != "failed" && + normalizedReason != "failedpull" && + normalizedReason != "errimagepull" && + normalizedReason != "imagepullbackoff" { + return false + } + return imagePullMessageHasCredentialFailure(message) +} + +// imagePullMessageHasCredentialFailure runs one orchestration or CLI step. +// Signature: imagePullMessageHasCredentialFailure(message string) bool. +// Why: credential blockers need a tight predicate so missing images and registry +// DNS outages keep their own repair paths. +func imagePullMessageHasCredentialFailure(message string) bool { + lower := strings.ToLower(strings.TrimSpace(message)) + if lower == "" || imagePullMessageHasDNSFailure(message) { + return false + } + return strings.Contains(lower, "failed to retrieve image pull secret") || + strings.Contains(lower, "failedtoretrieveimagepullsecret") || + strings.Contains(lower, "unable to retrieve some image pull secrets") || + (strings.Contains(lower, "image pull secret") && strings.Contains(lower, "not found")) || + (strings.Contains(lower, "pull secret") && strings.Contains(lower, "not found")) || + (strings.Contains(lower, "secret ") && strings.Contains(lower, " not found") && strings.Contains(lower, "pull")) || + strings.Contains(lower, "no basic auth credentials") || + strings.Contains(lower, "unauthorized") || + strings.Contains(lower, "authentication required") || + strings.Contains(lower, "authorization failed") || + strings.Contains(lower, "failed to authorize") || + strings.Contains(lower, "invalid username/password") || + strings.Contains(lower, "401 unauthorized") || + strings.Contains(lower, "403 forbidden") || + strings.Contains(lower, "pull access denied") || + strings.Contains(lower, "requested access to the resource is denied") || + strings.Contains(lower, "failed to fetch anonymous token") +} + +// imagePullCredentialFailureClass runs one orchestration or CLI step. +// Signature: imagePullCredentialFailureClass(reason string, message string) string. +// Why: compact blocker classes keep logs and status useful without copying long +// kubelet event messages. +func imagePullCredentialFailureClass(reason string, message string) string { + lowerReason := strings.ToLower(strings.TrimSpace(reason)) + lower := strings.ToLower(strings.TrimSpace(message)) + switch { + case lowerReason == "failedtoretrieveimagepullsecret", + strings.Contains(lower, "failed to retrieve image pull secret"), + strings.Contains(lower, "unable to retrieve some image pull secrets"), + strings.Contains(lower, "image pull secret") && strings.Contains(lower, "not found"), + strings.Contains(lower, "pull secret") && strings.Contains(lower, "not found"), + strings.Contains(lower, "secret ") && strings.Contains(lower, " not found") && strings.Contains(lower, "pull"): + return "missing-pull-secret" + case strings.Contains(lower, "no basic auth credentials"): + return "no-basic-auth" + case strings.Contains(lower, "invalid username/password"): + return "invalid-credentials" + case strings.Contains(lower, "unauthorized") || strings.Contains(lower, "authentication required"): + return "unauthorized" + case strings.Contains(lower, "forbidden") || strings.Contains(lower, "authorization failed") || strings.Contains(lower, "failed to authorize") || strings.Contains(lower, "failed to fetch anonymous token"): + return "authorization-failed" + case strings.Contains(lower, "pull access denied") || strings.Contains(lower, "requested access to the resource is denied"): + return "pull-access-denied" + default: + return "registry-credential-error" + } +} + +// vaultSyncDeployment runs one orchestration or CLI step. +// Signature: vaultSyncDeployment(name string, labels map[string]string) bool. +// Why: different apps may prefix their sync deployment names, but they consistently +// identify the narrow Vault sync helper with a vault-sync name or label. +func vaultSyncDeployment(name string, labels map[string]string) bool { + if strings.Contains(strings.ToLower(strings.TrimSpace(name)), "vault-sync") { + return true + } + for key, value := range labels { + if strings.Contains(strings.ToLower(strings.TrimSpace(key)), "vault-sync") || + strings.Contains(strings.ToLower(strings.TrimSpace(value)), "vault-sync") { + return true + } + } + return false +} diff --git a/internal/cluster/orchestrator_image_pull_credentials_test.go b/internal/cluster/orchestrator_image_pull_credentials_test.go new file mode 100644 index 0000000..f88b045 --- /dev/null +++ b/internal/cluster/orchestrator_image_pull_credentials_test.go @@ -0,0 +1,87 @@ +package cluster + +import ( + "context" + "strings" + "testing" + + "scm.bstein.dev/bstein/ananke/internal/config" +) + +// TestImagePullCredentialBlockerReasonsClassifiesHarborAuth runs one +// orchestration or CLI step. +// Signature: TestImagePullCredentialBlockerReasonsClassifiesHarborAuth(t *testing.T). +// Why: Harbor auth failures should drive secret-sync repair instead of pod +// recycling or DNS remediation. +func TestImagePullCredentialBlockerReasonsClassifiesHarborAuth(t *testing.T) { + events := `{"items":[` + + `{"involvedObject":{"kind":"Pod","namespace":"veles","name":"veles-backend"},"type":"Warning","reason":"Failed","message":"Failed to pull image \"registry.bstein.dev/veles/veles-backend:0.5.25\": no basic auth credentials"},` + + `{"metadata":{"namespace":"veles"},"involvedObject":{"kind":"Pod","name":"veles-frontend"},"type":"Warning","reason":"FailedToRetrieveImagePullSecret","message":"Unable to retrieve some image pull secrets (harbor-regcred); attempting to pull the image may not succeed."},` + + `{"involvedObject":{"kind":"Pod","namespace":"logging","name":"oauth2"},"type":"Warning","reason":"Failed","message":"Failed to pull image: lookup registry-1.docker.io: Try again"}` + + `]}` + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events}, + }) + + reasons, err := orch.imagePullCredentialBlockerReasons(context.Background()) + if err != nil { + t.Fatalf("imagePullCredentialBlockerReasons failed: %v", err) + } + if got := reasons["veles/veles-backend"]; got != "ImagePullCredentialBlocker:no-basic-auth" { + t.Fatalf("unexpected backend credential reason %q", got) + } + if got := reasons["veles/veles-frontend"]; got != "ImagePullCredentialBlocker:missing-pull-secret" { + t.Fatalf("unexpected frontend credential reason %q", got) + } + if _, ok := reasons["logging/oauth2"]; ok { + t.Fatalf("DNS image-pull blocker must not be classified as credential failure") + } +} + +// TestHealImagePullCredentialSyncRestartsVaultSyncDeployment runs one +// orchestration or CLI step. +// Signature: TestHealImagePullCredentialSyncRestartsVaultSyncDeployment(t *testing.T). +// Why: a namespace with registry credential blockers and a Vault CSI sync helper +// should get that sync helper rolled instead of application pods deleted. +func TestHealImagePullCredentialSyncRestartsVaultSyncDeployment(t *testing.T) { + events := `{"items":[{"involvedObject":{"kind":"Pod","namespace":"veles","name":"veles-backend"},"type":"Warning","reason":"Failed","message":"Failed to pull image \"registry.bstein.dev/veles/veles-backend:0.5.25\": no basic auth credentials"}]}` + deployments := `{"items":[` + + `{"metadata":{"name":"veles-backend","labels":{"app.kubernetes.io/name":"veles-backend"}}},` + + `{"metadata":{"name":"veles-vault-sync","labels":{"app.kubernetes.io/component":"vault-sync"}}}` + + `]}` + restarted := false + rolledOut := false + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events}, + {match: matchContains("kubectl", "-n", "veles", "get", "deployment", "-o", "json"), out: deployments}, + { + match: func(name string, args []string) bool { + if !matchContains("kubectl", "-n", "veles", "rollout", "restart", "deployment", "veles-vault-sync")(name, args) { + return false + } + restarted = true + return true + }, + }, + { + match: func(name string, args []string) bool { + if !matchContains("kubectl", "-n", "veles", "rollout", "status", "deployment/veles-vault-sync", "--timeout=60s")(name, args) { + return false + } + rolledOut = true + return true + }, + }, + }) + + repaired, err := orch.healImagePullCredentialSync(context.Background()) + if err != nil { + t.Fatalf("healImagePullCredentialSync failed: %v", err) + } + if strings.Join(repaired, ",") != "veles/deployment/veles-vault-sync" { + t.Fatalf("unexpected image-pull credential repair list: %#v", repaired) + } + if !restarted || !rolledOut { + t.Fatalf("expected vault-sync deployment restart and rollout wait, restarted=%v rolledOut=%v", restarted, rolledOut) + } +} diff --git a/internal/cluster/orchestrator_workload_convergence.go b/internal/cluster/orchestrator_workload_convergence.go index f450226..dfe028f 100644 --- a/internal/cluster/orchestrator_workload_convergence.go +++ b/internal/cluster/orchestrator_workload_convergence.go @@ -200,6 +200,12 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error { } else { imagePullDNSReasons = reasons } + imagePullCredentialReasons := map[string]string{} + if reasons, scanErr := o.imagePullCredentialBlockerReasons(ctx); scanErr != nil { + o.log.Printf("warning: image-pull credential blocker scan failed: %v", scanErr) + } else { + imagePullCredentialReasons = reasons + } containerRuntimeWedgeReasons := map[string]string{} if reasons, scanErr := o.containerRuntimeWedgePodReasons(ctx, list, grace); scanErr != nil { o.log.Printf("warning: container runtime wedge scan failed: %v", scanErr) @@ -235,6 +241,10 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error { o.log.Printf("warning: not recycling pod %s/%s because image pull is blocked by DNS/registry lookup: %s", ns, name, imagePullDNSReasons[ns+"/"+name]) continue } + if (reason == "ImagePullBackOff" || reason == "ErrImagePull") && imagePullCredentialReasons[ns+"/"+name] != "" { + o.log.Printf("warning: not recycling pod %s/%s because image pull is blocked by registry credentials: %s", ns, name, imagePullCredentialReasons[ns+"/"+name]) + continue + } if reason == "" { reason = stuckVaultInitReason(pod, grace) } diff --git a/testing/hygiene/in_tree_test_allowlist.txt b/testing/hygiene/in_tree_test_allowlist.txt index ffaf5be..924b6c4 100644 --- a/testing/hygiene/in_tree_test_allowlist.txt +++ b/testing/hygiene/in_tree_test_allowlist.txt @@ -22,6 +22,7 @@ internal/cluster/orchestrator_critical_endpoint_additional_test.go internal/cluster/orchestrator_cordon_lease_test.go internal/cluster/orchestrator_test.go internal/cluster/orchestrator_hardening_test.go +internal/cluster/orchestrator_image_pull_credentials_test.go internal/cluster/orchestrator_unit_additional_test.go internal/cluster/orchestrator_workload_recovery_test.go internal/cluster/orchestrator_vault_test.go