recovery: repair image pull credential blockers
This commit is contained in:
codex
parent
72ce4a942c
commit
390f1da114
@ -118,6 +118,16 @@ func (o *Orchestrator) postStartAutoHeal(ctx context.Context) error {
|
||||
o.log.Printf("post-start auto-heal repaired configured service backend(s): %s", joinLimited(serviceRepairs, 8))
|
||||
}
|
||||
|
||||
imagePullCredentialRepairs, err := o.healImagePullCredentialSync(ctx)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Sprintf("image-pull credential sync repair: %v", err))
|
||||
} else if len(imagePullCredentialRepairs) > 0 {
|
||||
requestReconcile = true
|
||||
sort.Strings(imagePullCredentialRepairs)
|
||||
o.log.Printf("post-start auto-heal refreshed image-pull credential sync deployment(s): %s", joinLimited(imagePullCredentialRepairs, 8))
|
||||
o.noteStartupAutoHeal(fmt.Sprintf("refreshed image-pull credential sync deployment(s): %s", joinLimited(imagePullCredentialRepairs, 8)))
|
||||
}
|
||||
|
||||
repairedProxies, err := o.repairBrokenKubeletProxies(ctx)
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Sprintf("kubelet proxy auto-repair: %v", err))
|
||||
|
||||
237
internal/cluster/orchestrator_image_pull_credentials.go
Normal file
237
internal/cluster/orchestrator_image_pull_credentials.go
Normal file
@ -0,0 +1,237 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type imagePullCredentialDeploymentList struct {
|
||||
Items []struct {
|
||||
Metadata struct {
|
||||
Name string `json:"name"`
|
||||
Labels map[string]string `json:"labels"`
|
||||
} `json:"metadata"`
|
||||
} `json:"items"`
|
||||
}
|
||||
|
||||
// imagePullCredentialBlockerReasons classifies image pull failures caused by
|
||||
// registry auth or missing imagePullSecret material.
|
||||
// Signature: (o *Orchestrator) imagePullCredentialBlockerReasons(ctx context.Context) (map[string]string, error).
|
||||
// Why: deleting a pod with bad registry credentials does not refresh Vault/CSI
|
||||
// secret material; the secret sync path needs a direct repair signal.
|
||||
func (o *Orchestrator) imagePullCredentialBlockerReasons(ctx context.Context) (map[string]string, error) {
|
||||
eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query events for image-pull credential scan: %w", err)
|
||||
}
|
||||
reasons := map[string]string{}
|
||||
if strings.TrimSpace(eventsOut) == "" {
|
||||
return reasons, nil
|
||||
}
|
||||
var events eventList
|
||||
if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
|
||||
return nil, fmt.Errorf("decode events for image-pull credential scan: %w", err)
|
||||
}
|
||||
for _, event := range events.Items {
|
||||
if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
|
||||
continue
|
||||
}
|
||||
if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
|
||||
continue
|
||||
}
|
||||
reason := strings.TrimSpace(event.Reason)
|
||||
message := strings.TrimSpace(event.Message)
|
||||
if !imagePullCredentialEvent(reason, message) {
|
||||
continue
|
||||
}
|
||||
namespace := strings.TrimSpace(event.InvolvedObject.Namespace)
|
||||
if namespace == "" {
|
||||
namespace = strings.TrimSpace(event.Metadata.Namespace)
|
||||
}
|
||||
name := strings.TrimSpace(event.InvolvedObject.Name)
|
||||
if namespace == "" || name == "" {
|
||||
continue
|
||||
}
|
||||
reasons[namespace+"/"+name] = "ImagePullCredentialBlocker:" + imagePullCredentialFailureClass(reason, message)
|
||||
}
|
||||
return reasons, nil
|
||||
}
|
||||
|
||||
// healImagePullCredentialSync restarts namespace-local Vault sync deployments
|
||||
// when pods are blocked on registry credentials.
|
||||
// Signature: (o *Orchestrator) healImagePullCredentialSync(ctx context.Context) ([]string, error).
|
||||
// Why: Secrets Store CSI secretObjects are materialized by mounted workloads; a
|
||||
// restarted vault-sync deployment is the bounded repair that refreshes pull
|
||||
// secrets such as harbor-regcred without mutating application deployments.
|
||||
func (o *Orchestrator) healImagePullCredentialSync(ctx context.Context) ([]string, error) {
|
||||
if o.runner.DryRun {
|
||||
return nil, nil
|
||||
}
|
||||
blockers, err := o.imagePullCredentialBlockerReasons(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(blockers) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
namespaces := map[string]struct{}{}
|
||||
for key := range blockers {
|
||||
namespace, _, ok := strings.Cut(key, "/")
|
||||
if ok && strings.TrimSpace(namespace) != "" {
|
||||
namespaces[namespace] = struct{}{}
|
||||
}
|
||||
}
|
||||
orderedNamespaces := make([]string, 0, len(namespaces))
|
||||
for namespace := range namespaces {
|
||||
orderedNamespaces = append(orderedNamespaces, namespace)
|
||||
}
|
||||
sort.Strings(orderedNamespaces)
|
||||
|
||||
repaired := []string{}
|
||||
errs := []string{}
|
||||
for _, namespace := range orderedNamespaces {
|
||||
namespaceRepairs, restartErr := o.restartVaultSyncDeployments(ctx, namespace)
|
||||
repaired = append(repaired, namespaceRepairs...)
|
||||
if restartErr != nil {
|
||||
errs = append(errs, restartErr.Error())
|
||||
}
|
||||
}
|
||||
sort.Strings(repaired)
|
||||
if len(errs) > 0 {
|
||||
return repaired, errors.New(strings.Join(errs, "; "))
|
||||
}
|
||||
return repaired, nil
|
||||
}
|
||||
|
||||
// restartVaultSyncDeployments runs one orchestration or CLI step.
|
||||
// Signature: (o *Orchestrator) restartVaultSyncDeployments(ctx context.Context, namespace string) ([]string, error).
|
||||
// Why: namespaces using Vault/CSI secret material have tiny sync deployments
|
||||
// named or labeled vault-sync; restarting those nudges secretObject rotation.
|
||||
func (o *Orchestrator) restartVaultSyncDeployments(ctx context.Context, namespace string) ([]string, error) {
|
||||
out, err := o.kubectl(ctx, 20*time.Second, "-n", namespace, "get", "deployment", "-o", "json")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query deployments in %s for image-pull credential repair: %w", namespace, err)
|
||||
}
|
||||
var deployments imagePullCredentialDeploymentList
|
||||
if err := json.Unmarshal([]byte(out), &deployments); err != nil {
|
||||
return nil, fmt.Errorf("decode deployments in %s for image-pull credential repair: %w", namespace, err)
|
||||
}
|
||||
|
||||
repaired := []string{}
|
||||
for _, deployment := range deployments.Items {
|
||||
name := strings.TrimSpace(deployment.Metadata.Name)
|
||||
if name == "" || !vaultSyncDeployment(name, deployment.Metadata.Labels) {
|
||||
continue
|
||||
}
|
||||
if _, err := o.kubectl(ctx, 25*time.Second, "-n", namespace, "rollout", "restart", "deployment", name); err != nil {
|
||||
return repaired, fmt.Errorf("restart %s/deployment/%s for image-pull credential repair: %w", namespace, name, err)
|
||||
}
|
||||
if _, err := o.kubectl(ctx, 75*time.Second, "-n", namespace, "rollout", "status", "deployment/"+name, "--timeout=60s"); err != nil {
|
||||
return repaired, fmt.Errorf("wait for %s/deployment/%s after image-pull credential repair: %w", namespace, name, err)
|
||||
}
|
||||
repaired = append(repaired, namespace+"/deployment/"+name)
|
||||
}
|
||||
if len(repaired) == 0 {
|
||||
return nil, fmt.Errorf("image-pull credential blocker in namespace %s but no vault-sync deployment was found", namespace)
|
||||
}
|
||||
return repaired, nil
|
||||
}
|
||||
|
||||
// imagePullCredentialEvent runs one orchestration or CLI step.
|
||||
// Signature: imagePullCredentialEvent(reason string, message string) bool.
|
||||
// Why: image-pull auth detection must include kubelet pull-secret events while
|
||||
// staying separate from DNS and ordinary transient pull backoff.
|
||||
func imagePullCredentialEvent(reason string, message string) bool {
|
||||
normalizedReason := strings.ToLower(strings.TrimSpace(reason))
|
||||
if normalizedReason == "failedtoretrieveimagepullsecret" {
|
||||
return true
|
||||
}
|
||||
if normalizedReason != "failed" &&
|
||||
normalizedReason != "failedpull" &&
|
||||
normalizedReason != "errimagepull" &&
|
||||
normalizedReason != "imagepullbackoff" {
|
||||
return false
|
||||
}
|
||||
return imagePullMessageHasCredentialFailure(message)
|
||||
}
|
||||
|
||||
// imagePullMessageHasCredentialFailure runs one orchestration or CLI step.
|
||||
// Signature: imagePullMessageHasCredentialFailure(message string) bool.
|
||||
// Why: credential blockers need a tight predicate so missing images and registry
|
||||
// DNS outages keep their own repair paths.
|
||||
func imagePullMessageHasCredentialFailure(message string) bool {
|
||||
lower := strings.ToLower(strings.TrimSpace(message))
|
||||
if lower == "" || imagePullMessageHasDNSFailure(message) {
|
||||
return false
|
||||
}
|
||||
return strings.Contains(lower, "failed to retrieve image pull secret") ||
|
||||
strings.Contains(lower, "failedtoretrieveimagepullsecret") ||
|
||||
strings.Contains(lower, "unable to retrieve some image pull secrets") ||
|
||||
(strings.Contains(lower, "image pull secret") && strings.Contains(lower, "not found")) ||
|
||||
(strings.Contains(lower, "pull secret") && strings.Contains(lower, "not found")) ||
|
||||
(strings.Contains(lower, "secret ") && strings.Contains(lower, " not found") && strings.Contains(lower, "pull")) ||
|
||||
strings.Contains(lower, "no basic auth credentials") ||
|
||||
strings.Contains(lower, "unauthorized") ||
|
||||
strings.Contains(lower, "authentication required") ||
|
||||
strings.Contains(lower, "authorization failed") ||
|
||||
strings.Contains(lower, "failed to authorize") ||
|
||||
strings.Contains(lower, "invalid username/password") ||
|
||||
strings.Contains(lower, "401 unauthorized") ||
|
||||
strings.Contains(lower, "403 forbidden") ||
|
||||
strings.Contains(lower, "pull access denied") ||
|
||||
strings.Contains(lower, "requested access to the resource is denied") ||
|
||||
strings.Contains(lower, "failed to fetch anonymous token")
|
||||
}
|
||||
|
||||
// imagePullCredentialFailureClass runs one orchestration or CLI step.
|
||||
// Signature: imagePullCredentialFailureClass(reason string, message string) string.
|
||||
// Why: compact blocker classes keep logs and status useful without copying long
|
||||
// kubelet event messages.
|
||||
func imagePullCredentialFailureClass(reason string, message string) string {
|
||||
lowerReason := strings.ToLower(strings.TrimSpace(reason))
|
||||
lower := strings.ToLower(strings.TrimSpace(message))
|
||||
switch {
|
||||
case lowerReason == "failedtoretrieveimagepullsecret",
|
||||
strings.Contains(lower, "failed to retrieve image pull secret"),
|
||||
strings.Contains(lower, "unable to retrieve some image pull secrets"),
|
||||
strings.Contains(lower, "image pull secret") && strings.Contains(lower, "not found"),
|
||||
strings.Contains(lower, "pull secret") && strings.Contains(lower, "not found"),
|
||||
strings.Contains(lower, "secret ") && strings.Contains(lower, " not found") && strings.Contains(lower, "pull"):
|
||||
return "missing-pull-secret"
|
||||
case strings.Contains(lower, "no basic auth credentials"):
|
||||
return "no-basic-auth"
|
||||
case strings.Contains(lower, "invalid username/password"):
|
||||
return "invalid-credentials"
|
||||
case strings.Contains(lower, "unauthorized") || strings.Contains(lower, "authentication required"):
|
||||
return "unauthorized"
|
||||
case strings.Contains(lower, "forbidden") || strings.Contains(lower, "authorization failed") || strings.Contains(lower, "failed to authorize") || strings.Contains(lower, "failed to fetch anonymous token"):
|
||||
return "authorization-failed"
|
||||
case strings.Contains(lower, "pull access denied") || strings.Contains(lower, "requested access to the resource is denied"):
|
||||
return "pull-access-denied"
|
||||
default:
|
||||
return "registry-credential-error"
|
||||
}
|
||||
}
|
||||
|
||||
// vaultSyncDeployment runs one orchestration or CLI step.
|
||||
// Signature: vaultSyncDeployment(name string, labels map[string]string) bool.
|
||||
// Why: different apps may prefix their sync deployment names, but they consistently
|
||||
// identify the narrow Vault sync helper with a vault-sync name or label.
|
||||
func vaultSyncDeployment(name string, labels map[string]string) bool {
|
||||
if strings.Contains(strings.ToLower(strings.TrimSpace(name)), "vault-sync") {
|
||||
return true
|
||||
}
|
||||
for key, value := range labels {
|
||||
if strings.Contains(strings.ToLower(strings.TrimSpace(key)), "vault-sync") ||
|
||||
strings.Contains(strings.ToLower(strings.TrimSpace(value)), "vault-sync") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
87
internal/cluster/orchestrator_image_pull_credentials_test.go
Normal file
87
internal/cluster/orchestrator_image_pull_credentials_test.go
Normal file
@ -0,0 +1,87 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"scm.bstein.dev/bstein/ananke/internal/config"
|
||||
)
|
||||
|
||||
// TestImagePullCredentialBlockerReasonsClassifiesHarborAuth runs one
|
||||
// orchestration or CLI step.
|
||||
// Signature: TestImagePullCredentialBlockerReasonsClassifiesHarborAuth(t *testing.T).
|
||||
// Why: Harbor auth failures should drive secret-sync repair instead of pod
|
||||
// recycling or DNS remediation.
|
||||
func TestImagePullCredentialBlockerReasonsClassifiesHarborAuth(t *testing.T) {
|
||||
events := `{"items":[` +
|
||||
`{"involvedObject":{"kind":"Pod","namespace":"veles","name":"veles-backend"},"type":"Warning","reason":"Failed","message":"Failed to pull image \"registry.bstein.dev/veles/veles-backend:0.5.25\": no basic auth credentials"},` +
|
||||
`{"metadata":{"namespace":"veles"},"involvedObject":{"kind":"Pod","name":"veles-frontend"},"type":"Warning","reason":"FailedToRetrieveImagePullSecret","message":"Unable to retrieve some image pull secrets (harbor-regcred); attempting to pull the image may not succeed."},` +
|
||||
`{"involvedObject":{"kind":"Pod","namespace":"logging","name":"oauth2"},"type":"Warning","reason":"Failed","message":"Failed to pull image: lookup registry-1.docker.io: Try again"}` +
|
||||
`]}`
|
||||
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
|
||||
{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
|
||||
})
|
||||
|
||||
reasons, err := orch.imagePullCredentialBlockerReasons(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("imagePullCredentialBlockerReasons failed: %v", err)
|
||||
}
|
||||
if got := reasons["veles/veles-backend"]; got != "ImagePullCredentialBlocker:no-basic-auth" {
|
||||
t.Fatalf("unexpected backend credential reason %q", got)
|
||||
}
|
||||
if got := reasons["veles/veles-frontend"]; got != "ImagePullCredentialBlocker:missing-pull-secret" {
|
||||
t.Fatalf("unexpected frontend credential reason %q", got)
|
||||
}
|
||||
if _, ok := reasons["logging/oauth2"]; ok {
|
||||
t.Fatalf("DNS image-pull blocker must not be classified as credential failure")
|
||||
}
|
||||
}
|
||||
|
||||
// TestHealImagePullCredentialSyncRestartsVaultSyncDeployment runs one
|
||||
// orchestration or CLI step.
|
||||
// Signature: TestHealImagePullCredentialSyncRestartsVaultSyncDeployment(t *testing.T).
|
||||
// Why: a namespace with registry credential blockers and a Vault CSI sync helper
|
||||
// should get that sync helper rolled instead of application pods deleted.
|
||||
func TestHealImagePullCredentialSyncRestartsVaultSyncDeployment(t *testing.T) {
|
||||
events := `{"items":[{"involvedObject":{"kind":"Pod","namespace":"veles","name":"veles-backend"},"type":"Warning","reason":"Failed","message":"Failed to pull image \"registry.bstein.dev/veles/veles-backend:0.5.25\": no basic auth credentials"}]}`
|
||||
deployments := `{"items":[` +
|
||||
`{"metadata":{"name":"veles-backend","labels":{"app.kubernetes.io/name":"veles-backend"}}},` +
|
||||
`{"metadata":{"name":"veles-vault-sync","labels":{"app.kubernetes.io/component":"vault-sync"}}}` +
|
||||
`]}`
|
||||
restarted := false
|
||||
rolledOut := false
|
||||
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
|
||||
{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
|
||||
{match: matchContains("kubectl", "-n", "veles", "get", "deployment", "-o", "json"), out: deployments},
|
||||
{
|
||||
match: func(name string, args []string) bool {
|
||||
if !matchContains("kubectl", "-n", "veles", "rollout", "restart", "deployment", "veles-vault-sync")(name, args) {
|
||||
return false
|
||||
}
|
||||
restarted = true
|
||||
return true
|
||||
},
|
||||
},
|
||||
{
|
||||
match: func(name string, args []string) bool {
|
||||
if !matchContains("kubectl", "-n", "veles", "rollout", "status", "deployment/veles-vault-sync", "--timeout=60s")(name, args) {
|
||||
return false
|
||||
}
|
||||
rolledOut = true
|
||||
return true
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
repaired, err := orch.healImagePullCredentialSync(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("healImagePullCredentialSync failed: %v", err)
|
||||
}
|
||||
if strings.Join(repaired, ",") != "veles/deployment/veles-vault-sync" {
|
||||
t.Fatalf("unexpected image-pull credential repair list: %#v", repaired)
|
||||
}
|
||||
if !restarted || !rolledOut {
|
||||
t.Fatalf("expected vault-sync deployment restart and rollout wait, restarted=%v rolledOut=%v", restarted, rolledOut)
|
||||
}
|
||||
}
|
||||
@ -200,6 +200,12 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error {
|
||||
} else {
|
||||
imagePullDNSReasons = reasons
|
||||
}
|
||||
imagePullCredentialReasons := map[string]string{}
|
||||
if reasons, scanErr := o.imagePullCredentialBlockerReasons(ctx); scanErr != nil {
|
||||
o.log.Printf("warning: image-pull credential blocker scan failed: %v", scanErr)
|
||||
} else {
|
||||
imagePullCredentialReasons = reasons
|
||||
}
|
||||
containerRuntimeWedgeReasons := map[string]string{}
|
||||
if reasons, scanErr := o.containerRuntimeWedgePodReasons(ctx, list, grace); scanErr != nil {
|
||||
o.log.Printf("warning: container runtime wedge scan failed: %v", scanErr)
|
||||
@ -235,6 +241,10 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error {
|
||||
o.log.Printf("warning: not recycling pod %s/%s because image pull is blocked by DNS/registry lookup: %s", ns, name, imagePullDNSReasons[ns+"/"+name])
|
||||
continue
|
||||
}
|
||||
if (reason == "ImagePullBackOff" || reason == "ErrImagePull") && imagePullCredentialReasons[ns+"/"+name] != "" {
|
||||
o.log.Printf("warning: not recycling pod %s/%s because image pull is blocked by registry credentials: %s", ns, name, imagePullCredentialReasons[ns+"/"+name])
|
||||
continue
|
||||
}
|
||||
if reason == "" {
|
||||
reason = stuckVaultInitReason(pod, grace)
|
||||
}
|
||||
|
||||
@ -22,6 +22,7 @@ internal/cluster/orchestrator_critical_endpoint_additional_test.go
|
||||
internal/cluster/orchestrator_cordon_lease_test.go
|
||||
internal/cluster/orchestrator_test.go
|
||||
internal/cluster/orchestrator_hardening_test.go
|
||||
internal/cluster/orchestrator_image_pull_credentials_test.go
|
||||
internal/cluster/orchestrator_unit_additional_test.go
|
||||
internal/cluster/orchestrator_workload_recovery_test.go
|
||||
internal/cluster/orchestrator_vault_test.go
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user