238 lines
10 KiB
Go
238 lines
10 KiB
Go
|
package cluster
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"encoding/json"
|
||
|
|
"errors"
|
||
|
|
"fmt"
|
||
|
|
"sort"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
)
|
||
|
|
|
||
|
|
type imagePullCredentialDeploymentList struct {
|
||
|
|
Items []struct {
|
||
|
|
Metadata struct {
|
||
|
|
Name string `json:"name"`
|
||
|
|
Labels map[string]string `json:"labels"`
|
||
|
|
} `json:"metadata"`
|
||
|
|
} `json:"items"`
|
||
|
|
}
|
||
|
|
|
||
|
|
// imagePullCredentialBlockerReasons classifies image pull failures caused by
|
||
|
|
// registry auth or missing imagePullSecret material.
|
||
|
|
// Signature: (o *Orchestrator) imagePullCredentialBlockerReasons(ctx context.Context) (map[string]string, error).
|
||
|
|
// Why: deleting a pod with bad registry credentials does not refresh Vault/CSI
|
||
|
|
// secret material; the secret sync path needs a direct repair signal.
|
||
|
|
func (o *Orchestrator) imagePullCredentialBlockerReasons(ctx context.Context) (map[string]string, error) {
|
||
|
|
eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
|
||
|
|
if err != nil {
|
||
|
|
return nil, fmt.Errorf("query events for image-pull credential scan: %w", err)
|
||
|
|
}
|
||
|
|
reasons := map[string]string{}
|
||
|
|
if strings.TrimSpace(eventsOut) == "" {
|
||
|
|
return reasons, nil
|
||
|
|
}
|
||
|
|
var events eventList
|
||
|
|
if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
|
||
|
|
return nil, fmt.Errorf("decode events for image-pull credential scan: %w", err)
|
||
|
|
}
|
||
|
|
for _, event := range events.Items {
|
||
|
|
if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
reason := strings.TrimSpace(event.Reason)
|
||
|
|
message := strings.TrimSpace(event.Message)
|
||
|
|
if !imagePullCredentialEvent(reason, message) {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
namespace := strings.TrimSpace(event.InvolvedObject.Namespace)
|
||
|
|
if namespace == "" {
|
||
|
|
namespace = strings.TrimSpace(event.Metadata.Namespace)
|
||
|
|
}
|
||
|
|
name := strings.TrimSpace(event.InvolvedObject.Name)
|
||
|
|
if namespace == "" || name == "" {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
reasons[namespace+"/"+name] = "ImagePullCredentialBlocker:" + imagePullCredentialFailureClass(reason, message)
|
||
|
|
}
|
||
|
|
return reasons, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// healImagePullCredentialSync restarts namespace-local Vault sync deployments
|
||
|
|
// when pods are blocked on registry credentials.
|
||
|
|
// Signature: (o *Orchestrator) healImagePullCredentialSync(ctx context.Context) ([]string, error).
|
||
|
|
// Why: Secrets Store CSI secretObjects are materialized by mounted workloads; a
|
||
|
|
// restarted vault-sync deployment is the bounded repair that refreshes pull
|
||
|
|
// secrets such as harbor-regcred without mutating application deployments.
|
||
|
|
func (o *Orchestrator) healImagePullCredentialSync(ctx context.Context) ([]string, error) {
|
||
|
|
if o.runner.DryRun {
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
blockers, err := o.imagePullCredentialBlockerReasons(ctx)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
if len(blockers) == 0 {
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
namespaces := map[string]struct{}{}
|
||
|
|
for key := range blockers {
|
||
|
|
namespace, _, ok := strings.Cut(key, "/")
|
||
|
|
if ok && strings.TrimSpace(namespace) != "" {
|
||
|
|
namespaces[namespace] = struct{}{}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orderedNamespaces := make([]string, 0, len(namespaces))
|
||
|
|
for namespace := range namespaces {
|
||
|
|
orderedNamespaces = append(orderedNamespaces, namespace)
|
||
|
|
}
|
||
|
|
sort.Strings(orderedNamespaces)
|
||
|
|
|
||
|
|
repaired := []string{}
|
||
|
|
errs := []string{}
|
||
|
|
for _, namespace := range orderedNamespaces {
|
||
|
|
namespaceRepairs, restartErr := o.restartVaultSyncDeployments(ctx, namespace)
|
||
|
|
repaired = append(repaired, namespaceRepairs...)
|
||
|
|
if restartErr != nil {
|
||
|
|
errs = append(errs, restartErr.Error())
|
||
|
|
}
|
||
|
|
}
|
||
|
|
sort.Strings(repaired)
|
||
|
|
if len(errs) > 0 {
|
||
|
|
return repaired, errors.New(strings.Join(errs, "; "))
|
||
|
|
}
|
||
|
|
return repaired, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// restartVaultSyncDeployments runs one orchestration or CLI step.
|
||
|
|
// Signature: (o *Orchestrator) restartVaultSyncDeployments(ctx context.Context, namespace string) ([]string, error).
|
||
|
|
// Why: namespaces using Vault/CSI secret material have tiny sync deployments
|
||
|
|
// named or labeled vault-sync; restarting those nudges secretObject rotation.
|
||
|
|
func (o *Orchestrator) restartVaultSyncDeployments(ctx context.Context, namespace string) ([]string, error) {
|
||
|
|
out, err := o.kubectl(ctx, 20*time.Second, "-n", namespace, "get", "deployment", "-o", "json")
|
||
|
|
if err != nil {
|
||
|
|
return nil, fmt.Errorf("query deployments in %s for image-pull credential repair: %w", namespace, err)
|
||
|
|
}
|
||
|
|
var deployments imagePullCredentialDeploymentList
|
||
|
|
if err := json.Unmarshal([]byte(out), &deployments); err != nil {
|
||
|
|
return nil, fmt.Errorf("decode deployments in %s for image-pull credential repair: %w", namespace, err)
|
||
|
|
}
|
||
|
|
|
||
|
|
repaired := []string{}
|
||
|
|
for _, deployment := range deployments.Items {
|
||
|
|
name := strings.TrimSpace(deployment.Metadata.Name)
|
||
|
|
if name == "" || !vaultSyncDeployment(name, deployment.Metadata.Labels) {
|
||
|
|
continue
|
||
|
|
}
|
||
|
|
if _, err := o.kubectl(ctx, 25*time.Second, "-n", namespace, "rollout", "restart", "deployment", name); err != nil {
|
||
|
|
return repaired, fmt.Errorf("restart %s/deployment/%s for image-pull credential repair: %w", namespace, name, err)
|
||
|
|
}
|
||
|
|
if _, err := o.kubectl(ctx, 75*time.Second, "-n", namespace, "rollout", "status", "deployment/"+name, "--timeout=60s"); err != nil {
|
||
|
|
return repaired, fmt.Errorf("wait for %s/deployment/%s after image-pull credential repair: %w", namespace, name, err)
|
||
|
|
}
|
||
|
|
repaired = append(repaired, namespace+"/deployment/"+name)
|
||
|
|
}
|
||
|
|
if len(repaired) == 0 {
|
||
|
|
return nil, fmt.Errorf("image-pull credential blocker in namespace %s but no vault-sync deployment was found", namespace)
|
||
|
|
}
|
||
|
|
return repaired, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// imagePullCredentialEvent runs one orchestration or CLI step.
|
||
|
|
// Signature: imagePullCredentialEvent(reason string, message string) bool.
|
||
|
|
// Why: image-pull auth detection must include kubelet pull-secret events while
|
||
|
|
// staying separate from DNS and ordinary transient pull backoff.
|
||
|
|
func imagePullCredentialEvent(reason string, message string) bool {
|
||
|
|
normalizedReason := strings.ToLower(strings.TrimSpace(reason))
|
||
|
|
if normalizedReason == "failedtoretrieveimagepullsecret" {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
if normalizedReason != "failed" &&
|
||
|
|
normalizedReason != "failedpull" &&
|
||
|
|
normalizedReason != "errimagepull" &&
|
||
|
|
normalizedReason != "imagepullbackoff" {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
return imagePullMessageHasCredentialFailure(message)
|
||
|
|
}
|
||
|
|
|
||
|
|
// imagePullMessageHasCredentialFailure runs one orchestration or CLI step.
|
||
|
|
// Signature: imagePullMessageHasCredentialFailure(message string) bool.
|
||
|
|
// Why: credential blockers need a tight predicate so missing images and registry
|
||
|
|
// DNS outages keep their own repair paths.
|
||
|
|
func imagePullMessageHasCredentialFailure(message string) bool {
|
||
|
|
lower := strings.ToLower(strings.TrimSpace(message))
|
||
|
|
if lower == "" || imagePullMessageHasDNSFailure(message) {
|
||
|
|
return false
|
||
|
|
}
|
||
|
|
return strings.Contains(lower, "failed to retrieve image pull secret") ||
|
||
|
|
strings.Contains(lower, "failedtoretrieveimagepullsecret") ||
|
||
|
|
strings.Contains(lower, "unable to retrieve some image pull secrets") ||
|
||
|
|
(strings.Contains(lower, "image pull secret") && strings.Contains(lower, "not found")) ||
|
||
|
|
(strings.Contains(lower, "pull secret") && strings.Contains(lower, "not found")) ||
|
||
|
|
(strings.Contains(lower, "secret ") && strings.Contains(lower, " not found") && strings.Contains(lower, "pull")) ||
|
||
|
|
strings.Contains(lower, "no basic auth credentials") ||
|
||
|
|
strings.Contains(lower, "unauthorized") ||
|
||
|
|
strings.Contains(lower, "authentication required") ||
|
||
|
|
strings.Contains(lower, "authorization failed") ||
|
||
|
|
strings.Contains(lower, "failed to authorize") ||
|
||
|
|
strings.Contains(lower, "invalid username/password") ||
|
||
|
|
strings.Contains(lower, "401 unauthorized") ||
|
||
|
|
strings.Contains(lower, "403 forbidden") ||
|
||
|
|
strings.Contains(lower, "pull access denied") ||
|
||
|
|
strings.Contains(lower, "requested access to the resource is denied") ||
|
||
|
|
strings.Contains(lower, "failed to fetch anonymous token")
|
||
|
|
}
|
||
|
|
|
||
|
|
// imagePullCredentialFailureClass runs one orchestration or CLI step.
|
||
|
|
// Signature: imagePullCredentialFailureClass(reason string, message string) string.
|
||
|
|
// Why: compact blocker classes keep logs and status useful without copying long
|
||
|
|
// kubelet event messages.
|
||
|
|
func imagePullCredentialFailureClass(reason string, message string) string {
|
||
|
|
lowerReason := strings.ToLower(strings.TrimSpace(reason))
|
||
|
|
lower := strings.ToLower(strings.TrimSpace(message))
|
||
|
|
switch {
|
||
|
|
case lowerReason == "failedtoretrieveimagepullsecret",
|
||
|
|
strings.Contains(lower, "failed to retrieve image pull secret"),
|
||
|
|
strings.Contains(lower, "unable to retrieve some image pull secrets"),
|
||
|
|
strings.Contains(lower, "image pull secret") && strings.Contains(lower, "not found"),
|
||
|
|
strings.Contains(lower, "pull secret") && strings.Contains(lower, "not found"),
|
||
|
|
strings.Contains(lower, "secret ") && strings.Contains(lower, " not found") && strings.Contains(lower, "pull"):
|
||
|
|
return "missing-pull-secret"
|
||
|
|
case strings.Contains(lower, "no basic auth credentials"):
|
||
|
|
return "no-basic-auth"
|
||
|
|
case strings.Contains(lower, "invalid username/password"):
|
||
|
|
return "invalid-credentials"
|
||
|
|
case strings.Contains(lower, "unauthorized") || strings.Contains(lower, "authentication required"):
|
||
|
|
return "unauthorized"
|
||
|
|
case strings.Contains(lower, "forbidden") || strings.Contains(lower, "authorization failed") || strings.Contains(lower, "failed to authorize") || strings.Contains(lower, "failed to fetch anonymous token"):
|
||
|
|
return "authorization-failed"
|
||
|
|
case strings.Contains(lower, "pull access denied") || strings.Contains(lower, "requested access to the resource is denied"):
|
||
|
|
return "pull-access-denied"
|
||
|
|
default:
|
||
|
|
return "registry-credential-error"
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// vaultSyncDeployment runs one orchestration or CLI step.
|
||
|
|
// Signature: vaultSyncDeployment(name string, labels map[string]string) bool.
|
||
|
|
// Why: different apps may prefix their sync deployment names, but they consistently
|
||
|
|
// identify the narrow Vault sync helper with a vault-sync name or label.
|
||
|
|
func vaultSyncDeployment(name string, labels map[string]string) bool {
|
||
|
|
if strings.Contains(strings.ToLower(strings.TrimSpace(name)), "vault-sync") {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
for key, value := range labels {
|
||
|
|
if strings.Contains(strings.ToLower(strings.TrimSpace(key)), "vault-sync") ||
|
||
|
|
strings.Contains(strings.ToLower(strings.TrimSpace(value)), "vault-sync") {
|
||
|
|
return true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return false
|
||
|
|
}
|