ananke/internal/cluster/testing_hooks_workloads.go

463 lines
21 KiB
Go

package cluster
import (
"context"
"fmt"
"sort"
"strings"
"time"
)
// TestHookWaitForCriticalServiceEndpoints runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookWaitForCriticalServiceEndpoints(ctx context.Context) error.
// Why: exposes critical endpoint gate internals to top-level tests.
func (o *Orchestrator) TestHookWaitForCriticalServiceEndpoints(ctx context.Context) error {
return o.waitForCriticalServiceEndpoints(ctx)
}
// TestHookMaybeHealCriticalEndpointBackends runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookMaybeHealCriticalEndpointBackends(ctx context.Context, namespace string, service string) ([]string, error).
// Why: exposes critical endpoint backend-heal internals to top-level tests.
func (o *Orchestrator) TestHookMaybeHealCriticalEndpointBackends(ctx context.Context, namespace string, service string) ([]string, error) {
return o.maybeHealCriticalEndpointBackends(ctx, strings.TrimSpace(namespace), strings.TrimSpace(service))
}
// TestHookCriticalServiceEndpointsReady runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookCriticalServiceEndpointsReady(ctx context.Context) (bool, string, string, string, error).
// Why: exposes critical endpoint poll internals to top-level tests.
func (o *Orchestrator) TestHookCriticalServiceEndpointsReady(ctx context.Context) (bool, string, string, string, error) {
return o.criticalServiceEndpointsReady(ctx)
}
// TestHookWaitForWorkloadConvergence runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookWaitForWorkloadConvergence(ctx context.Context) error.
// Why: exposes workload convergence gate internals to top-level tests.
func (o *Orchestrator) TestHookWaitForWorkloadConvergence(ctx context.Context) error {
return o.waitForWorkloadConvergence(ctx)
}
// TestHookWorkloadConvergenceReady runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookWorkloadConvergenceReady(ctx context.Context) (bool, string, error).
// Why: exposes workload convergence poll internals to top-level tests.
func (o *Orchestrator) TestHookWorkloadConvergenceReady(ctx context.Context) (bool, string, error) {
return o.workloadConvergenceReady(ctx)
}
// TestHookRecycleStuckControllerPods runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookRecycleStuckControllerPods(ctx context.Context) error.
// Why: exposes stuck-pod recycle internals to top-level tests.
func (o *Orchestrator) TestHookRecycleStuckControllerPods(ctx context.Context) error {
return o.recycleStuckControllerPods(ctx)
}
// TestHookMaybeAutoRecycleStuckPods runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookMaybeAutoRecycleStuckPods(ctx context.Context, lastAttempt *time.Time).
// Why: exposes auto-recycle trigger internals to top-level tests.
func (o *Orchestrator) TestHookMaybeAutoRecycleStuckPods(ctx context.Context, lastAttempt *time.Time) {
o.maybeAutoRecycleStuckPods(ctx, lastAttempt)
}
// TestHookMaybeAutoHealCriticalWorkloadReplicas runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookMaybeAutoHealCriticalWorkloadReplicas(ctx context.Context, lastAttempt *time.Time).
// Why: exposes critical workload auto-heal trigger internals to top-level tests.
func (o *Orchestrator) TestHookMaybeAutoHealCriticalWorkloadReplicas(ctx context.Context, lastAttempt *time.Time) {
o.maybeAutoHealCriticalWorkloadReplicas(ctx, lastAttempt)
}
// TestHookHealCriticalWorkloadReplicas runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookHealCriticalWorkloadReplicas(ctx context.Context) ([]string, error).
// Why: exposes critical workload replica-heal internals to top-level tests.
func (o *Orchestrator) TestHookHealCriticalWorkloadReplicas(ctx context.Context) ([]string, error) {
return o.healCriticalWorkloadReplicas(ctx)
}
// TestHookStartupFailurePods runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookStartupFailurePods(ctx context.Context) ([]string, error).
// Why: exposes startup failure pod summarizer internals to top-level tests.
func (o *Orchestrator) TestHookStartupFailurePods(ctx context.Context) ([]string, error) {
return o.startupFailurePods(ctx)
}
// TestHookEnsureCriticalStartupWorkloads runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookEnsureCriticalStartupWorkloads(ctx context.Context) error.
// Why: exposes critical workload ensure path to top-level tests.
func (o *Orchestrator) TestHookEnsureCriticalStartupWorkloads(ctx context.Context) error {
return o.ensureCriticalStartupWorkloads(ctx)
}
// TestHookMissingCriticalStartupWorkloads runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookMissingCriticalStartupWorkloads(ctx context.Context) ([]string, error).
// Why: exposes missing critical workload scanner internals to top-level tests.
func (o *Orchestrator) TestHookMissingCriticalStartupWorkloads(ctx context.Context) ([]string, error) {
return o.missingCriticalStartupWorkloads(ctx)
}
// TestHookCleanupStaleCriticalWorkloadPods runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookCleanupStaleCriticalWorkloadPods(ctx context.Context, namespace string, kind string, name string) error.
// Why: exposes stale critical-pod cleanup internals to top-level tests.
func (o *Orchestrator) TestHookCleanupStaleCriticalWorkloadPods(ctx context.Context, namespace string, kind string, name string) error {
return o.cleanupStaleCriticalWorkloadPods(ctx, startupWorkload{
Namespace: strings.TrimSpace(namespace),
Kind: strings.TrimSpace(kind),
Name: strings.TrimSpace(name),
})
}
// TestHookEnsureWorkloadReplicas runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookEnsureWorkloadReplicas(ctx context.Context, namespace string, kind string, name string, replicas int) error.
// Why: exposes workload scaling helper internals to top-level tests.
func (o *Orchestrator) TestHookEnsureWorkloadReplicas(ctx context.Context, namespace string, kind string, name string, replicas int) error {
return o.ensureWorkloadReplicas(ctx, startupWorkload{
Namespace: strings.TrimSpace(namespace),
Kind: strings.TrimSpace(kind),
Name: strings.TrimSpace(name),
}, replicas)
}
// TestHookWaitWorkloadReady runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookWaitWorkloadReady(ctx context.Context, namespace string, kind string, name string) error.
// Why: exposes workload readiness waiter internals to top-level tests.
func (o *Orchestrator) TestHookWaitWorkloadReady(ctx context.Context, namespace string, kind string, name string) error {
return o.waitWorkloadReady(ctx, startupWorkload{
Namespace: strings.TrimSpace(namespace),
Kind: strings.TrimSpace(kind),
Name: strings.TrimSpace(name),
})
}
// TestHookWaitVaultReady runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookWaitVaultReady(ctx context.Context, namespace string, kind string, name string) error.
// Why: exposes vault-specific readiness loop internals to top-level tests.
func (o *Orchestrator) TestHookWaitVaultReady(ctx context.Context, namespace string, kind string, name string) error {
return o.waitVaultReady(ctx, startupWorkload{
Namespace: strings.TrimSpace(namespace),
Kind: strings.TrimSpace(kind),
Name: strings.TrimSpace(name),
})
}
// TestHookEnsureVaultUnsealed runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookEnsureVaultUnsealed(ctx context.Context) error.
// Why: exposes vault auto-unseal internals to top-level tests.
func (o *Orchestrator) TestHookEnsureVaultUnsealed(ctx context.Context) error {
return o.ensureVaultUnsealed(ctx)
}
// TestHookVaultSealed runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookVaultSealed(ctx context.Context) (bool, error).
// Why: exposes vault sealed-state parser internals to top-level tests.
func (o *Orchestrator) TestHookVaultSealed(ctx context.Context) (bool, error) {
return o.vaultSealed(ctx)
}
// TestHookVaultUnsealKey runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookVaultUnsealKey(ctx context.Context) (string, error).
// Why: exposes unseal key resolution internals to top-level tests.
func (o *Orchestrator) TestHookVaultUnsealKey(ctx context.Context) (string, error) {
return o.vaultUnsealKey(ctx)
}
// TestHookReadVaultUnsealKeyBreakglass runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookReadVaultUnsealKeyBreakglass(ctx context.Context) (string, error).
// Why: exposes break-glass fallback key command internals to top-level tests.
func (o *Orchestrator) TestHookReadVaultUnsealKeyBreakglass(ctx context.Context) (string, error) {
return o.readVaultUnsealKeyBreakglass(ctx)
}
// TestHookWriteVaultUnsealKeyFile runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookWriteVaultUnsealKeyFile(key string) error.
// Why: exposes local unseal key cache writer internals to top-level tests.
func (o *Orchestrator) TestHookWriteVaultUnsealKeyFile(key string) error {
return o.writeVaultUnsealKeyFile(key)
}
// TestHookReadVaultUnsealKeyFile runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookReadVaultUnsealKeyFile() (string, error).
// Why: exposes local unseal key cache reader internals to top-level tests.
func (o *Orchestrator) TestHookReadVaultUnsealKeyFile() (string, error) {
return o.readVaultUnsealKeyFile()
}
// TestHookWorkloadReady runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookWorkloadReady(ctx context.Context, namespace string, kind string, name string) (bool, error).
// Why: exposes readyReplicas readiness parser internals to top-level tests.
func (o *Orchestrator) TestHookWorkloadReady(ctx context.Context, namespace string, kind string, name string) (bool, error) {
return o.workloadReady(ctx, startupWorkload{
Namespace: strings.TrimSpace(namespace),
Kind: strings.TrimSpace(kind),
Name: strings.TrimSpace(name),
})
}
// TestHookParseVaultSealed runs one orchestration or CLI step.
// Signature: TestHookParseVaultSealed(raw string) (bool, error).
// Why: exposes vault status JSON parser helper to top-level tests.
func TestHookParseVaultSealed(raw string) (bool, error) {
return parseVaultSealed(raw)
}
// TestHookIsNotFoundErr runs one orchestration or CLI step.
// Signature: TestHookIsNotFoundErr(errText string) bool.
// Why: exposes not-found error classifier helper to top-level tests.
func TestHookIsNotFoundErr(errText string) bool {
if strings.TrimSpace(errText) == "" {
return isNotFoundErr(nil)
}
return isNotFoundErr(fmt.Errorf("%s", errText))
}
// TestHookWaitForServiceChecklistAlias runs one orchestration or CLI step.
// Signature: (o *Orchestrator) TestHookWaitForServiceChecklistAlias(ctx context.Context) error.
// Why: alias exported for top-level tests that focus on workload-driven checklist behavior.
func (o *Orchestrator) TestHookWaitForServiceChecklistAlias(ctx context.Context) error {
return o.waitForServiceChecklist(ctx)
}
// TestHookDesiredReady runs one orchestration or CLI step.
// Signature: TestHookDesiredReady(kind string, hasReplicas bool, replicas int32, ready int32, desiredScheduled int32, numberReady int32) (int32, int32, bool).
// Why: exposes controller desired/ready resolver helper to top-level tests.
func TestHookDesiredReady(kind string, hasReplicas bool, replicas int32, ready int32, desiredScheduled int32, numberReady int32) (int32, int32, bool) {
var item workloadResource
item.Kind = kind
if hasReplicas {
value := replicas
item.Spec.Replicas = &value
}
item.Status.ReadyReplicas = ready
item.Status.DesiredNumberScheduled = desiredScheduled
item.Status.NumberReady = numberReady
return desiredReady(item)
}
// TestHookPodControllerOwned runs one orchestration or CLI step.
// Signature: TestHookPodControllerOwned(ownerKinds []string) bool.
// Why: exposes controller ownership helper to top-level tests.
func TestHookPodControllerOwned(ownerKinds []string) bool {
var pod podResource
for _, kind := range ownerKinds {
pod.Metadata.OwnerReferences = append(pod.Metadata.OwnerReferences, ownerReference{Kind: kind})
}
return podControllerOwned(pod)
}
// TestHookStuckContainerReason runs one orchestration or CLI step.
// Signature: TestHookStuckContainerReason(initReasons []string, containerReasons []string, allowed []string) string.
// Why: exposes stuck-container reason helper to top-level tests.
func TestHookStuckContainerReason(initReasons []string, containerReasons []string, allowed []string) string {
reasons := map[string]struct{}{}
for _, reason := range allowed {
reasons[reason] = struct{}{}
}
var pod podResource
for _, reason := range initReasons {
pod.Status.InitContainerStatuses = append(pod.Status.InitContainerStatuses, podContainerStatus{
State: podContainerState{Waiting: &podContainerWaitingState{Reason: reason}},
})
}
for _, reason := range containerReasons {
pod.Status.ContainerStatuses = append(pod.Status.ContainerStatuses, podContainerStatus{
State: podContainerState{Waiting: &podContainerWaitingState{Reason: reason}},
})
}
return stuckContainerReason(pod, reasons)
}
// TestHookStuckVaultInitReason runs one orchestration or CLI step.
// Signature: TestHookStuckVaultInitReason(phase string, inject bool, startedAgo time.Duration, grace time.Duration) string.
// Why: exposes vault-init stuck detector helper to top-level tests.
func TestHookStuckVaultInitReason(phase string, inject bool, startedAgo time.Duration, grace time.Duration) string {
var pod podResource
pod.Status.Phase = phase
pod.Metadata.Annotations = map[string]string{}
if inject {
pod.Metadata.Annotations["vault.hashicorp.com/agent-inject"] = "true"
}
if startedAgo > 0 {
pod.Status.InitContainerStatuses = append(pod.Status.InitContainerStatuses, podContainerStatus{
Name: "vault-agent-init",
State: podContainerState{
Running: &podContainerRunningState{StartedAt: time.Now().Add(-startedAgo)},
},
})
}
return stuckVaultInitReason(pod, grace)
}
// TestHookPodTargetsIgnoredNode runs one orchestration or CLI step.
// Signature: TestHookPodTargetsIgnoredNode(nodeName string, ignored []string) bool.
// Why: exposes ignored-node targeting helper to top-level tests.
func TestHookPodTargetsIgnoredNode(nodeName string, ignored []string) bool {
var pod podResource
pod.Spec.NodeName = nodeName
return podTargetsIgnoredNode(pod, makeStringSet(ignored))
}
// TestHookWorkloadTargetsIgnoredNodes runs one orchestration or CLI step.
// Signature: TestHookWorkloadTargetsIgnoredNodes(nodeSelectorHost string, affinityHosts []string, ignored []string) bool.
// Why: exposes ignored-node selector/affinity helper to top-level tests.
func TestHookWorkloadTargetsIgnoredNodes(nodeSelectorHost string, affinityHosts []string, ignored []string) bool {
spec := podSpec{NodeSelector: map[string]string{}}
if nodeSelectorHost != "" {
spec.NodeSelector["kubernetes.io/hostname"] = nodeSelectorHost
}
if len(affinityHosts) > 0 {
spec.Affinity = &podAffinity{
NodeAffinity: &nodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &nodeSelector{
NodeSelectorTerms: []nodeSelectorTerm{
{
MatchExpressions: []nodeSelectorRequirement{
{
Key: "kubernetes.io/hostname",
Operator: "In",
Values: affinityHosts,
},
},
},
},
},
},
}
}
return workloadTargetsIgnoredNodes(spec, makeStringSet(ignored))
}
// TestHookNodeSelectorExpr defines one synthetic affinity expression for tests.
// Signature: type TestHookNodeSelectorExpr struct { Key string; Operator string; Values []string }.
// Why: allows top-level tests to drive workload ignore affinity branches that
// require malformed or non-default expression shapes.
type TestHookNodeSelectorExpr struct {
Key string
Operator string
Values []string
}
// TestHookWorkloadTargetsIgnoredNodesRaw runs one orchestration or CLI step.
// Signature: TestHookWorkloadTargetsIgnoredNodesRaw(termExpressions [][]TestHookNodeSelectorExpr, ignored []string) bool.
// Why: exposes raw affinity-shape branch coverage for ignored-node matching.
func TestHookWorkloadTargetsIgnoredNodesRaw(termExpressions [][]TestHookNodeSelectorExpr, ignored []string) bool {
spec := podSpec{}
if len(termExpressions) > 0 {
terms := make([]nodeSelectorTerm, 0, len(termExpressions))
for _, exprs := range termExpressions {
term := nodeSelectorTerm{
MatchExpressions: make([]nodeSelectorRequirement, 0, len(exprs)),
}
for _, expr := range exprs {
term.MatchExpressions = append(term.MatchExpressions, nodeSelectorRequirement{
Key: strings.TrimSpace(expr.Key),
Operator: strings.TrimSpace(expr.Operator),
Values: append([]string{}, expr.Values...),
})
}
terms = append(terms, term)
}
spec.Affinity = &podAffinity{
NodeAffinity: &nodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &nodeSelector{
NodeSelectorTerms: terms,
},
},
}
}
return workloadTargetsIgnoredNodes(spec, makeStringSet(ignored))
}
// TestHookStuckVaultInitInput defines one synthetic vault-init pod state.
// Signature: type TestHookStuckVaultInitInput struct { Phase string; Inject bool; InitContainerName string; Running bool; StartedAtOffsetSec int64; GraceSeconds int64 }.
// Why: allows top-level tests to drive all stuckVaultInitReason branches,
// including edge inputs that are awkward to represent with high-level helpers.
type TestHookStuckVaultInitInput struct {
Phase string
Inject bool
InitContainerName string
Running bool
StartedAtOffsetSec int64
GraceSeconds int64
}
// TestHookStuckVaultInitReasonRaw runs one orchestration or CLI step.
// Signature: TestHookStuckVaultInitReasonRaw(in TestHookStuckVaultInitInput) string.
// Why: exposes raw vault-init stuck detector inputs for full branch coverage.
func TestHookStuckVaultInitReasonRaw(in TestHookStuckVaultInitInput) string {
var pod podResource
pod.Status.Phase = strings.TrimSpace(in.Phase)
pod.Metadata.Annotations = map[string]string{}
if in.Inject {
pod.Metadata.Annotations["vault.hashicorp.com/agent-inject"] = "true"
}
state := podContainerState{}
if in.Running {
startedAt := time.Time{}
if in.StartedAtOffsetSec != 0 {
startedAt = time.Now().Add(-time.Duration(in.StartedAtOffsetSec) * time.Second)
}
state.Running = &podContainerRunningState{StartedAt: startedAt}
}
name := strings.TrimSpace(in.InitContainerName)
if name == "" {
name = "vault-agent-init"
}
pod.Status.InitContainerStatuses = append(pod.Status.InitContainerStatuses, podContainerStatus{
Name: name,
State: state,
})
grace := time.Duration(in.GraceSeconds) * time.Second
return stuckVaultInitReason(pod, grace)
}
// TestHookParseWorkloadIgnoreRulesCount runs one orchestration or CLI step.
// Signature: TestHookParseWorkloadIgnoreRulesCount(entries []string) int.
// Why: exposes workload-ignore rule parser helper to top-level tests.
func TestHookParseWorkloadIgnoreRulesCount(entries []string) int {
return len(parseWorkloadIgnoreRules(entries))
}
// TestHookWorkloadIgnoredEntries runs one orchestration or CLI step.
// Signature: TestHookWorkloadIgnoredEntries(entries []string, namespace string, kind string, name string) bool.
// Why: exposes workload-ignore matcher helper to top-level tests.
func TestHookWorkloadIgnoredEntries(entries []string, namespace string, kind string, name string) bool {
rules := parseWorkloadIgnoreRules(entries)
return workloadIgnored(rules, namespace, kind, name)
}
// TestHookReadyConditionIndex runs one orchestration or CLI step.
// Signature: TestHookReadyConditionIndex(types []string) int.
// Why: exposes ready-condition selector helper to top-level tests.
func TestHookReadyConditionIndex(types []string) int {
conditions := make([]fluxCondition, 0, len(types))
for _, t := range types {
conditions = append(conditions, fluxCondition{Type: t})
}
cond := readyCondition(conditions)
if cond == nil {
return -1
}
for i := range conditions {
if &conditions[i] == cond {
return i
}
}
return -1
}
// TestHookJoinLimited runs one orchestration or CLI step.
// Signature: TestHookJoinLimited(items []string, limit int) string.
// Why: exposes compact list formatter helper to top-level tests.
func TestHookJoinLimited(items []string, limit int) string {
return joinLimited(items, limit)
}
// TestHookNamespaceCandidatesFromIgnoreKustomizations runs one orchestration or CLI step.
// Signature: TestHookNamespaceCandidatesFromIgnoreKustomizations(entries []string) []string.
// Why: exposes namespace-candidate helper to top-level tests.
func TestHookNamespaceCandidatesFromIgnoreKustomizations(entries []string) []string {
set := namespaceCandidatesFromIgnoreKustomizations(entries)
out := make([]string, 0, len(set))
for ns := range set {
out = append(out, ns)
}
sort.Strings(out)
return out
}