package orchestrator import ( "context" "errors" "strings" "testing" "time" "scm.bstein.dev/bstein/ananke/internal/cluster" ) // TestHookCriticalVaultMatrixExtra runs one orchestration or CLI step. // Signature: TestHookCriticalVaultMatrixExtra(t *testing.T). // Why: drives deeper vault/critical-workload recovery branches so sealed/unready // states are exercised before the next power-cycle drill. func TestHookCriticalVaultMatrixExtra(t *testing.T) { t.Run("missing-and-ensure-critical-workloads", func(t *testing.T) { cfg := lifecycleConfig(t) run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "get deployment source-controller"): return "", errors.New("Error from server (NotFound): deployments.apps \"source-controller\" not found") case name == "kubectl" && strings.Contains(command, "get deployment kustomize-controller"): return "0", nil case name == "kubectl" && strings.Contains(command, "get") && strings.Contains(command, "jsonpath={.status.readyReplicas}"): return "1", nil case name == "kubectl" && strings.Contains(command, " scale deployment source-controller --replicas=1"): return "", errors.New("Error from server (NotFound): deployments.apps \"source-controller\" not found") case name == "kubectl" && strings.Contains(command, " scale statefulset vault --replicas=1"): return "", nil case name == "kubectl" && strings.Contains(command, " -n vault get pods -o custom-columns="): return "", errors.New("Error from server (NotFound): pods not found") case name == "kubectl" && strings.Contains(command, " scale deployment gitea --replicas=1"): return "", nil case name == "kubectl" && strings.Contains(command, "rollout status deployment/gitea"): return "", errors.New("Error from server (NotFound): deployments.apps \"gitea\" not found") case name == "kubectl" && strings.Contains(command, "rollout status"): return "rolled out", nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orch, _ := newHookOrchestrator(t, cfg, run, run) missing, err := orch.TestHookMissingCriticalStartupWorkloads(context.Background()) if err != nil || len(missing) == 0 { t.Fatalf("expected missing critical workload list, missing=%v err=%v", missing, err) } if err := orch.TestHookEnsureCriticalStartupWorkloads(context.Background()); err != nil { t.Fatalf("expected ensureCriticalStartupWorkloads notfound branches to continue, got %v", err) } }) t.Run("vault-ready-unsealed-and-key-branches", func(t *testing.T) { cfg := lifecycleConfig(t) cfg.Startup.VaultUnsealBreakglassCommand = "echo breakglass-key" readyCalls := 0 run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "-n vault get statefulset vault -o jsonpath={.status.readyReplicas}"): readyCalls++ if readyCalls < 2 { return "0", nil } return "1", nil case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"): return "Running", nil case name == "kubectl" && strings.Contains(command, "vault status -format=json"): if readyCalls < 2 { return `{"sealed":true}`, nil } return `{"sealed":false}`, nil case name == "kubectl" && strings.Contains(command, "get secret vault-init"): return "", errors.New("secret missing") case name == "sh" && strings.Contains(command, "echo breakglass-key"): return "breakglass-key", nil case name == "kubectl" && strings.Contains(command, "vault operator unseal"): return "unsealed", nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orch, _ := newHookOrchestrator(t, cfg, run, run) if err := orch.TestHookWaitVaultReady(context.Background(), "vault", "statefulset", "vault"); err != nil { t.Fatalf("expected waitVaultReady unseal branch success, got %v", err) } if sealed, err := orch.TestHookVaultSealed(context.Background()); err != nil || sealed { t.Fatalf("expected vaultSealed false after unseal, sealed=%v err=%v", sealed, err) } key, err := orch.TestHookVaultUnsealKey(context.Background()) if err != nil || strings.TrimSpace(key) == "" { t.Fatalf("expected breakglass vault key fallback, key=%q err=%v", key, err) } if _, err := cluster.TestHookParseVaultSealed("not-json"); err == nil { t.Fatalf("expected parseVaultSealed error for malformed payload") } if !cluster.TestHookIsNotFoundErr("Error from server (NotFound)") { t.Fatalf("expected notfound helper to detect notfound text") } }) t.Run("vault-write-workload-and-phase-errors", func(t *testing.T) { cfg := lifecycleConfig(t) cfg.Startup.VaultUnsealKeyFile = "" runPhase := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"): return "Pending", nil case name == "kubectl" && strings.Contains(command, "-n vault get statefulset vault -o jsonpath={.status.readyReplicas}"): return "not-a-number", nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orchPhase, _ := newHookOrchestrator(t, cfg, runPhase, runPhase) if err := orchPhase.TestHookEnsureVaultUnsealed(context.Background()); err == nil { t.Fatalf("expected ensureVaultUnsealed phase guard error") } if err := orchPhase.TestHookWriteVaultUnsealKeyFile("x"); err == nil { t.Fatalf("expected writeVaultUnsealKeyFile empty-path error") } if _, err := orchPhase.TestHookWorkloadReady(context.Background(), "vault", "statefulset", "vault"); err == nil { t.Fatalf("expected workloadReady parse/lookup failure branch") } cfgCancel := lifecycleConfig(t) runCancel := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "-n vault get statefulset vault -o jsonpath={.status.readyReplicas}"): return "0", nil case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"): return "", errors.New("transient phase query error") default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orchCancel, _ := newHookOrchestrator(t, cfgCancel, runCancel, runCancel) cancelCtx, cancel := context.WithCancel(context.Background()) cancel() if err := orchCancel.TestHookWaitVaultReady(cancelCtx, "vault", "statefulset", "vault"); !errors.Is(err, context.Canceled) { t.Fatalf("expected canceled waitVaultReady branch, got %v", err) } }) }