package orchestrator import ( "context" "encoding/base64" "errors" "os" "path/filepath" "strings" "testing" "time" "scm.bstein.dev/bstein/ananke/internal/cluster" ) // TestHookVaultParserAndClassifierBranches runs one orchestration or CLI step. // Signature: TestHookVaultParserAndClassifierBranches(t *testing.T). // Why: closes parser/classifier helper branches that gate vault recovery decisions. func TestHookVaultParserAndClassifierBranches(t *testing.T) { if _, err := cluster.TestHookParseVaultSealed(""); err == nil { t.Fatalf("expected empty vault payload parse error") } if _, err := cluster.TestHookParseVaultSealed("not-json"); err == nil { t.Fatalf("expected missing JSON object parse error") } sealed, err := cluster.TestHookParseVaultSealed("prefix {\"sealed\":true} suffix") if err != nil || !sealed { t.Fatalf("expected sealed=true parse success, got sealed=%v err=%v", sealed, err) } if !cluster.TestHookIsNotFoundErr("Error from server (NotFound): pods \"vault-0\" not found") { t.Fatalf("expected NotFound classifier to match") } if cluster.TestHookIsNotFoundErr("permission denied") { t.Fatalf("expected non-not-found classifier miss") } } // TestHookVaultWorkloadAndCleanupBranches runs one orchestration or CLI step. // Signature: TestHookVaultWorkloadAndCleanupBranches(t *testing.T). // Why: validates workload readiness and stale-pod cleanup branches that matter // during repeated post-outage startup attempts. func TestHookVaultWorkloadAndCleanupBranches(t *testing.T) { cfg := lifecycleConfig(t) run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "-n vault get statefulset vault -o jsonpath={.status.readyReplicas}"): return "1", nil case name == "kubectl" && strings.Contains(command, "-n vault get deployment missing -o jsonpath={.status.readyReplicas}"): return "", errors.New("Error from server (NotFound): deployments.apps \"missing\" not found") case name == "kubectl" && strings.Contains(command, "-n vault get statefulset parse-bad -o jsonpath={.status.readyReplicas}"): return "not-a-number", nil case name == "kubectl" && strings.Contains(command, "-n vault get pods -o custom-columns="): return "vault-0 Failed StatefulSet vault\nvault-other Running StatefulSet vault\n", nil case name == "kubectl" && strings.Contains(command, "-n vault delete pod vault-0"): return "", nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orch, _ := newHookOrchestrator(t, cfg, run, run) ready, err := orch.TestHookWorkloadReady(context.Background(), "vault", "statefulset", "vault") if err != nil || !ready { t.Fatalf("expected workloadReady true path, got ready=%v err=%v", ready, err) } if _, err := orch.TestHookWorkloadReady(context.Background(), "vault", "deployment", "missing"); err == nil { t.Fatalf("expected workloadReady error propagation branch") } if _, err := orch.TestHookWorkloadReady(context.Background(), "vault", "statefulset", "parse-bad"); err == nil { t.Fatalf("expected workloadReady parse error branch") } if err := orch.TestHookCleanupStaleCriticalWorkloadPods(context.Background(), "vault", "statefulset", "vault"); err != nil { t.Fatalf("expected stale critical pod cleanup success, got %v", err) } if err := orch.TestHookCleanupStaleCriticalWorkloadPods(context.Background(), "vault", "deployment", "vault"); err != nil { t.Fatalf("expected non-statefulset stale cleanup no-op, got %v", err) } } // TestHookVaultKeyResolutionBranches runs one orchestration or CLI step. // Signature: TestHookVaultKeyResolutionBranches(t *testing.T). // Why: ensures all vault unseal key fallback paths remain deterministic. func TestHookVaultKeyResolutionBranches(t *testing.T) { cfg := lifecycleConfig(t) cfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "vault", "unseal.key") cfg.Startup.VaultUnsealBreakglassCommand = "printf breakglass-key" cfg.Startup.VaultUnsealBreakglassTimeout = 1 validKey := base64.StdEncoding.EncodeToString([]byte("secret-key")) run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "-n vault get secret vault-init -o jsonpath={.data.unseal_key_b64}"): return validKey, nil case name == "sh" && strings.Contains(command, "printf breakglass-key"): return "breakglass-key", nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orch, _ := newHookOrchestrator(t, cfg, run, run) key, err := orch.TestHookVaultUnsealKey(context.Background()) if err != nil || key != "secret-key" { t.Fatalf("expected primary vault-init key path, got key=%q err=%v", key, err) } if cached, err := orch.TestHookReadVaultUnsealKeyFile(); err != nil || cached == "" { t.Fatalf("expected cached unseal key file after primary path, got key=%q err=%v", cached, err) } decodeFailRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "-n vault get secret vault-init -o jsonpath={.data.unseal_key_b64}"): return "%%%not-base64%%%", nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orchDecodeFallback, _ := newHookOrchestrator(t, cfg, decodeFailRun, decodeFailRun) if key, err := orchDecodeFallback.TestHookVaultUnsealKey(context.Background()); err != nil || key == "" { t.Fatalf("expected file fallback after decode error, got key=%q err=%v", key, err) } breakglassOnlyCfg := lifecycleConfig(t) breakglassOnlyCfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "missing", "unseal.key") breakglassOnlyCfg.Startup.VaultUnsealBreakglassCommand = "printf breakglass-key" breakglassOnlyCfg.Startup.VaultUnsealBreakglassTimeout = 1 breakglassRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "-n vault get secret vault-init -o jsonpath={.data.unseal_key_b64}"): return "", errors.New("Error from server (NotFound): secrets \"vault-init\" not found") case name == "sh" && strings.Contains(command, "printf breakglass-key"): return "breakglass-key", nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orchBreakglass, _ := newHookOrchestrator(t, breakglassOnlyCfg, breakglassRun, breakglassRun) if key, err := orchBreakglass.TestHookVaultUnsealKey(context.Background()); err != nil || key != "breakglass-key" { t.Fatalf("expected breakglass fallback key, got key=%q err=%v", key, err) } } // TestHookVaultBreakglassAndFileErrorBranches runs one orchestration or CLI step. // Signature: TestHookVaultBreakglassAndFileErrorBranches(t *testing.T). // Why: covers local file/breakglass error paths used when secrets are unavailable. func TestHookVaultBreakglassAndFileErrorBranches(t *testing.T) { cfg := lifecycleConfig(t) cfg.Startup.VaultUnsealKeyFile = "" orchEmptyPath, _ := newHookOrchestrator(t, cfg, nil, nil) if err := orchEmptyPath.TestHookWriteVaultUnsealKeyFile("x"); err == nil { t.Fatalf("expected empty key-file path write error") } if _, err := orchEmptyPath.TestHookReadVaultUnsealKeyFile(); err == nil { t.Fatalf("expected empty key-file path read error") } cfg = lifecycleConfig(t) cfg.Startup.VaultUnsealBreakglassCommand = "" orchNoBreakglass, _ := newHookOrchestrator(t, cfg, nil, nil) if _, err := orchNoBreakglass.TestHookReadVaultUnsealKeyBreakglass(context.Background()); err == nil { t.Fatalf("expected breakglass-not-configured error") } cfg.Startup.VaultUnsealBreakglassCommand = "printf ''" breakglassEmptyRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") if name == "sh" && strings.Contains(command, "printf ''") { return "", nil } return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } orchBreakglassEmpty, _ := newHookOrchestrator(t, cfg, breakglassEmptyRun, breakglassEmptyRun) if _, err := orchBreakglassEmpty.TestHookReadVaultUnsealKeyBreakglass(context.Background()); err == nil { t.Fatalf("expected breakglass empty-output error") } cfg.Startup.VaultUnsealBreakglassCommand = "exit 1" breakglassErrRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") if name == "sh" && strings.Contains(command, "exit 1") { return "", errors.New("command failed") } return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } orchBreakglassErr, _ := newHookOrchestrator(t, cfg, breakglassErrRun, breakglassErrRun) if _, err := orchBreakglassErr.TestHookReadVaultUnsealKeyBreakglass(context.Background()); err == nil { t.Fatalf("expected breakglass command error") } cfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "vault", "unseal.key") orchFile, _ := newHookOrchestrator(t, cfg, nil, nil) if err := orchFile.TestHookWriteVaultUnsealKeyFile("from-file"); err != nil { t.Fatalf("expected file write success, got %v", err) } if got, err := orchFile.TestHookReadVaultUnsealKeyFile(); err != nil || got != "from-file" { t.Fatalf("expected file read success, got key=%q err=%v", got, err) } if err := os.WriteFile(cfg.Startup.VaultUnsealKeyFile, []byte(" \n"), 0o600); err != nil { t.Fatalf("write empty key file fixture: %v", err) } if _, err := orchFile.TestHookReadVaultUnsealKeyFile(); err == nil { t.Fatalf("expected empty key-file content error") } } // TestHookVaultSealedAndUnsealBranches runs one orchestration or CLI step. // Signature: TestHookVaultSealedAndUnsealBranches(t *testing.T). // Why: covers sealed-state checks and unseal flow branches that can block startup. func TestHookVaultSealedAndUnsealBranches(t *testing.T) { cfg := lifecycleConfig(t) cfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "vault", "unseal.key") phaseErrRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") if name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}") { return "", errors.New("phase check failed") } return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } orchPhaseErr, _ := newHookOrchestrator(t, cfg, phaseErrRun, phaseErrRun) if err := orchPhaseErr.TestHookEnsureVaultUnsealed(context.Background()); err == nil { t.Fatalf("expected vault phase error branch") } notRunningRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") if name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}") { return "Pending", nil } return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } orchNotRunning, _ := newHookOrchestrator(t, cfg, notRunningRun, notRunningRun) if err := orchNotRunning.TestHookEnsureVaultUnsealed(context.Background()); err == nil { t.Fatalf("expected non-running vault pod branch") } sealedFalseRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"): return "Running", nil case name == "kubectl" && strings.Contains(command, "vault status -format=json"): return `{"sealed":false}`, nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orchSealedFalse, _ := newHookOrchestrator(t, cfg, sealedFalseRun, sealedFalseRun) if err := orchSealedFalse.TestHookEnsureVaultUnsealed(context.Background()); err != nil { t.Fatalf("expected already-unsealed branch, got %v", err) } unsealed := false sealedTrueRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"): return "Running", nil case name == "kubectl" && strings.Contains(command, "vault status -format=json"): if unsealed { return `{"sealed":false}`, nil } return `{"sealed":true}`, nil case name == "kubectl" && strings.Contains(command, "-n vault get secret vault-init -o jsonpath={.data.unseal_key_b64}"): return base64.StdEncoding.EncodeToString([]byte("secret-key")), nil case name == "kubectl" && strings.Contains(command, "vault operator unseal secret-key"): unsealed = true return "", nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orchSealedTrue, _ := newHookOrchestrator(t, cfg, sealedTrueRun, sealedTrueRun) if err := orchSealedTrue.TestHookEnsureVaultUnsealed(context.Background()); err != nil { t.Fatalf("expected auto-unseal success path, got %v", err) } } // TestHookVaultWrapperSurfaceCoverage runs one orchestration or CLI step. // Signature: TestHookVaultWrapperSurfaceCoverage(t *testing.T). // Why: ensures newly-exposed vault test hooks stay directly exercised so wrapper // coverage does not regress below quality-gate thresholds. func TestHookVaultWrapperSurfaceCoverage(t *testing.T) { cfg := lifecycleConfig(t) cfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "vault", "unseal.key") run := lifecycleDispatcher(&commandRecorder{}) orch, _ := newHookOrchestrator(t, cfg, run, run) _ = orch.TestHookEnsureWorkloadReplicas(context.Background(), "vault", "statefulset", "vault", 1) _ = orch.TestHookWaitWorkloadReady(context.Background(), "vault", "deployment", "grafana") _ = orch.TestHookWaitVaultReady(context.Background(), "vault", "statefulset", "vault") _, _ = orch.TestHookVaultSealed(context.Background()) }