package orchestrator import ( "context" "errors" "os" "strings" "testing" "time" "scm.bstein.dev/bstein/ananke/internal/cluster" ) // readStartupProgress runs one orchestration or CLI step. // Signature: readStartupProgress(t *testing.T, orch *cluster.Orchestrator) string. // Why: startup helper tests need to inspect progress artifacts without reaching // into internal package state from the top-level testing module. func readStartupProgress(t *testing.T, orch *cluster.Orchestrator) string { t.Helper() payload, err := os.ReadFile(orch.TestHookStartupProgressPath()) if err != nil { t.Fatalf("read startup progress: %v", err) } return string(payload) } // TestHookStartupScopeAndVaultHelpers runs one orchestration or CLI step. // Signature: TestHookStartupScopeAndVaultHelpers(t *testing.T). // Why: keeps startup-scope and startup-Vault helper branches covered from the // split top-level testing module required by the repo hygiene contract. func TestHookStartupScopeAndVaultHelpers(t *testing.T) { t.Run("startup-scope-helpers", func(t *testing.T) { nodes := []string{"titan-db", " titan-23 ", "", "titan-24"} if got := cluster.TestHookStartupRequiredNodes(nodes, nil); len(got) != len(nodes) { t.Fatalf("expected passthrough node list, got %v", got) } got := cluster.TestHookStartupRequiredNodes(nodes, []string{"titan-db", "titan-24"}) if len(got) != 2 || got[0] != "titan-db" || got[1] != "titan-24" { t.Fatalf("unexpected filtered node list: %v", got) } if !cluster.TestHookContainsNode([]string{"titan-db", " titan-23 "}, " titan-23 ") { t.Fatalf("expected trimmed node membership match") } if cluster.TestHookContainsNode([]string{"titan-db"}, " ") { t.Fatalf("expected blank node probe to be ignored") } cfg := lifecycleConfig(t) orch, _ := newHookOrchestrator(t, cfg, nil, nil) if !orch.TestHookStartupNodeStrictlyRequired("titan-23") { t.Fatalf("expected all nodes to be strict when no recovery scopes are configured") } cfgScoped := lifecycleConfig(t) cfgScoped.Startup.NodeInventoryReachRequiredNodes = []string{"titan-23"} cfgScoped.Startup.NodeSSHAuthRequiredNodes = []string{"titan-24"} cfgScoped.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/core", " flux-system/gitea "} cfgScoped.Startup.WorkloadConvergenceRequiredNamespaces = []string{"vault", " monitoring "} orchScoped, _ := newHookOrchestrator(t, cfgScoped, nil, nil) if !orchScoped.TestHookStartupNodeStrictlyRequired("titan-db") { t.Fatalf("expected control plane to remain strict") } if !orchScoped.TestHookStartupNodeStrictlyRequired("titan-23") { t.Fatalf("expected inventory-scoped node to remain strict") } if !orchScoped.TestHookStartupNodeStrictlyRequired("titan-24") { t.Fatalf("expected ssh-scoped node to remain strict") } if orchScoped.TestHookStartupNodeStrictlyRequired("titan-25") { t.Fatalf("expected non-core worker to stop being strict") } flux := orchScoped.TestHookStartupRequiredFluxKustomizations() if _, ok := flux["flux-system/core"]; !ok { t.Fatalf("expected core flux kustomization in required set: %v", flux) } if _, ok := flux["flux-system/gitea"]; !ok { t.Fatalf("expected trimmed gitea kustomization in required set: %v", flux) } namespaces := orchScoped.TestHookStartupRequiredWorkloadNamespaces() if _, ok := namespaces["vault"]; !ok { t.Fatalf("expected vault namespace in required set: %v", namespaces) } if _, ok := namespaces["monitoring"]; !ok { t.Fatalf("expected trimmed monitoring namespace in required set: %v", namespaces) } }) t.Run("startup-vault-helpers", func(t *testing.T) { t.Run("early-vault-unseal-paths", func(t *testing.T) { cfgAPI := lifecycleConfig(t) runAPI := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") if name == "kubectl" && strings.Contains(command, "version --request-timeout=5s") { return "", errors.New("api down") } return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } orchAPI, _ := newHookOrchestrator(t, cfgAPI, runAPI, runAPI) orchAPI.TestHookBeginStartupReport("startup-vault") orchAPI.TestHookMaybeRunEarlyVaultUnseal(context.Background()) if payload := readStartupProgress(t, orchAPI); strings.Contains(payload, "vault-unseal-early") { t.Fatalf("expected no early vault check when api is unavailable, payload=%s", payload) } cfgErr := lifecycleConfig(t) runErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "version --request-timeout=5s"): return "v1.31.0", nil case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"): return "", errors.New("phase probe failed") default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orchErr, _ := newHookOrchestrator(t, cfgErr, runErr, runErr) orchErr.TestHookBeginStartupReport("startup-vault") orchErr.TestHookMaybeRunEarlyVaultUnseal(context.Background()) if payload := readStartupProgress(t, orchErr); !strings.Contains(payload, "deferred early vault unseal") { t.Fatalf("expected early vault auto-heal detail, payload=%s", payload) } cfgDeferred := lifecycleConfig(t) runDeferred := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "version --request-timeout=5s"): return "v1.31.0", nil case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"): return "Pending", nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orchDeferred, _ := newHookOrchestrator(t, cfgDeferred, runDeferred, runDeferred) orchDeferred.TestHookBeginStartupReport("startup-vault") orchDeferred.TestHookMaybeRunEarlyVaultUnseal(context.Background()) if payload := readStartupProgress(t, orchDeferred); !strings.Contains(payload, `vault-0 pod phase is \"Pending\"`) { t.Fatalf("expected deferred early vault detail, payload=%s", payload) } cfgSuccess := lifecycleConfig(t) runSuccess := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "version --request-timeout=5s"): return "v1.31.0", nil case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"): return "Running", nil case name == "kubectl" && strings.Contains(command, "vault status -format=json"): return `{"sealed":false,"initialized":true}`, nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orchSuccess, _ := newHookOrchestrator(t, cfgSuccess, runSuccess, runSuccess) orchSuccess.TestHookBeginStartupReport("startup-vault") orchSuccess.TestHookMaybeRunEarlyVaultUnseal(context.Background()) if payload := readStartupProgress(t, orchSuccess); !strings.Contains(payload, `"vault-unseal-early"`) || !strings.Contains(payload, `"passed"`) { t.Fatalf("expected successful early vault check, payload=%s", payload) } }) t.Run("startup-vault-gate-paths", func(t *testing.T) { cfgErr := lifecycleConfig(t) runErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") if name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}") { return "", errors.New("phase probe failed") } return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } orchErr, _ := newHookOrchestrator(t, cfgErr, runErr, runErr) orchErr.TestHookBeginStartupReport("startup-vault") if err := orchErr.TestHookRunStartupVaultUnsealGate(context.Background()); err == nil || !strings.Contains(err.Error(), "phase probe failed") { t.Fatalf("expected startup vault gate error, got %v", err) } cfgDeferred := lifecycleConfig(t) runDeferred := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") if name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}") { return "Pending", nil } return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } orchDeferred, _ := newHookOrchestrator(t, cfgDeferred, runDeferred, runDeferred) orchDeferred.TestHookBeginStartupReport("startup-vault") if err := orchDeferred.TestHookRunStartupVaultUnsealGate(context.Background()); err != nil { t.Fatalf("expected deferred startup vault gate to succeed, got %v", err) } if payload := readStartupProgress(t, orchDeferred); !strings.Contains(payload, `vault-0 pod phase is \"Pending\"`) { t.Fatalf("expected deferred startup vault detail, payload=%s", payload) } cfgSuccess := lifecycleConfig(t) runSuccess := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"): return "Running", nil case name == "kubectl" && strings.Contains(command, "vault status -format=json"): return `{"sealed":false,"initialized":true}`, nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orchSuccess, _ := newHookOrchestrator(t, cfgSuccess, runSuccess, runSuccess) orchSuccess.TestHookBeginStartupReport("startup-vault") if err := orchSuccess.TestHookRunStartupVaultUnsealGate(context.Background()); err != nil { t.Fatalf("expected successful startup vault gate, got %v", err) } if payload := readStartupProgress(t, orchSuccess); !strings.Contains(payload, `"vault is unsealed"`) { t.Fatalf("expected successful startup vault detail, payload=%s", payload) } }) }) }