223 lines
10 KiB
Go
223 lines
10 KiB
Go
|
package orchestrator
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"os"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"scm.bstein.dev/bstein/ananke/internal/cluster"
|
||
|
|
)
|
||
|
|
|
||
|
|
// readStartupProgress runs one orchestration or CLI step.
|
||
|
|
// Signature: readStartupProgress(t *testing.T, orch *cluster.Orchestrator) string.
|
||
|
|
// Why: startup helper tests need to inspect progress artifacts without reaching
|
||
|
|
// into internal package state from the top-level testing module.
|
||
|
|
func readStartupProgress(t *testing.T, orch *cluster.Orchestrator) string {
|
||
|
|
t.Helper()
|
||
|
|
payload, err := os.ReadFile(orch.TestHookStartupProgressPath())
|
||
|
|
if err != nil {
|
||
|
|
t.Fatalf("read startup progress: %v", err)
|
||
|
|
}
|
||
|
|
return string(payload)
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestHookStartupScopeAndVaultHelpers runs one orchestration or CLI step.
|
||
|
|
// Signature: TestHookStartupScopeAndVaultHelpers(t *testing.T).
|
||
|
|
// Why: keeps startup-scope and startup-Vault helper branches covered from the
|
||
|
|
// split top-level testing module required by the repo hygiene contract.
|
||
|
|
func TestHookStartupScopeAndVaultHelpers(t *testing.T) {
|
||
|
|
t.Run("startup-scope-helpers", func(t *testing.T) {
|
||
|
|
nodes := []string{"titan-db", " titan-23 ", "", "titan-24"}
|
||
|
|
if got := cluster.TestHookStartupRequiredNodes(nodes, nil); len(got) != len(nodes) {
|
||
|
|
t.Fatalf("expected passthrough node list, got %v", got)
|
||
|
|
}
|
||
|
|
got := cluster.TestHookStartupRequiredNodes(nodes, []string{"titan-db", "titan-24"})
|
||
|
|
if len(got) != 2 || got[0] != "titan-db" || got[1] != "titan-24" {
|
||
|
|
t.Fatalf("unexpected filtered node list: %v", got)
|
||
|
|
}
|
||
|
|
|
||
|
|
if !cluster.TestHookContainsNode([]string{"titan-db", " titan-23 "}, " titan-23 ") {
|
||
|
|
t.Fatalf("expected trimmed node membership match")
|
||
|
|
}
|
||
|
|
if cluster.TestHookContainsNode([]string{"titan-db"}, " ") {
|
||
|
|
t.Fatalf("expected blank node probe to be ignored")
|
||
|
|
}
|
||
|
|
|
||
|
|
cfg := lifecycleConfig(t)
|
||
|
|
orch, _ := newHookOrchestrator(t, cfg, nil, nil)
|
||
|
|
if !orch.TestHookStartupNodeStrictlyRequired("titan-23") {
|
||
|
|
t.Fatalf("expected all nodes to be strict when no recovery scopes are configured")
|
||
|
|
}
|
||
|
|
|
||
|
|
cfgScoped := lifecycleConfig(t)
|
||
|
|
cfgScoped.Startup.NodeInventoryReachRequiredNodes = []string{"titan-23"}
|
||
|
|
cfgScoped.Startup.NodeSSHAuthRequiredNodes = []string{"titan-24"}
|
||
|
|
cfgScoped.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/core", " flux-system/gitea "}
|
||
|
|
cfgScoped.Startup.WorkloadConvergenceRequiredNamespaces = []string{"vault", " monitoring "}
|
||
|
|
orchScoped, _ := newHookOrchestrator(t, cfgScoped, nil, nil)
|
||
|
|
if !orchScoped.TestHookStartupNodeStrictlyRequired("titan-db") {
|
||
|
|
t.Fatalf("expected control plane to remain strict")
|
||
|
|
}
|
||
|
|
if !orchScoped.TestHookStartupNodeStrictlyRequired("titan-23") {
|
||
|
|
t.Fatalf("expected inventory-scoped node to remain strict")
|
||
|
|
}
|
||
|
|
if !orchScoped.TestHookStartupNodeStrictlyRequired("titan-24") {
|
||
|
|
t.Fatalf("expected ssh-scoped node to remain strict")
|
||
|
|
}
|
||
|
|
if orchScoped.TestHookStartupNodeStrictlyRequired("titan-25") {
|
||
|
|
t.Fatalf("expected non-core worker to stop being strict")
|
||
|
|
}
|
||
|
|
|
||
|
|
flux := orchScoped.TestHookStartupRequiredFluxKustomizations()
|
||
|
|
if _, ok := flux["flux-system/core"]; !ok {
|
||
|
|
t.Fatalf("expected core flux kustomization in required set: %v", flux)
|
||
|
|
}
|
||
|
|
if _, ok := flux["flux-system/gitea"]; !ok {
|
||
|
|
t.Fatalf("expected trimmed gitea kustomization in required set: %v", flux)
|
||
|
|
}
|
||
|
|
|
||
|
|
namespaces := orchScoped.TestHookStartupRequiredWorkloadNamespaces()
|
||
|
|
if _, ok := namespaces["vault"]; !ok {
|
||
|
|
t.Fatalf("expected vault namespace in required set: %v", namespaces)
|
||
|
|
}
|
||
|
|
if _, ok := namespaces["monitoring"]; !ok {
|
||
|
|
t.Fatalf("expected trimmed monitoring namespace in required set: %v", namespaces)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
t.Run("startup-vault-helpers", func(t *testing.T) {
|
||
|
|
t.Run("early-vault-unseal-paths", func(t *testing.T) {
|
||
|
|
cfgAPI := lifecycleConfig(t)
|
||
|
|
runAPI := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "kubectl" && strings.Contains(command, "version --request-timeout=5s") {
|
||
|
|
return "", errors.New("api down")
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orchAPI, _ := newHookOrchestrator(t, cfgAPI, runAPI, runAPI)
|
||
|
|
orchAPI.TestHookBeginStartupReport("startup-vault")
|
||
|
|
orchAPI.TestHookMaybeRunEarlyVaultUnseal(context.Background())
|
||
|
|
if payload := readStartupProgress(t, orchAPI); strings.Contains(payload, "vault-unseal-early") {
|
||
|
|
t.Fatalf("expected no early vault check when api is unavailable, payload=%s", payload)
|
||
|
|
}
|
||
|
|
|
||
|
|
cfgErr := lifecycleConfig(t)
|
||
|
|
runErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
switch {
|
||
|
|
case name == "kubectl" && strings.Contains(command, "version --request-timeout=5s"):
|
||
|
|
return "v1.31.0", nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
|
||
|
|
return "", errors.New("phase probe failed")
|
||
|
|
default:
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orchErr, _ := newHookOrchestrator(t, cfgErr, runErr, runErr)
|
||
|
|
orchErr.TestHookBeginStartupReport("startup-vault")
|
||
|
|
orchErr.TestHookMaybeRunEarlyVaultUnseal(context.Background())
|
||
|
|
if payload := readStartupProgress(t, orchErr); !strings.Contains(payload, "deferred early vault unseal") {
|
||
|
|
t.Fatalf("expected early vault auto-heal detail, payload=%s", payload)
|
||
|
|
}
|
||
|
|
|
||
|
|
cfgDeferred := lifecycleConfig(t)
|
||
|
|
runDeferred := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
switch {
|
||
|
|
case name == "kubectl" && strings.Contains(command, "version --request-timeout=5s"):
|
||
|
|
return "v1.31.0", nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
|
||
|
|
return "Pending", nil
|
||
|
|
default:
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orchDeferred, _ := newHookOrchestrator(t, cfgDeferred, runDeferred, runDeferred)
|
||
|
|
orchDeferred.TestHookBeginStartupReport("startup-vault")
|
||
|
|
orchDeferred.TestHookMaybeRunEarlyVaultUnseal(context.Background())
|
||
|
|
if payload := readStartupProgress(t, orchDeferred); !strings.Contains(payload, `vault-0 pod phase is \"Pending\"`) {
|
||
|
|
t.Fatalf("expected deferred early vault detail, payload=%s", payload)
|
||
|
|
}
|
||
|
|
|
||
|
|
cfgSuccess := lifecycleConfig(t)
|
||
|
|
runSuccess := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
switch {
|
||
|
|
case name == "kubectl" && strings.Contains(command, "version --request-timeout=5s"):
|
||
|
|
return "v1.31.0", nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
|
||
|
|
return "Running", nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "vault status -format=json"):
|
||
|
|
return `{"sealed":false,"initialized":true}`, nil
|
||
|
|
default:
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orchSuccess, _ := newHookOrchestrator(t, cfgSuccess, runSuccess, runSuccess)
|
||
|
|
orchSuccess.TestHookBeginStartupReport("startup-vault")
|
||
|
|
orchSuccess.TestHookMaybeRunEarlyVaultUnseal(context.Background())
|
||
|
|
if payload := readStartupProgress(t, orchSuccess); !strings.Contains(payload, `"vault-unseal-early"`) || !strings.Contains(payload, `"passed"`) {
|
||
|
|
t.Fatalf("expected successful early vault check, payload=%s", payload)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
t.Run("startup-vault-gate-paths", func(t *testing.T) {
|
||
|
|
cfgErr := lifecycleConfig(t)
|
||
|
|
runErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}") {
|
||
|
|
return "", errors.New("phase probe failed")
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orchErr, _ := newHookOrchestrator(t, cfgErr, runErr, runErr)
|
||
|
|
orchErr.TestHookBeginStartupReport("startup-vault")
|
||
|
|
if err := orchErr.TestHookRunStartupVaultUnsealGate(context.Background()); err == nil || !strings.Contains(err.Error(), "phase probe failed") {
|
||
|
|
t.Fatalf("expected startup vault gate error, got %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
cfgDeferred := lifecycleConfig(t)
|
||
|
|
runDeferred := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}") {
|
||
|
|
return "Pending", nil
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orchDeferred, _ := newHookOrchestrator(t, cfgDeferred, runDeferred, runDeferred)
|
||
|
|
orchDeferred.TestHookBeginStartupReport("startup-vault")
|
||
|
|
if err := orchDeferred.TestHookRunStartupVaultUnsealGate(context.Background()); err != nil {
|
||
|
|
t.Fatalf("expected deferred startup vault gate to succeed, got %v", err)
|
||
|
|
}
|
||
|
|
if payload := readStartupProgress(t, orchDeferred); !strings.Contains(payload, `vault-0 pod phase is \"Pending\"`) {
|
||
|
|
t.Fatalf("expected deferred startup vault detail, payload=%s", payload)
|
||
|
|
}
|
||
|
|
|
||
|
|
cfgSuccess := lifecycleConfig(t)
|
||
|
|
runSuccess := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
switch {
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
|
||
|
|
return "Running", nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "vault status -format=json"):
|
||
|
|
return `{"sealed":false,"initialized":true}`, nil
|
||
|
|
default:
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orchSuccess, _ := newHookOrchestrator(t, cfgSuccess, runSuccess, runSuccess)
|
||
|
|
orchSuccess.TestHookBeginStartupReport("startup-vault")
|
||
|
|
if err := orchSuccess.TestHookRunStartupVaultUnsealGate(context.Background()); err != nil {
|
||
|
|
t.Fatalf("expected successful startup vault gate, got %v", err)
|
||
|
|
}
|
||
|
|
if payload := readStartupProgress(t, orchSuccess); !strings.Contains(payload, `"vault is unsealed"`) {
|
||
|
|
t.Fatalf("expected successful startup vault detail, payload=%s", payload)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
})
|
||
|
|
}
|