298 lines
14 KiB
Go
298 lines
14 KiB
Go
|
package orchestrator
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"encoding/base64"
|
||
|
|
"errors"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"scm.bstein.dev/bstein/ananke/internal/cluster"
|
||
|
|
)
|
||
|
|
|
||
|
|
// TestHookVaultParserAndClassifierBranches runs one orchestration or CLI step.
|
||
|
|
// Signature: TestHookVaultParserAndClassifierBranches(t *testing.T).
|
||
|
|
// Why: closes parser/classifier helper branches that gate vault recovery decisions.
|
||
|
|
func TestHookVaultParserAndClassifierBranches(t *testing.T) {
|
||
|
|
if _, err := cluster.TestHookParseVaultSealed(""); err == nil {
|
||
|
|
t.Fatalf("expected empty vault payload parse error")
|
||
|
|
}
|
||
|
|
if _, err := cluster.TestHookParseVaultSealed("not-json"); err == nil {
|
||
|
|
t.Fatalf("expected missing JSON object parse error")
|
||
|
|
}
|
||
|
|
sealed, err := cluster.TestHookParseVaultSealed("prefix {\"sealed\":true} suffix")
|
||
|
|
if err != nil || !sealed {
|
||
|
|
t.Fatalf("expected sealed=true parse success, got sealed=%v err=%v", sealed, err)
|
||
|
|
}
|
||
|
|
if !cluster.TestHookIsNotFoundErr("Error from server (NotFound): pods \"vault-0\" not found") {
|
||
|
|
t.Fatalf("expected NotFound classifier to match")
|
||
|
|
}
|
||
|
|
if cluster.TestHookIsNotFoundErr("permission denied") {
|
||
|
|
t.Fatalf("expected non-not-found classifier miss")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestHookVaultWorkloadAndCleanupBranches runs one orchestration or CLI step.
|
||
|
|
// Signature: TestHookVaultWorkloadAndCleanupBranches(t *testing.T).
|
||
|
|
// Why: validates workload readiness and stale-pod cleanup branches that matter
|
||
|
|
// during repeated post-outage startup attempts.
|
||
|
|
func TestHookVaultWorkloadAndCleanupBranches(t *testing.T) {
|
||
|
|
cfg := lifecycleConfig(t)
|
||
|
|
run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
switch {
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get statefulset vault -o jsonpath={.status.readyReplicas}"):
|
||
|
|
return "1", nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get deployment missing -o jsonpath={.status.readyReplicas}"):
|
||
|
|
return "", errors.New("Error from server (NotFound): deployments.apps \"missing\" not found")
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get statefulset parse-bad -o jsonpath={.status.readyReplicas}"):
|
||
|
|
return "not-a-number", nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get pods -o custom-columns="):
|
||
|
|
return "vault-0 Failed StatefulSet vault\nvault-other Running StatefulSet vault\n", nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault delete pod vault-0"):
|
||
|
|
return "", nil
|
||
|
|
default:
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orch, _ := newHookOrchestrator(t, cfg, run, run)
|
||
|
|
|
||
|
|
ready, err := orch.TestHookWorkloadReady(context.Background(), "vault", "statefulset", "vault")
|
||
|
|
if err != nil || !ready {
|
||
|
|
t.Fatalf("expected workloadReady true path, got ready=%v err=%v", ready, err)
|
||
|
|
}
|
||
|
|
if _, err := orch.TestHookWorkloadReady(context.Background(), "vault", "deployment", "missing"); err == nil {
|
||
|
|
t.Fatalf("expected workloadReady error propagation branch")
|
||
|
|
}
|
||
|
|
if _, err := orch.TestHookWorkloadReady(context.Background(), "vault", "statefulset", "parse-bad"); err == nil {
|
||
|
|
t.Fatalf("expected workloadReady parse error branch")
|
||
|
|
}
|
||
|
|
|
||
|
|
if err := orch.TestHookCleanupStaleCriticalWorkloadPods(context.Background(), "vault", "statefulset", "vault"); err != nil {
|
||
|
|
t.Fatalf("expected stale critical pod cleanup success, got %v", err)
|
||
|
|
}
|
||
|
|
if err := orch.TestHookCleanupStaleCriticalWorkloadPods(context.Background(), "vault", "deployment", "vault"); err != nil {
|
||
|
|
t.Fatalf("expected non-statefulset stale cleanup no-op, got %v", err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestHookVaultKeyResolutionBranches runs one orchestration or CLI step.
|
||
|
|
// Signature: TestHookVaultKeyResolutionBranches(t *testing.T).
|
||
|
|
// Why: ensures all vault unseal key fallback paths remain deterministic.
|
||
|
|
func TestHookVaultKeyResolutionBranches(t *testing.T) {
|
||
|
|
cfg := lifecycleConfig(t)
|
||
|
|
cfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "vault", "unseal.key")
|
||
|
|
cfg.Startup.VaultUnsealBreakglassCommand = "printf breakglass-key"
|
||
|
|
cfg.Startup.VaultUnsealBreakglassTimeout = 1
|
||
|
|
|
||
|
|
validKey := base64.StdEncoding.EncodeToString([]byte("secret-key"))
|
||
|
|
run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
switch {
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get secret vault-init -o jsonpath={.data.unseal_key_b64}"):
|
||
|
|
return validKey, nil
|
||
|
|
case name == "sh" && strings.Contains(command, "printf breakglass-key"):
|
||
|
|
return "breakglass-key", nil
|
||
|
|
default:
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orch, _ := newHookOrchestrator(t, cfg, run, run)
|
||
|
|
key, err := orch.TestHookVaultUnsealKey(context.Background())
|
||
|
|
if err != nil || key != "secret-key" {
|
||
|
|
t.Fatalf("expected primary vault-init key path, got key=%q err=%v", key, err)
|
||
|
|
}
|
||
|
|
if cached, err := orch.TestHookReadVaultUnsealKeyFile(); err != nil || cached == "" {
|
||
|
|
t.Fatalf("expected cached unseal key file after primary path, got key=%q err=%v", cached, err)
|
||
|
|
}
|
||
|
|
|
||
|
|
decodeFailRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
switch {
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get secret vault-init -o jsonpath={.data.unseal_key_b64}"):
|
||
|
|
return "%%%not-base64%%%", nil
|
||
|
|
default:
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orchDecodeFallback, _ := newHookOrchestrator(t, cfg, decodeFailRun, decodeFailRun)
|
||
|
|
if key, err := orchDecodeFallback.TestHookVaultUnsealKey(context.Background()); err != nil || key == "" {
|
||
|
|
t.Fatalf("expected file fallback after decode error, got key=%q err=%v", key, err)
|
||
|
|
}
|
||
|
|
|
||
|
|
breakglassOnlyCfg := lifecycleConfig(t)
|
||
|
|
breakglassOnlyCfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "missing", "unseal.key")
|
||
|
|
breakglassOnlyCfg.Startup.VaultUnsealBreakglassCommand = "printf breakglass-key"
|
||
|
|
breakglassOnlyCfg.Startup.VaultUnsealBreakglassTimeout = 1
|
||
|
|
breakglassRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
switch {
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get secret vault-init -o jsonpath={.data.unseal_key_b64}"):
|
||
|
|
return "", errors.New("Error from server (NotFound): secrets \"vault-init\" not found")
|
||
|
|
case name == "sh" && strings.Contains(command, "printf breakglass-key"):
|
||
|
|
return "breakglass-key", nil
|
||
|
|
default:
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orchBreakglass, _ := newHookOrchestrator(t, breakglassOnlyCfg, breakglassRun, breakglassRun)
|
||
|
|
if key, err := orchBreakglass.TestHookVaultUnsealKey(context.Background()); err != nil || key != "breakglass-key" {
|
||
|
|
t.Fatalf("expected breakglass fallback key, got key=%q err=%v", key, err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestHookVaultBreakglassAndFileErrorBranches runs one orchestration or CLI step.
|
||
|
|
// Signature: TestHookVaultBreakglassAndFileErrorBranches(t *testing.T).
|
||
|
|
// Why: covers local file/breakglass error paths used when secrets are unavailable.
|
||
|
|
func TestHookVaultBreakglassAndFileErrorBranches(t *testing.T) {
|
||
|
|
cfg := lifecycleConfig(t)
|
||
|
|
cfg.Startup.VaultUnsealKeyFile = ""
|
||
|
|
orchEmptyPath, _ := newHookOrchestrator(t, cfg, nil, nil)
|
||
|
|
if err := orchEmptyPath.TestHookWriteVaultUnsealKeyFile("x"); err == nil {
|
||
|
|
t.Fatalf("expected empty key-file path write error")
|
||
|
|
}
|
||
|
|
if _, err := orchEmptyPath.TestHookReadVaultUnsealKeyFile(); err == nil {
|
||
|
|
t.Fatalf("expected empty key-file path read error")
|
||
|
|
}
|
||
|
|
|
||
|
|
cfg = lifecycleConfig(t)
|
||
|
|
cfg.Startup.VaultUnsealBreakglassCommand = ""
|
||
|
|
orchNoBreakglass, _ := newHookOrchestrator(t, cfg, nil, nil)
|
||
|
|
if _, err := orchNoBreakglass.TestHookReadVaultUnsealKeyBreakglass(context.Background()); err == nil {
|
||
|
|
t.Fatalf("expected breakglass-not-configured error")
|
||
|
|
}
|
||
|
|
|
||
|
|
cfg.Startup.VaultUnsealBreakglassCommand = "printf ''"
|
||
|
|
breakglassEmptyRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "sh" && strings.Contains(command, "printf ''") {
|
||
|
|
return "", nil
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orchBreakglassEmpty, _ := newHookOrchestrator(t, cfg, breakglassEmptyRun, breakglassEmptyRun)
|
||
|
|
if _, err := orchBreakglassEmpty.TestHookReadVaultUnsealKeyBreakglass(context.Background()); err == nil {
|
||
|
|
t.Fatalf("expected breakglass empty-output error")
|
||
|
|
}
|
||
|
|
|
||
|
|
cfg.Startup.VaultUnsealBreakglassCommand = "exit 1"
|
||
|
|
breakglassErrRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "sh" && strings.Contains(command, "exit 1") {
|
||
|
|
return "", errors.New("command failed")
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orchBreakglassErr, _ := newHookOrchestrator(t, cfg, breakglassErrRun, breakglassErrRun)
|
||
|
|
if _, err := orchBreakglassErr.TestHookReadVaultUnsealKeyBreakglass(context.Background()); err == nil {
|
||
|
|
t.Fatalf("expected breakglass command error")
|
||
|
|
}
|
||
|
|
|
||
|
|
cfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "vault", "unseal.key")
|
||
|
|
orchFile, _ := newHookOrchestrator(t, cfg, nil, nil)
|
||
|
|
if err := orchFile.TestHookWriteVaultUnsealKeyFile("from-file"); err != nil {
|
||
|
|
t.Fatalf("expected file write success, got %v", err)
|
||
|
|
}
|
||
|
|
if got, err := orchFile.TestHookReadVaultUnsealKeyFile(); err != nil || got != "from-file" {
|
||
|
|
t.Fatalf("expected file read success, got key=%q err=%v", got, err)
|
||
|
|
}
|
||
|
|
if err := os.WriteFile(cfg.Startup.VaultUnsealKeyFile, []byte(" \n"), 0o600); err != nil {
|
||
|
|
t.Fatalf("write empty key file fixture: %v", err)
|
||
|
|
}
|
||
|
|
if _, err := orchFile.TestHookReadVaultUnsealKeyFile(); err == nil {
|
||
|
|
t.Fatalf("expected empty key-file content error")
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestHookVaultSealedAndUnsealBranches runs one orchestration or CLI step.
|
||
|
|
// Signature: TestHookVaultSealedAndUnsealBranches(t *testing.T).
|
||
|
|
// Why: covers sealed-state checks and unseal flow branches that can block startup.
|
||
|
|
func TestHookVaultSealedAndUnsealBranches(t *testing.T) {
|
||
|
|
cfg := lifecycleConfig(t)
|
||
|
|
cfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "vault", "unseal.key")
|
||
|
|
|
||
|
|
phaseErrRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}") {
|
||
|
|
return "", errors.New("phase check failed")
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orchPhaseErr, _ := newHookOrchestrator(t, cfg, phaseErrRun, phaseErrRun)
|
||
|
|
if err := orchPhaseErr.TestHookEnsureVaultUnsealed(context.Background()); err == nil {
|
||
|
|
t.Fatalf("expected vault phase error branch")
|
||
|
|
}
|
||
|
|
|
||
|
|
notRunningRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}") {
|
||
|
|
return "Pending", nil
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orchNotRunning, _ := newHookOrchestrator(t, cfg, notRunningRun, notRunningRun)
|
||
|
|
if err := orchNotRunning.TestHookEnsureVaultUnsealed(context.Background()); err == nil {
|
||
|
|
t.Fatalf("expected non-running vault pod branch")
|
||
|
|
}
|
||
|
|
|
||
|
|
sealedFalseRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
switch {
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
|
||
|
|
return "Running", nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "vault status -format=json"):
|
||
|
|
return `{"sealed":false}`, nil
|
||
|
|
default:
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orchSealedFalse, _ := newHookOrchestrator(t, cfg, sealedFalseRun, sealedFalseRun)
|
||
|
|
if err := orchSealedFalse.TestHookEnsureVaultUnsealed(context.Background()); err != nil {
|
||
|
|
t.Fatalf("expected already-unsealed branch, got %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
unsealed := false
|
||
|
|
sealedTrueRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
switch {
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
|
||
|
|
return "Running", nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "vault status -format=json"):
|
||
|
|
if unsealed {
|
||
|
|
return `{"sealed":false}`, nil
|
||
|
|
}
|
||
|
|
return `{"sealed":true}`, nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "-n vault get secret vault-init -o jsonpath={.data.unseal_key_b64}"):
|
||
|
|
return base64.StdEncoding.EncodeToString([]byte("secret-key")), nil
|
||
|
|
case name == "kubectl" && strings.Contains(command, "vault operator unseal secret-key"):
|
||
|
|
unsealed = true
|
||
|
|
return "", nil
|
||
|
|
default:
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orchSealedTrue, _ := newHookOrchestrator(t, cfg, sealedTrueRun, sealedTrueRun)
|
||
|
|
if err := orchSealedTrue.TestHookEnsureVaultUnsealed(context.Background()); err != nil {
|
||
|
|
t.Fatalf("expected auto-unseal success path, got %v", err)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestHookVaultWrapperSurfaceCoverage runs one orchestration or CLI step.
|
||
|
|
// Signature: TestHookVaultWrapperSurfaceCoverage(t *testing.T).
|
||
|
|
// Why: ensures newly-exposed vault test hooks stay directly exercised so wrapper
|
||
|
|
// coverage does not regress below quality-gate thresholds.
|
||
|
|
func TestHookVaultWrapperSurfaceCoverage(t *testing.T) {
|
||
|
|
cfg := lifecycleConfig(t)
|
||
|
|
cfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "vault", "unseal.key")
|
||
|
|
run := lifecycleDispatcher(&commandRecorder{})
|
||
|
|
orch, _ := newHookOrchestrator(t, cfg, run, run)
|
||
|
|
|
||
|
|
_ = orch.TestHookEnsureWorkloadReplicas(context.Background(), "vault", "statefulset", "vault", 1)
|
||
|
|
_ = orch.TestHookWaitWorkloadReady(context.Background(), "vault", "deployment", "grafana")
|
||
|
|
_ = orch.TestHookWaitVaultReady(context.Background(), "vault", "statefulset", "vault")
|
||
|
|
_, _ = orch.TestHookVaultSealed(context.Background())
|
||
|
|
}
|