ananke/testing/orchestrator/hooks_vault_matrix_extra_test.go

package orchestrator

import (
	"context"
	"errors"
	"strings"
	"testing"
	"time"

	"scm.bstein.dev/bstein/ananke/internal/cluster"
)

// TestHookCriticalVaultMatrixExtra runs one orchestration or CLI step.
// Signature: TestHookCriticalVaultMatrixExtra(t *testing.T).
// Why: drives deeper vault/critical-workload recovery branches so sealed/unready
// states are exercised before the next power-cycle drill.
func TestHookCriticalVaultMatrixExtra(t *testing.T) {
	t.Run("missing-and-ensure-critical-workloads", func(t *testing.T) {
		cfg := lifecycleConfig(t)
		run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "get deployment source-controller"):
				return "", errors.New("Error from server (NotFound): deployments.apps \"source-controller\" not found")
			case name == "kubectl" && strings.Contains(command, "get deployment kustomize-controller"):
				return "0", nil
			case name == "kubectl" && strings.Contains(command, "get") && strings.Contains(command, "jsonpath={.status.readyReplicas}"):
				return "1", nil
			case name == "kubectl" && strings.Contains(command, " scale deployment source-controller --replicas=1"):
				return "", errors.New("Error from server (NotFound): deployments.apps \"source-controller\" not found")
			case name == "kubectl" && strings.Contains(command, " scale statefulset vault --replicas=1"):
				return "", nil
			case name == "kubectl" && strings.Contains(command, " -n vault get pods -o custom-columns="):
				return "", errors.New("Error from server (NotFound): pods not found")
			case name == "kubectl" && strings.Contains(command, " scale deployment gitea --replicas=1"):
				return "", nil
			case name == "kubectl" && strings.Contains(command, "rollout status deployment/gitea"):
				return "", errors.New("Error from server (NotFound): deployments.apps \"gitea\" not found")
			case name == "kubectl" && strings.Contains(command, "rollout status"):
				return "rolled out", nil
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orch, _ := newHookOrchestrator(t, cfg, run, run)
		missing, err := orch.TestHookMissingCriticalStartupWorkloads(context.Background())
		if err != nil || len(missing) == 0 {
			t.Fatalf("expected missing critical workload list, missing=%v err=%v", missing, err)
		}
		if err := orch.TestHookEnsureCriticalStartupWorkloads(context.Background()); err != nil {
			t.Fatalf("expected ensureCriticalStartupWorkloads notfound branches to continue, got %v", err)
		}
	})

	t.Run("vault-ready-unsealed-and-key-branches", func(t *testing.T) {
		cfg := lifecycleConfig(t)
		cfg.Startup.VaultUnsealBreakglassCommand = "echo breakglass-key"
		readyCalls := 0
		run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "-n vault get statefulset vault -o jsonpath={.status.readyReplicas}"):
				readyCalls++
				if readyCalls < 2 {
					return "0", nil
				}
				return "1", nil
			case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
				return "Running", nil
			case name == "kubectl" && strings.Contains(command, "vault status -format=json"):
				if readyCalls < 2 {
					return `{"sealed":true}`, nil
				}
				return `{"sealed":false}`, nil
			case name == "kubectl" && strings.Contains(command, "get secret vault-init"):
				return "", errors.New("secret missing")
			case name == "sh" && strings.Contains(command, "echo breakglass-key"):
				return "breakglass-key", nil
			case name == "kubectl" && strings.Contains(command, "vault operator unseal"):
				return "unsealed", nil
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orch, _ := newHookOrchestrator(t, cfg, run, run)
		if err := orch.TestHookWaitVaultReady(context.Background(), "vault", "statefulset", "vault"); err != nil {
			t.Fatalf("expected waitVaultReady unseal branch success, got %v", err)
		}

		if sealed, err := orch.TestHookVaultSealed(context.Background()); err != nil || sealed {
			t.Fatalf("expected vaultSealed false after unseal, sealed=%v err=%v", sealed, err)
		}
		key, err := orch.TestHookVaultUnsealKey(context.Background())
		if err != nil || strings.TrimSpace(key) == "" {
			t.Fatalf("expected breakglass vault key fallback, key=%q err=%v", key, err)
		}

		if _, err := cluster.TestHookParseVaultSealed("not-json"); err == nil {
			t.Fatalf("expected parseVaultSealed error for malformed payload")
		}
		if !cluster.TestHookIsNotFoundErr("Error from server (NotFound)") {
			t.Fatalf("expected notfound helper to detect notfound text")
		}
	})

	t.Run("vault-write-workload-and-phase-errors", func(t *testing.T) {
		cfg := lifecycleConfig(t)
		cfg.Startup.VaultUnsealKeyFile = ""
		runPhase := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
				return "Pending", nil
			case name == "kubectl" && strings.Contains(command, "-n vault get statefulset vault -o jsonpath={.status.readyReplicas}"):
				return "not-a-number", nil
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchPhase, _ := newHookOrchestrator(t, cfg, runPhase, runPhase)
		if err := orchPhase.TestHookEnsureVaultUnsealed(context.Background()); err == nil {
			t.Fatalf("expected ensureVaultUnsealed phase guard error")
		}
		if err := orchPhase.TestHookWriteVaultUnsealKeyFile("x"); err == nil {
			t.Fatalf("expected writeVaultUnsealKeyFile empty-path error")
		}
		if _, err := orchPhase.TestHookWorkloadReady(context.Background(), "vault", "statefulset", "vault"); err == nil {
			t.Fatalf("expected workloadReady parse/lookup failure branch")
		}

		cfgCancel := lifecycleConfig(t)
		runCancel := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "-n vault get statefulset vault -o jsonpath={.status.readyReplicas}"):
				return "0", nil
			case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
				return "", errors.New("transient phase query error")
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchCancel, _ := newHookOrchestrator(t, cfgCancel, runCancel, runCancel)
		cancelCtx, cancel := context.WithCancel(context.Background())
		cancel()
		if err := orchCancel.TestHookWaitVaultReady(cancelCtx, "vault", "statefulset", "vault"); !errors.Is(err, context.Canceled) {
			t.Fatalf("expected canceled waitVaultReady branch, got %v", err)
		}
	})
}