ananke/testing/orchestrator/hooks_lowfile_coverage_test.go

package orchestrator

import (
	"context"
	"errors"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"scm.bstein.dev/bstein/ananke/internal/cluster"
	"scm.bstein.dev/bstein/ananke/internal/execx"
	"scm.bstein.dev/bstein/ananke/internal/state"
)

// TestHookLowFileCoverageBoost runs one orchestration or CLI step.
// Signature: TestHookLowFileCoverageBoost(t *testing.T).
// Why: raises the low-coverage orchestrator files through deterministic top-level
// tests that only use the exported hook surface.
func TestHookLowFileCoverageBoost(t *testing.T) {
	t.Run("workload-convergence-helpers-and-gates", func(t *testing.T) {
		desiredCases := []struct {
			kind  string
			has   bool
			rep   int32
			ready int32
			sched int32
			num   int32
			wantD int32
			wantR int32
			wantB bool
		}{
			{kind: "Deployment", has: false, ready: 1, wantD: 1, wantR: 1, wantB: true},
			{kind: "deployment", has: true, rep: 3, ready: 2, wantD: 3, wantR: 2, wantB: true},
			{kind: "daemonset", sched: 4, num: 3, wantD: 4, wantR: 3, wantB: true},
			{kind: "job", wantD: 0, wantR: 0, wantB: false},
		}
		for _, tc := range desiredCases {
			gotD, gotR, gotB := cluster.TestHookDesiredReady(tc.kind, tc.has, tc.rep, tc.ready, tc.sched, tc.num)
			if gotD != tc.wantD || gotR != tc.wantR || gotB != tc.wantB {
				t.Fatalf("desiredReady(%q)=%d,%d,%v want %d,%d,%v", tc.kind, gotD, gotR, gotB, tc.wantD, tc.wantR, tc.wantB)
			}
		}

		if !cluster.TestHookPodControllerOwned([]string{"ReplicaSet"}) {
			t.Fatalf("expected ReplicaSet owner to be controller-owned")
		}
		if !cluster.TestHookPodControllerOwned([]string{"StatefulSet"}) {
			t.Fatalf("expected StatefulSet owner to be controller-owned")
		}
		if !cluster.TestHookPodControllerOwned([]string{"DaemonSet"}) {
			t.Fatalf("expected DaemonSet owner to be controller-owned")
		}
		if !cluster.TestHookPodControllerOwned([]string{"Job"}) {
			t.Fatalf("expected Job owner to be controller-owned")
		}

		if got := cluster.TestHookStuckContainerReason([]string{"ImagePullBackOff"}, nil, []string{"ImagePullBackOff"}); got != "ImagePullBackOff" {
			t.Fatalf("expected init-container stuck reason, got %q", got)
		}
		if got := cluster.TestHookStuckContainerReason(nil, []string{"CrashLoopBackOff"}, []string{"CrashLoopBackOff"}); got != "CrashLoopBackOff" {
			t.Fatalf("expected container stuck reason, got %q", got)
		}
		if got := cluster.TestHookStuckContainerReason([]string{"ImagePullBackOff"}, []string{"CrashLoopBackOff"}, []string{"Missing"}); got != "" {
			t.Fatalf("expected filtered stuck reason to be empty, got %q", got)
		}

		vaultCases := []struct {
			name       string
			phase      string
			inject     bool
			startedAgo time.Duration
			grace      time.Duration
			want       string
		}{
			{name: "phase-running", phase: "Running", inject: true, startedAgo: 10 * time.Minute, grace: time.Minute, want: ""},
			{name: "inject-false", phase: "Pending", inject: false, startedAgo: 10 * time.Minute, grace: time.Minute, want: ""},
			{name: "within-grace", phase: "Pending", inject: true, startedAgo: 30 * time.Second, grace: time.Minute, want: ""},
			{name: "stuck", phase: "Pending", inject: true, startedAgo: 10 * time.Minute, grace: time.Minute, want: "VaultInitStuck"},
		}
		for _, tc := range vaultCases {
			got := cluster.TestHookStuckVaultInitReason(tc.phase, tc.inject, tc.startedAgo, tc.grace)
			if got != tc.want {
				t.Fatalf("%s: stuckVaultInitReason=%q want %q", tc.name, got, tc.want)
			}
		}

		cfg := lifecycleConfig(t)
		cfg.Startup.WorkloadConvergenceWaitSeconds = 1
		cfg.Startup.WorkloadConvergencePollSeconds = 1
		cfg.Startup.StuckPodGraceSeconds = 1
		cfg.Startup.IgnoreWorkloadNamespaces = []string{"ignored-ns"}
		cfg.Startup.IgnoreUnavailableNodes = []string{"titan-22"}
		cfg.Startup.IgnoreWorkloads = []string{"monitoring/deployment/ignore-me"}
		cfg.Startup.IgnoreFluxKustomizations = []string{"ignored/flux-system"}

		readyRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"):
				return `{"items":[
	{"kind":"Deployment","metadata":{"namespace":"monitoring","name":"grafana"},"spec":{"replicas":1,"template":{"spec":{"nodeSelector":{"kubernetes.io/hostname":"titan-23"}}}},"status":{"readyReplicas":1}},
	{"kind":"DaemonSet","metadata":{"namespace":"monitoring","name":"node-exporter"},"spec":{"template":{"spec":{"nodeName":"titan-23"}}},"status":{"desiredNumberScheduled":2,"numberReady":1}},
	{"kind":"Deployment","metadata":{"namespace":"ignored-ns","name":"skip"},"spec":{"replicas":1},"status":{"readyReplicas":0}},
	{"kind":"Deployment","metadata":{"namespace":"flux-system","name":"ignored"},"spec":{"replicas":1},"status":{"readyReplicas":0}},
	{"kind":"Deployment","metadata":{"namespace":"monitoring","name":"ignore-me"},"spec":{"replicas":1},"status":{"readyReplicas":0}}
	]}`, nil
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchReady, _ := newHookOrchestrator(t, cfg, readyRun, readyRun)
		ready, detail, err := orchReady.TestHookWorkloadConvergenceReady(context.Background())
		if err != nil || !ready || !strings.Contains(detail, "controllers ready=") {
			t.Fatalf("expected workload convergence ready path, ready=%v detail=%q err=%v", ready, detail, err)
		}

		pendingRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"):
				return `{"items":[{"kind":"Deployment","metadata":{"namespace":"monitoring","name":"grafana"},"spec":{"replicas":1},"status":{"readyReplicas":0}}]}`, nil
			case name == "kubectl" && strings.Contains(command, "get pods -A -o json"):
				return `{"items":[]}`, nil
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchPending, _ := newHookOrchestrator(t, cfg, pendingRun, pendingRun)
		if err := orchPending.TestHookWaitForWorkloadConvergence(context.Background()); err == nil || !strings.Contains(err.Error(), "workload convergence not satisfied") {
			t.Fatalf("expected workload convergence timeout, got %v", err)
		}

		podRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "get pods -A -o json"):
				return `{"items":[
	{"metadata":{"namespace":"","name":"missing-ns"}},
	{"metadata":{"namespace":"kube-system","name":"ignored","creationTimestamp":"2020-01-01T00:00:00Z","ownerReferences":[{"kind":"ReplicaSet","name":"ignored"}]},"spec":{"nodeName":"titan-23","containers":[{"name":"c"}]},"status":{"containerStatuses":[{"state":{"waiting":{"reason":"CrashLoopBackOff"}}}]}},
	{"metadata":{"namespace":"monitoring","name":"ignore-me","creationTimestamp":"2020-01-01T00:00:00Z","ownerReferences":[{"kind":"ReplicaSet","name":"ignore-me"}]},"spec":{"nodeName":"titan-23","containers":[{"name":"c"}]},"status":{"containerStatuses":[{"state":{"waiting":{"reason":"CrashLoopBackOff"}}}]}},
	{"metadata":{"namespace":"monitoring","name":"node-ignored","creationTimestamp":"2020-01-01T00:00:00Z","ownerReferences":[{"kind":"ReplicaSet","name":"node-ignored"}]},"spec":{"nodeName":"titan-22","containers":[{"name":"c"}]},"status":{"containerStatuses":[{"state":{"waiting":{"reason":"CrashLoopBackOff"}}}]}},
	{"metadata":{"namespace":"monitoring","name":"unowned","creationTimestamp":"2020-01-01T00:00:00Z"},"spec":{"nodeName":"titan-23","containers":[{"name":"c"}]},"status":{"containerStatuses":[{"state":{"waiting":{"reason":"CrashLoopBackOff"}}}]}},
	{"metadata":{"namespace":"monitoring","name":"recent","creationTimestamp":"` + time.Now().UTC().Format(time.RFC3339) + `","ownerReferences":[{"kind":"ReplicaSet","name":"recent"}]},"spec":{"nodeName":"titan-23","containers":[{"name":"c"}]},"status":{"containerStatuses":[{"state":{"waiting":{"reason":"CrashLoopBackOff"}}}]}},
	{"metadata":{"namespace":"monitoring","name":"grafana-0","creationTimestamp":"2020-01-01T00:00:00Z","ownerReferences":[{"kind":"ReplicaSet","name":"grafana"}]},"spec":{"nodeName":"titan-23","containers":[{"name":"c"}]},"status":{"containerStatuses":[{"state":{"waiting":{"reason":"CrashLoopBackOff"}}}]}},
	{"metadata":{"namespace":"vault","name":"vault-0","creationTimestamp":"2020-01-01T00:00:00Z","ownerReferences":[{"kind":"StatefulSet","name":"vault"}],"annotations":{"vault.hashicorp.com/agent-inject":"true"}},"spec":{"nodeName":"titan-23","containers":[{"name":"vault"}]},"status":{"phase":"Pending","initContainerStatuses":[{"name":"vault-agent-init","state":{"running":{"startedAt":"2020-01-01T00:00:00Z"}}}]}}
	]}`, nil
			case name == "kubectl" && strings.Contains(command, "delete pod grafana-0"):
				return "", errors.New("boom")
			case name == "kubectl" && strings.Contains(command, "delete pod vault-0"):
				return "", nil
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchPods, _ := newHookOrchestrator(t, cfg, podRun, podRun)
		if err := orchPods.TestHookRecycleStuckControllerPods(context.Background()); err != nil {
			t.Fatalf("expected recycleStuckControllerPods best-effort success, got %v", err)
		}
	})

	t.Run("scaling-helpers-and-recovery", func(t *testing.T) {
		cfg := lifecycleConfig(t)
		cfg.ExcludedNamespaces = []string{"flux-system", "vault"}

		explicit := []string{"worker-a", "worker-b"}
		cfg.Workers = append([]string{}, explicit...)
		orchWorkers, _ := newHookOrchestrator(t, cfg, nil, nil)
		gotWorkers, err := orchWorkers.TestHookEffectiveWorkers(context.Background())
		if err != nil || len(gotWorkers) != len(explicit) || gotWorkers[0] != explicit[0] || gotWorkers[1] != explicit[1] {
			t.Fatalf("expected explicit workers copy, got %v err=%v", gotWorkers, err)
		}
		gotWorkers[0] = "mutated"
		if cfg.Workers[0] != explicit[0] {
			t.Fatalf("expected effectiveWorkers to return a copy")
		}

		cfg.Workers = nil
		cfg.SSHManagedNodes = nil
		cfg.SSHNodeHosts = map[string]string{
			"worker-c": "worker-c",
			"worker-b": "worker-b",
			"worker-a": "worker-a",
			"titan-db": "titan-db",
		}
		discoverErrRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			if name == "kubectl" && strings.Contains(command, "get nodes -o custom-columns=") {
				return "", errors.New("nodes unavailable")
			}
			return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
		}
		orchFallback, _ := newHookOrchestrator(t, cfg, discoverErrRun, discoverErrRun)
		fallbackWorkers, err := orchFallback.TestHookEffectiveWorkers(context.Background())
		if err != nil || strings.Join(fallbackWorkers, ",") != "worker-a,worker-b,worker-c" {
			t.Fatalf("expected fallback workers, got %v err=%v", fallbackWorkers, err)
		}

		discoverRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "get nodes -o custom-columns="):
				return "titan-db <none> <none>\nworker-b <none> <none>\nworker-c control-plane <none>\nbadline\n", nil
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchDiscover, _ := newHookOrchestrator(t, lifecycleConfig(t), discoverRun, discoverRun)
		discovered, err := orchDiscover.TestHookDiscoverWorkers(context.Background())
		if err != nil || strings.Join(discovered, ",") != "titan-db,worker-b" {
			t.Fatalf("expected discovered workers, got %v err=%v", discovered, err)
		}

		cfgPatch := lifecycleConfig(t)
		patchRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "-n flux-system get kustomizations.kustomize.toolkit.fluxcd.io -o jsonpath="):
				return "services\nignored\n", nil
			case name == "kubectl" && strings.Contains(command, "get helmreleases.helm.toolkit.fluxcd.io -A -o jsonpath="):
				return "monitoring/grafana\nmonitoring/failing\n", nil
			case name == "kubectl" && strings.Contains(command, "-n flux-system patch kustomization services"):
				return "", errors.New("patch failed")
			case name == "kubectl" && strings.Contains(command, "-n flux-system patch kustomization ignored"):
				return "", nil
			case name == "kubectl" && strings.Contains(command, "-n monitoring patch helmrelease grafana"):
				return "", nil
			case name == "kubectl" && strings.Contains(command, "-n monitoring patch helmrelease failing"):
				return "", errors.New("patch failed")
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchPatch, _ := newHookOrchestrator(t, cfgPatch, patchRun, patchRun)
		if err := orchPatch.TestHookPatchFluxSuspendAll(context.Background(), true); err != nil {
			t.Fatalf("expected patchFluxSuspendAll best-effort success, got %v", err)
		}

		listRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "get deployment -A -o jsonpath="):
				return "monitoring\tgrafana\t2\nflux-system\tsource-controller\t1\nmonitoring\tbad\tbogus\nmonitoring\tempty\t0\n", nil
			case name == "kubectl" && strings.Contains(command, "get statefulset -A -o jsonpath="):
				return "monitoring\tvictoria-metrics-single-server\t3\nvault\tvault\t1\n", nil
			case name == "kubectl" && strings.Contains(command, "scale deployment grafana --replicas=0"):
				return "", errors.New("scale failed")
			case name == "kubectl" && strings.Contains(command, "scale deployment grafana --replicas=1"):
				return "", nil
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchList, _ := newHookOrchestrator(t, cfgPatch, listRun, listRun)
		entries, err := orchList.TestHookListScalableWorkloads(context.Background())
		if err != nil || len(entries) != 4 {
			t.Fatalf("expected four scalable workloads, got %v err=%v", entries, err)
		}
		if err := orchList.TestHookScaleWorkloads(context.Background(), entries[:1], 0, 0); err != nil {
			t.Fatalf("expected single-entry scaleWorkloads success, got %v", err)
		}
		if err := orchList.TestHookScaleWorkloads(context.Background(), entries[1:], 0, 0); err == nil || !strings.Contains(err.Error(), "scaling had") {
			t.Fatalf("expected scaleWorkloads error aggregation, got %v", err)
		}
		if err := orchList.TestHookScaleWorkloads(context.Background(), nil, 0, 1); err != nil {
			t.Fatalf("expected empty scaleWorkloads success, got %v", err)
		}

		orchSnapshotWrite, _ := newHookOrchestrator(t, lifecycleConfig(t), nil, nil)
		if err := orchSnapshotWrite.TestHookWriteScaledWorkloadSnapshot(nil); err != nil {
			t.Fatalf("expected snapshot write with empty entries, got %v", err)
		}
		orchSnapshotRead, _ := newHookOrchestrator(t, lifecycleConfig(t), nil, nil)
		if snapshot, err := orchSnapshotRead.TestHookReadScaledWorkloadSnapshot(); err != nil || snapshot != nil {
			t.Fatalf("expected missing snapshot to read as nil,nil, got snapshot=%v err=%v", snapshot, err)
		}

		manualCfg := lifecycleConfig(t)
		manualCfg.State.Dir = filepath.Join(t.TempDir(), "state")
		manualOrch := cluster.New(manualCfg, &execx.Runner{DryRun: true}, state.New(manualCfg.State.RunHistoryPath), log.New(io.Discard, "", 0))
		if err := manualOrch.TestHookWriteScaledWorkloadSnapshot(nil); err != nil {
			t.Fatalf("expected dry-run snapshot write success, got %v", err)
		}
		if snapshot, err := manualOrch.TestHookReadScaledWorkloadSnapshot(); err != nil || snapshot != nil {
			t.Fatalf("expected dry-run snapshot read to return nil,nil, got snapshot=%v err=%v", snapshot, err)
		}

		restorePath := filepath.Join(t.TempDir(), "state", "scaled-workloads.json")
		if err := os.MkdirAll(filepath.Dir(restorePath), 0o755); err != nil {
			t.Fatalf("mkdir restore path: %v", err)
		}
		valid := `{"generated_at":"2026-01-01T00:00:00Z","entries":[{"namespace":"monitoring","kind":"deployment","name":"grafana","replicas":1}]}`
		if err := os.WriteFile(restorePath, []byte(valid), 0o600); err != nil {
			t.Fatalf("write restore snapshot: %v", err)
		}
		restoreCfg := lifecycleConfig(t)
		restoreCfg.State.Dir = filepath.Dir(restorePath)
		restoreOrch, _ := newHookOrchestrator(t, restoreCfg, listRun, listRun)
		if err := restoreOrch.TestHookRestoreScaledApps(context.Background()); err != nil {
			t.Fatalf("expected restoreScaledApps success, got %v", err)
		}
		if _, err := os.Stat(restorePath); !os.IsNotExist(err) {
			t.Fatalf("expected restore snapshot to be removed, stat err=%v", err)
		}

		pendingCfg := lifecycleConfig(t)
		pendingCfg.Startup.WorkloadConvergenceWaitSeconds = 1
		pendingCfg.Startup.WorkloadConvergencePollSeconds = 1
		pendingRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"):
				return `{"items":[{"kind":"Deployment","metadata":{"namespace":"monitoring","name":"grafana"},"spec":{"replicas":1},"status":{"readyReplicas":0}}]}`, nil
			case name == "kubectl" && strings.Contains(command, "get pods -A -o json"):
				return `{"items":[]}`, nil
			case name == "kubectl" && strings.Contains(command, "scale deployment grafana --replicas=0"):
				return "", errors.New("scale failed")
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchPendingScale, _ := newHookOrchestrator(t, pendingCfg, pendingRun, pendingRun)
		if err := orchPendingScale.TestHookScaleDownApps(context.Background()); err == nil {
			t.Fatalf("expected scaleDownApps to fail when workloads stay pending")
		}
	})

	t.Run("critical-vault-and-flux-health-helpers", func(t *testing.T) {
		fluxCfg := lifecycleConfig(t)
		fluxCfg.Startup.FluxHealthWaitSeconds = 1
		fluxCfg.Startup.FluxHealthPollSeconds = 1
		fluxCfg.Startup.IgnoreFluxKustomizations = []string{"flux-system/ignored"}

		fluxRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
				return `{"items":[
	{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false,"timeout":"40s"},"status":{"conditions":[{"type":"Ready","status":"False","message":"syncing"}]}},
	{"metadata":{"namespace":"flux-system","name":"no-condition"},"spec":{"suspend":false,"timeout":"10s"},"status":{"conditions":[{"type":"Reconciling","status":"True","message":"still"}]}},
	{"metadata":{"namespace":"flux-system","name":"ignored"},"spec":{"suspend":false,"timeout":"5s"},"status":{"conditions":[{"type":"Ready","status":"False","message":"ignored"}]}},
	{"metadata":{"namespace":"flux-system","name":"suspended"},"spec":{"suspend":true,"timeout":"2m"},"status":{"conditions":[{"type":"Ready","status":"False","message":"skip"}]}},
	{"metadata":{"namespace":"monitoring","name":"grafana"},"spec":{"suspend":false,"timeout":"5m"},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}
	]}`, nil
			case name == "kubectl" && strings.Contains(command, "get jobs -A -o json"):
				return `{"items":[
	{"metadata":{"namespace":"flux-system","name":"job-a","labels":{"kustomize.toolkit.fluxcd.io/name":"services"}},"status":{"failed":1,"conditions":[{"type":"Failed","status":"True"}]}},
	{"metadata":{"namespace":"flux-system","name":"job-b","labels":{"kustomize.toolkit.fluxcd.io/name":"services"}},"status":{"failed":1,"conditions":[{"type":"Failed","status":"True"}]}},
	{"metadata":{"namespace":"flux-system","name":"cronjob-owned","ownerReferences":[{"kind":"CronJob","name":"cron"}]},"status":{"failed":1,"conditions":[{"type":"Failed","status":"True"}]}},
	{"metadata":{"namespace":"flux-system","name":"succeeded"},"status":{"succeeded":1,"failed":1,"conditions":[{"type":"Failed","status":"True"}]}}
	]}`, nil
			case name == "kubectl" && strings.Contains(command, "delete job job-a"):
				return "", nil
			case name == "kubectl" && strings.Contains(command, "delete job job-b"):
				return "", errors.New("boom")
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchFlux, _ := newHookOrchestrator(t, fluxCfg, fluxRun, fluxRun)
		wait, reason, err := orchFlux.TestHookAdaptiveFluxHealthWait(context.Background(), 30*time.Second)
		if err != nil || wait <= 30*time.Second || !strings.Contains(reason, "max flux timeout") {
			t.Fatalf("expected adaptive flux wait extension, wait=%s reason=%q err=%v", wait, reason, err)
		}
		noTimeoutCfg := lifecycleConfig(t)
		noTimeoutRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
				return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"ready"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchNoTimeout, _ := newHookOrchestrator(t, noTimeoutCfg, noTimeoutRun, noTimeoutRun)
		if wait, reason, err := orchNoTimeout.TestHookAdaptiveFluxHealthWait(context.Background(), 0); err != nil || wait < 15*time.Minute || !strings.Contains(reason, "no explicit kustomization timeouts found") {
			t.Fatalf("expected adaptive wait fallback, wait=%s reason=%q err=%v", wait, reason, err)
		}
		if ready, detail, err := orchFlux.TestHookFluxHealthReady(context.Background()); err != nil || ready || !strings.Contains(detail, "not ready") {
			t.Fatalf("expected flux health not-ready result, ready=%v detail=%q err=%v", ready, detail, err)
		}
		if ready, detail, err := orchNoTimeout.TestHookFluxHealthReady(context.Background()); err != nil || !ready || !strings.Contains(detail, "ready=") {
			t.Fatalf("expected flux health ready result, ready=%v detail=%q err=%v", ready, detail, err)
		}
		if !cluster.TestHookLooksLikeImmutableJobError("Job update failed: field is immutable") {
			t.Fatalf("expected immutable-job detector true")
		}
		if cluster.TestHookLooksLikeImmutableJobError("") {
			t.Fatalf("expected empty immutable-job detail to be false")
		}
		if !cluster.TestHookJobLooksFluxManaged("flux-system", "job-a", map[string]string{"kustomize.toolkit.fluxcd.io/name": "services"}, nil) {
			t.Fatalf("expected label-based flux-managed job")
		}
		if cluster.TestHookJobLooksFluxManaged("flux-system", "cronjob-owned", nil, []string{"CronJob"}) {
			t.Fatalf("expected CronJob-owned job to be non flux-managed")
		}
		if !cluster.TestHookJobFailed(1, 0, []string{"Failed"}, []string{"True"}) {
			t.Fatalf("expected failed job detector to be true")
		}
		if cluster.TestHookJobFailed(0, 1, []string{"Complete"}, []string{"True"}) {
			t.Fatalf("expected succeeded job to be false")
		}
		if healed, err := orchFlux.TestHookHealImmutableFluxJobs(context.Background()); err != nil || !healed {
			t.Fatalf("expected immutable job heal success, healed=%v err=%v", healed, err)
		}

		critCfg := lifecycleConfig(t)
		critCfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "vault", "unseal.key")
		critCfg.Startup.VaultUnsealBreakglassCommand = "echo breakglass-key"
		critCfg.Startup.VaultUnsealBreakglassTimeout = 1
		critCfg.Startup.WorkloadConvergenceWaitSeconds = 1
		critCfg.Startup.WorkloadConvergencePollSeconds = 1
		critCfg.Startup.StuckPodGraceSeconds = 1
		critCfg.Startup.IgnoreWorkloadNamespaces = []string{"kube-system"}
		critCfg.Startup.IgnoreUnavailableNodes = []string{"titan-22"}
		critCfg.Startup.IgnoreWorkloads = []string{"monitoring/deployment/ignore-me"}

		critRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "-n flux-system get deployment source-controller -o jsonpath={.status.readyReplicas}"):
				return "", errors.New("boom")
			case name == "kubectl" && strings.Contains(command, "-n flux-system scale deployment source-controller --replicas=1"):
				return "", errors.New("boom")
			case name == "kubectl" && strings.Contains(command, "-n vault get pods -o custom-columns="):
				return "vault-0 Pending StatefulSet vault\nvault-1 Unknown StatefulSet vault\nvault-2 Running StatefulSet vault\nvault-other Failed Deployment vault\nbadline\n", nil
			case name == "kubectl" && strings.Contains(command, "delete pod vault-0"):
				return "", nil
			case name == "kubectl" && strings.Contains(command, "delete pod vault-1"):
				return "", errors.New("boom")
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchCrit, _ := newHookOrchestrator(t, critCfg, critRun, critRun)
		if missing, err := orchCrit.TestHookMissingCriticalStartupWorkloads(context.Background()); err == nil || len(missing) != 0 {
			t.Fatalf("expected missingCriticalStartupWorkloads generic error, missing=%v err=%v", missing, err)
		}
		if err := orchCrit.TestHookEnsureCriticalStartupWorkloads(context.Background()); err == nil || !strings.Contains(err.Error(), "scale") {
			t.Fatalf("expected ensureCriticalStartupWorkloads scale error, got %v", err)
		}
		if err := orchCrit.TestHookCleanupStaleCriticalWorkloadPods(context.Background(), "vault", "statefulset", "vault"); err == nil {
			t.Fatalf("expected stale critical workload cleanup error branch")
		}

		vaultCfg := lifecycleConfig(t)
		vaultCfg.Startup.VaultUnsealKeyFile = filepath.Join(t.TempDir(), "vault", "unseal.key")
		vaultCfg.Startup.VaultUnsealBreakglassCommand = "echo breakglass-key"
		vaultCfg.Startup.VaultUnsealBreakglassTimeout = 1
		ensureUnsealed := false
		ensureRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
				return "Running", nil
			case name == "kubectl" && strings.Contains(command, "vault status -format=json"):
				if ensureUnsealed {
					return `{"sealed":false}`, nil
				}
				return `{"sealed":true}`, nil
			case name == "kubectl" && strings.Contains(command, "get secret vault-init"):
				return "dmF1bHQta2V5", nil
			case name == "kubectl" && strings.Contains(command, "vault operator unseal"):
				ensureUnsealed = true
				return "", nil
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchVaultEnsure, _ := newHookOrchestrator(t, vaultCfg, ensureRun, ensureRun)
		if err := orchVaultEnsure.TestHookEnsureVaultUnsealed(context.Background()); err != nil {
			t.Fatalf("expected vault auto-unseal success path, got %v", err)
		}
		if sealed, err := orchVaultEnsure.TestHookVaultSealed(context.Background()); err != nil || sealed {
			t.Fatalf("expected vault sealed helper false after unseal, sealed=%v err=%v", sealed, err)
		}

		waitReady := false
		waitUnsealed := false
		waitRun := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
			command := name + " " + strings.Join(args, " ")
			switch {
			case name == "kubectl" && strings.Contains(command, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
				return "Running", nil
			case name == "kubectl" && strings.Contains(command, "vault status -format=json"):
				if waitUnsealed {
					return `{"sealed":false}`, nil
				}
				return `{"sealed":true}`, nil
			case name == "kubectl" && strings.Contains(command, "get secret vault-init"):
				return "dmF1bHQta2V5", nil
			case name == "kubectl" && strings.Contains(command, "vault operator unseal"):
				waitUnsealed = true
				waitReady = true
				return "", nil
			case name == "kubectl" && strings.Contains(command, "get statefulset vault -o jsonpath={.status.readyReplicas}"):
				if waitReady {
					return "1", nil
				}
				return "0", nil
			case name == "kubectl" && strings.Contains(command, "get deployment -A -o jsonpath="):
				return "monitoring\tgrafana\t1\n", nil
			case name == "kubectl" && strings.Contains(command, "get statefulset -A -o jsonpath="):
				return "monitoring\tvictoria-metrics-single-server\t1\n", nil
			default:
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
		}
		orchVaultWait, _ := newHookOrchestrator(t, vaultCfg, waitRun, waitRun)
		if err := orchVaultWait.TestHookWaitVaultReady(context.Background(), "vault", "statefulset", "vault"); err != nil {
			t.Fatalf("expected vault wait success path, got %v", err)
		}
		if err := orchVaultWait.TestHookWaitWorkloadReady(context.Background(), "monitoring", "deployment", "grafana"); err != nil {
			t.Fatalf("expected generic workload wait success, got %v", err)
		}
		if err := orchVaultWait.TestHookWriteVaultUnsealKeyFile("cached-key"); err != nil {
			t.Fatalf("expected vault key file write success, got %v", err)
		}
		if got, err := orchVaultWait.TestHookReadVaultUnsealKeyFile(); err != nil || got != "cached-key" {
			t.Fatalf("expected vault key file read success, got %q err=%v", got, err)
		}

		blockedDir := t.TempDir()
		blockedFile := filepath.Join(blockedDir, "blocked")
		if err := os.WriteFile(blockedFile, []byte("x"), 0o600); err != nil {
			t.Fatalf("write blocked file: %v", err)
		}
		blockedCfg := lifecycleConfig(t)
		blockedCfg.Startup.VaultUnsealKeyFile = filepath.Join(blockedFile, "vault.key")
		blockedOrch, _ := newHookOrchestrator(t, blockedCfg, nil, nil)
		if err := blockedOrch.TestHookWriteVaultUnsealKeyFile("x"); err == nil {
			t.Fatalf("expected vault key dir error")
		}
	})
}