ananke/testing/orchestrator/hooks_gap_matrix_part11_test.go

package orchestrator

import (
	"context"
	"errors"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"scm.bstein.dev/bstein/ananke/internal/cluster"
	"scm.bstein.dev/bstein/ananke/internal/config"
	"scm.bstein.dev/bstein/ananke/internal/execx"
	"scm.bstein.dev/bstein/ananke/internal/state"
)

// newLifecycleMatrixOrchestrator runs one orchestration or CLI step.
// Signature: newLifecycleMatrixOrchestrator(t *testing.T, cfg config.Config, dryRun bool, run commandOverride, runSensitive commandOverride, kubeconfig string) *cluster.Orchestrator.
// Why: part11 needs direct control over runner dry-run and kubeconfig branches.
func newLifecycleMatrixOrchestrator(
	t *testing.T,
	cfg config.Config,
	dryRun bool,
	run func(context.Context, time.Duration, string, ...string) (string, error),
	runSensitive func(context.Context, time.Duration, string, ...string) (string, error),
	kubeconfig string,
) *cluster.Orchestrator {
	t.Helper()
	if err := os.MkdirAll(cfg.State.Dir, 0o755); err != nil {
		t.Fatalf("ensure state dir: %v", err)
	}
	runner := &execx.Runner{DryRun: dryRun, Kubeconfig: kubeconfig}
	orch := cluster.New(cfg, runner, state.New(cfg.State.RunHistoryPath), log.New(io.Discard, "", 0))
	if run == nil || runSensitive == nil {
		recorder := &commandRecorder{}
		base := lifecycleDispatcher(recorder)
		if run == nil {
			run = base
		}
		if runSensitive == nil {
			runSensitive = base
		}
	}
	orch.SetCommandOverrides(run, runSensitive)
	return orch
}

// TestHookGapMatrixPart11RemainingClosure runs one orchestration or CLI step.
// Signature: TestHookGapMatrixPart11RemainingClosure(t *testing.T).
// Why: closes final branch gaps for lifecycle + remaining near-threshold
// orchestrator files so per-file coverage reaches the enforced 95% target.
func TestHookGapMatrixPart11RemainingClosure(t *testing.T) {
	t.Run("critical-vault-final-closures", func(t *testing.T) {
		t.Run("ensure-critical-cleanup-error-and-cleanup-branches", func(t *testing.T) {
			cfg := lifecycleConfig(t)
			run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				switch {
				case name == "kubectl" && strings.Contains(command, "get pods -o custom-columns"):
					return "", errors.New("pods query failed")
				default:
					return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
				}
			}
			orch := newLifecycleMatrixOrchestrator(t, cfg, false, run, run, "")
			if err := orch.TestHookEnsureCriticalStartupWorkloads(context.Background()); err == nil || !strings.Contains(err.Error(), "cleanup stale pods") {
				t.Fatalf("expected ensureCriticalStartupWorkloads cleanup error branch, got %v", err)
			}

			dry := newLifecycleMatrixOrchestrator(t, cfg, true, nil, nil, "")
			if err := dry.TestHookCleanupStaleCriticalWorkloadPods(context.Background(), "vault", "statefulset", "vault"); err != nil {
				t.Fatalf("expected cleanup stale pod dry-run branch, got %v", err)
			}

			runCleanup := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				if name == "kubectl" && strings.Contains(command, "get pods -o custom-columns") {
					return strings.Join([]string{
						"badline",
						"vault-0 Unknown Deployment vault",
						"otherpod Unknown StatefulSet vault",
					}, "\n"), nil
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchCleanup := newLifecycleMatrixOrchestrator(t, cfg, false, runCleanup, runCleanup, "")
			if err := orchCleanup.TestHookCleanupStaleCriticalWorkloadPods(context.Background(), "vault", "statefulset", "vault"); err != nil {
				t.Fatalf("expected cleanup stale pod parse/owner/prefix branches, got %v", err)
			}
		})

		t.Run("wait-vault-ready-error-tracking-and-ensure-unseal-branches", func(t *testing.T) {
			cfg := lifecycleConfig(t)
			run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				switch {
				case name == "kubectl" && strings.Contains(command, "jsonpath={.status.readyReplicas}"):
					return "", errors.New("ready query failed")
				case name == "kubectl" && strings.Contains(command, "get pod vault-0"):
					return "Running", nil
				case name == "kubectl" && strings.Contains(command, "vault status -format=json"):
					return `{"sealed":false}`, nil
				default:
					return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
				}
			}
			orch := newLifecycleMatrixOrchestrator(t, cfg, false, run, run, "")
			ctx, cancel := context.WithCancel(context.Background())
			cancel()
			if err := orch.TestHookWaitVaultReady(ctx, "vault", "statefulset", "vault"); !errors.Is(err, context.Canceled) {
				t.Fatalf("expected canceled vault wait branch with error tracking, got %v", err)
			}

			dry := newLifecycleMatrixOrchestrator(t, cfg, true, nil, nil, "")
			if err := dry.TestHookEnsureVaultUnsealed(context.Background()); err != nil {
				t.Fatalf("expected ensureVaultUnsealed dry-run fast path, got %v", err)
			}

			runSealedErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				switch {
				case name == "kubectl" && strings.Contains(command, "get pod vault-0"):
					return "Running", nil
				case name == "kubectl" && strings.Contains(command, "vault status -format=json"):
					return "", errors.New("vault status query failed")
				default:
					return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
				}
			}
			orchSealedErr := newLifecycleMatrixOrchestrator(t, cfg, false, runSealedErr, runSealedErr, "")
			if err := orchSealedErr.TestHookEnsureVaultUnsealed(context.Background()); err == nil || !strings.Contains(err.Error(), "vault status check failed") {
				t.Fatalf("expected ensureVaultUnsealed sealed-check error branch, got %v", err)
			}
		})

		t.Run("parse-vault-sealed-json-unmarshal-error", func(t *testing.T) {
			if _, err := cluster.TestHookParseVaultSealed(`{"sealed":tru}`); err == nil {
				t.Fatalf("expected parseVaultSealed json-unmarshal error branch")
			}
		})
	})

	t.Run("ingress-service-storage-timesync-final-closures", func(t *testing.T) {
		t.Run("startup-convergence-workload-and-stability-failure-branches", func(t *testing.T) {
			cfgWorkload := lifecycleConfig(t)
			cfgWorkload.Startup.RequireIngressChecklist = false
			cfgWorkload.Startup.RequireServiceChecklist = false
			cfgWorkload.Startup.RequireCriticalServiceEndpoints = false
			cfgWorkload.Startup.RequireFluxHealth = false
			cfgWorkload.Startup.RequireWorkloadConvergence = true
			cfgWorkload.Startup.ServiceChecklistStabilitySec = 0
			runWorkload := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				if name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json") {
					return "", errors.New("controllers query failed")
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchWorkload := newLifecycleMatrixOrchestrator(t, cfgWorkload, false, runWorkload, runWorkload, "")
			if err := orchWorkload.TestHookWaitForStartupConvergence(context.Background()); err == nil || !strings.Contains(err.Error(), "query controllers") {
				t.Fatalf("expected startup convergence workload failure branch, got %v", err)
			}

			cfgStability := lifecycleConfig(t)
			cfgStability.Startup.RequireIngressChecklist = false
			cfgStability.Startup.RequireServiceChecklist = false
			cfgStability.Startup.RequireCriticalServiceEndpoints = false
			cfgStability.Startup.RequireFluxHealth = false
			cfgStability.Startup.RequireWorkloadConvergence = false
			cfgStability.Startup.ServiceChecklistStabilitySec = 1
			cfgStability.Startup.ServiceChecklistPollSeconds = 1
			runStability := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				if name == "kubectl" && strings.Contains(command, "get pods -A -o json") {
					return "", errors.New("pods query failed")
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchStability := newLifecycleMatrixOrchestrator(t, cfgStability, false, runStability, runStability, "")
			if err := orchStability.TestHookWaitForStartupConvergence(context.Background()); err == nil || !strings.Contains(err.Error(), "stability window failed") {
				t.Fatalf("expected startup convergence stability failure branch, got %v", err)
			}
		})

		t.Run("ingress-host-discovery-and-autoheal-early-returns", func(t *testing.T) {
			cfg := lifecycleConfig(t)
			runHostsErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				if name == "kubectl" && strings.Contains(command, "get ingress -A -o json") {
					return "", errors.New("ingress list failed")
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchHostsErr := newLifecycleMatrixOrchestrator(t, cfg, false, runHostsErr, runHostsErr, "")
			if _, err := orchHostsErr.TestHookDiscoverIngressHosts(context.Background()); err == nil || !strings.Contains(err.Error(), "query ingresses") {
				t.Fatalf("expected discoverIngressHosts query error branch, got %v", err)
			}
			if _, detail := orchHostsErr.TestHookIngressChecklistReady(context.Background()); !strings.Contains(detail, "query ingresses") {
				t.Fatalf("expected ingress checklist to surface discovery error detail, got %q", detail)
			}

			runNs := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				if name == "kubectl" && strings.Contains(command, "get ingress -A -o json") {
					return `{"items":[{"metadata":{"namespace":""},"spec":{"rules":[{"host":""},{"host":"logs.bstein.dev"}]}}]}`, nil
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchNs := newLifecycleMatrixOrchestrator(t, cfg, false, runNs, runNs, "")
			namespaces, err := orchNs.TestHookDiscoverIngressNamespacesForHost(context.Background(), "logs.bstein.dev")
			if err != nil || len(namespaces) != 0 {
				t.Fatalf("expected namespace-empty/rule-empty skip branches, namespaces=%v err=%v", namespaces, err)
			}

			dry := newLifecycleMatrixOrchestrator(t, cfg, true, nil, nil, "")
			now := time.Now().UTC().Add(-time.Hour)
			dry.TestHookMaybeAutoHealIngressHostBackends(context.Background(), &now, "logs.bstein.dev: 502")

			orchEarly := newLifecycleMatrixOrchestrator(t, cfg, false, nil, nil, "")
			orchEarly.TestHookMaybeAutoHealIngressHostBackends(context.Background(), &now, "no-host-here")
			if host := orchEarly.TestHookChecklistFailureHost("https://logs.bstein.dev/login"); host != "" {
				t.Fatalf("expected checklistFailureHost colon-split edge branch, got %q", host)
			}
		})

		t.Run("service-stability-error-branches", func(t *testing.T) {
			cfg := lifecycleConfig(t)
			runWorkloadsErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				switch {
				case name == "kubectl" && strings.Contains(command, "get ingress -A -o json"):
					return `{"items":[{"metadata":{"namespace":"monitoring"},"spec":{"rules":[{"host":"logs.bstein.dev"}]}}]}`, nil
				case name == "kubectl" && strings.Contains(command, "get deploy,statefulset -A -o json"):
					return "", errors.New("workload list failed")
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchWorkloadsErr := newLifecycleMatrixOrchestrator(t, cfg, false, runWorkloadsErr, runWorkloadsErr, "")
			if _, err := orchWorkloadsErr.TestHookHealIngressHostBackendReplicas(context.Background(), "logs.bstein.dev"); err == nil || !strings.Contains(err.Error(), "query workloads") {
				t.Fatalf("expected healIngressHostBackendReplicas query error branch, got %v", err)
			}

			runDecodeErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				switch {
				case name == "kubectl" && strings.Contains(command, "get ingress -A -o json"):
					return `{"items":[{"metadata":{"namespace":"monitoring"},"spec":{"rules":[{"host":"logs.bstein.dev"}]}}]}`, nil
				case name == "kubectl" && strings.Contains(command, "get deploy,statefulset -A -o json"):
					return "{not-json", nil
				default:
					return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
				}
			}
			orchDecodeErr := newLifecycleMatrixOrchestrator(t, cfg, false, runDecodeErr, runDecodeErr, "")
			if _, err := orchDecodeErr.TestHookHealIngressHostBackendReplicas(context.Background(), "logs.bstein.dev"); err == nil || !strings.Contains(err.Error(), "decode workloads") {
				t.Fatalf("expected healIngressHostBackendReplicas decode error branch, got %v", err)
			}

			runBodyErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchBodyErr := newLifecycleMatrixOrchestrator(t, cfg, false, runBodyErr, runBodyErr, "")
			ln, err := net.Listen("tcp", "127.0.0.1:0")
			if err != nil {
				t.Fatalf("open probe listener: %v", err)
			}
			defer ln.Close()
			go func() {
				conn, acceptErr := ln.Accept()
				if acceptErr == nil {
					_, _ = conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nshort"))
					_ = conn.Close()
				}
			}()
			_, _, probeErr := orchBodyErr.TestHookHTTPChecklistProbe(context.Background(), config.ServiceChecklistCheck{
				URL: "http://" + ln.Addr().String() + "/health",
			})
			if probeErr == nil || !strings.Contains(probeErr.Error(), "read response body") {
				t.Fatalf("expected checklist body read-error branch, got %v", probeErr)
			}

			cfgStability := lifecycleConfig(t)
			cfgStability.Startup.RequireFluxHealth = false
			cfgStability.Startup.RequireWorkloadConvergence = true
			cfgStability.Startup.RequireServiceChecklist = false
			cfgStability.Startup.RequireIngressChecklist = false
			runStability := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				if name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json") {
					return "", errors.New("controllers failed")
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchStability := newLifecycleMatrixOrchestrator(t, cfgStability, false, runStability, runStability, "")
			if err := orchStability.TestHookStartupStabilityHealthy(context.Background()); err == nil || !strings.Contains(err.Error(), "workload check error") {
				t.Fatalf("expected startupStabilityHealthy workload-error branch, got %v", err)
			}

			cfgWindow := lifecycleConfig(t)
			cfgWindow.Startup.ServiceChecklistStabilitySec = 1
			cfgWindow.Startup.ServiceChecklistPollSeconds = 1
			cfgWindow.Startup.RequireFluxHealth = false
			cfgWindow.Startup.RequireWorkloadConvergence = false
			cfgWindow.Startup.RequireServiceChecklist = false
			cfgWindow.Startup.RequireIngressChecklist = false
			orchWindow := newLifecycleMatrixOrchestrator(t, cfgWindow, false, nil, nil, "")
			ctx, cancel := context.WithCancel(context.Background())
			cancel()
			if err := orchWindow.TestHookWaitForStabilityWindow(ctx); !errors.Is(err, context.Canceled) {
				t.Fatalf("expected waitForStabilityWindow ctx-canceled branch, got %v", err)
			}
		})

		t.Run("storage-and-timesync-small-branches", func(t *testing.T) {
			cfgStorage := lifecycleConfig(t)
			cfgStorage.Startup.StorageMinReadyNodes = 1
			orchStorageDry := newLifecycleMatrixOrchestrator(t, cfgStorage, true, nil, nil, "")
			if err := orchStorageDry.TestHookWaitForStorageReady(context.Background()); err != nil {
				t.Fatalf("expected waitForStorageReady dry-run branch, got %v", err)
			}

			cfgStorage.Startup.StorageCriticalPVCs = []string{"   "}
			runStorage := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				if name == "kubectl" && strings.Contains(command, "get nodes.longhorn.io") {
					return "malformed\nnode-a:True:True\n", nil
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchStorage := newLifecycleMatrixOrchestrator(t, cfgStorage, false, runStorage, runStorage, "")
			ready, _, err := orchStorage.TestHookStorageReady(context.Background())
			if err != nil || !ready {
				t.Fatalf("expected storage malformed-line and empty-pvc-entry branches, ready=%v err=%v", ready, err)
			}

			cfgSync := lifecycleConfig(t)
			cfgSync.Startup.TimeSyncMode = "quorum"
			cfgSync.Startup.TimeSyncQuorum = 99
			cfgSync.SSHManagedNodes = []string{"titan-db"}
			runSync := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				switch {
				case name == "sh" && strings.Contains(command, "timedatectl show -p NTPSynchronized"):
					return "yes", nil
				case name == "ssh" && strings.Contains(command, "timedatectl show -p NTPSynchronized"):
					return "yes", nil
				default:
					return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
				}
			}
			orchSync := newLifecycleMatrixOrchestrator(t, cfgSync, false, runSync, runSync, "")
			if err := orchSync.TestHookWaitForTimeSync(context.Background(), []string{"", "titan-db", "unmanaged"}); err != nil {
				t.Fatalf("expected timesync quorum clamp/success branches, got %v", err)
			}

			cfgSyncBad := lifecycleConfig(t)
			cfgSyncBad.SSHPort = 22
			cfgSyncBad.SSHUser = ""
			cfgSyncBad.ControlPlanes = []string{"", "cp1"}
			cfgSyncBad.Workers = []string{"", "wk1"}
			cfgSyncBad.SSHManagedNodes = []string{"cp1"}
			cfgSyncBad.SSHNodeHosts = map[string]string{
				"cp1": "bad host",
				"wk1": "wk1",
			}
			orchSyncBad := newLifecycleMatrixOrchestrator(t, cfgSyncBad, false, nil, nil, "")
			if err := orchSyncBad.TestHookValidateNodeInventory(); err == nil || !strings.Contains(err.Error(), "node inventory preflight failed") {
				t.Fatalf("expected validateNodeInventory host/user branches, got %v", err)
			}

			orchSyncDry := newLifecycleMatrixOrchestrator(t, lifecycleConfig(t), true, nil, nil, "")
			if err := orchSyncDry.TestHookWaitForTimeSync(context.Background(), []string{"titan-db"}); err != nil {
				t.Fatalf("expected waitForTimeSync dry-run branch, got %v", err)
			}
		})
	})

	t.Run("poststart-scaling-and-lifecycle-final-closures", func(t *testing.T) {
		t.Run("poststart-ssh-and-sensitive-branches", func(t *testing.T) {
			cfg := lifecycleConfig(t)
			cfg.SSHManagedNodes = nil
			cfg.SSHNodeUsers = map[string]string{"titan-db": "special"}
			cfg.SSHJumpHost = "jump-a"
			cfg.SSHJumpUser = "jumper"
			cfg.SSHNodeHosts["jump-a"] = "jump.example.internal"
			attempt := 0
			run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				if name == "ssh" {
					attempt++
					if attempt <= 2 {
						return "REMOTE HOST IDENTIFICATION HAS CHANGED!", errors.New("host key mismatch")
					}
					return "", errors.New("still failing")
				}
				if name == "curl" {
					return "200", nil
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orch := newLifecycleMatrixOrchestrator(t, cfg, false, run, run, "/tmp/fake-kubeconfig")
			if _, err := orch.TestHookSSHWithTimeout(context.Background(), "titan-db", "echo ok", 2*time.Second); err == nil {
				t.Fatalf("expected sshWithTimeout retry-failure branch")
			}
			if !orch.TestHookSSHManaged("any-node") {
				t.Fatalf("expected sshManaged empty-allowlist branch")
			}
			if _, err := orch.TestHookRunSensitive(context.Background(), 2*time.Second, "sh", "-lc", "echo ok"); err != nil {
				t.Fatalf("expected runSensitive kubeconfig-env branch, got %v", err)
			}
		})

		t.Run("scaling-final-branches", func(t *testing.T) {
			cfg := lifecycleConfig(t)
			cfg.Workers = nil
			runEffective := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				if name == "kubectl" && len(args) >= 2 && args[0] == "get" && args[1] == "nodes" {
					return "wk1 <none> <none>", nil
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchEffective := newLifecycleMatrixOrchestrator(t, cfg, false, runEffective, runEffective, "")
			workers, err := orchEffective.TestHookEffectiveWorkers(context.Background())
			if err != nil || len(workers) != 1 || workers[0] != "wk1" {
				t.Fatalf("expected effectiveWorkers discover-success branch, workers=%v err=%v", workers, err)
			}

			runNoWorkers := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				if name == "kubectl" && len(args) >= 2 && args[0] == "get" && args[1] == "nodes" {
					return "cp1 true <none>", nil
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchNoWorkers := newLifecycleMatrixOrchestrator(t, cfg, false, runNoWorkers, runNoWorkers, "")
			if _, err := orchNoWorkers.TestHookDiscoverWorkers(context.Background()); err == nil || !strings.Contains(err.Error(), "no workers discovered") {
				t.Fatalf("expected discoverWorkers no-workers branch, got %v", err)
			}

			runPatch := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				switch {
				case name == "kubectl" && strings.Contains(command, "-n flux-system get kustomizations.kustomize.toolkit.fluxcd.io"):
					return "apps\n", nil
				case name == "kubectl" && strings.Contains(command, "get helmreleases.helm.toolkit.fluxcd.io -A"):
					return "", errors.New("helmrelease list failed")
				default:
					return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
				}
			}
			orchPatch := newLifecycleMatrixOrchestrator(t, cfg, false, runPatch, runPatch, "")
			if err := orchPatch.TestHookPatchFluxSuspendAll(context.Background(), true); err == nil || !strings.Contains(err.Error(), "helmrelease list failed") {
				t.Fatalf("expected patchFluxSuspendAll helmrelease-query error branch, got %v", err)
			}

			cfgScale := lifecycleConfig(t)
			cfgScale.Shutdown.ScaleParallelism = 0
			runScale := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				switch {
				case name == "kubectl" && strings.Contains(command, "get deployment -A -o jsonpath="):
					return "monitoring\tgrafana\t1\nbad-line\n", nil
				case name == "kubectl" && strings.Contains(command, "get statefulset -A -o jsonpath="):
					return "", errors.New("statefulset query failed")
				default:
					return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
				}
			}
			orchScale := newLifecycleMatrixOrchestrator(t, cfgScale, false, runScale, runScale, "")
			if _, err := orchScale.TestHookListScalableWorkloads(context.Background()); err == nil || !strings.Contains(err.Error(), "collect statefulsets") {
				t.Fatalf("expected listScalableWorkloads malformed/statefulset-error branches, got %v", err)
			}

			cfgRestore := lifecycleConfig(t)
			cfgRestore.Shutdown.ScaleParallelism = 0
			if err := os.MkdirAll(cfgRestore.State.Dir, 0o755); err != nil {
				t.Fatalf("mkdir restore state dir: %v", err)
			}
			snapshotPath := filepath.Join(cfgRestore.State.Dir, "scaled-workloads.json")
			if err := os.WriteFile(snapshotPath, []byte(`{"generated_at":"2026-01-01T00:00:00Z","entries":[{"namespace":"monitoring","kind":"deployment","name":"grafana","replicas":1}]}`), 0o644); err != nil {
				t.Fatalf("write restore snapshot: %v", err)
			}
			runRestore := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
				command := name + " " + strings.Join(args, " ")
				if name == "kubectl" && strings.Contains(command, "scale deployment grafana --replicas=1") {
					return "", errors.New("scale restore failed")
				}
				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
			}
			orchRestore := newLifecycleMatrixOrchestrator(t, cfgRestore, false, runRestore, runRestore, "")
			if err := orchRestore.TestHookRestoreScaledApps(context.Background()); err == nil || !strings.Contains(err.Error(), "scaling had 1 errors") {
				t.Fatalf("expected restoreScaledApps scale error branch, got %v", err)
			}

			orchRestoreDry := newLifecycleMatrixOrchestrator(t, cfgRestore, true, runRestore, runRestore, "")
			if err := orchRestoreDry.TestHookRestoreScaledApps(context.Background()); err != nil {
				t.Fatalf("expected restoreScaledApps dry-run branch, got %v", err)
			}
		})

		t.Run("lifecycle-startup-etcd-shutdown-final-branches", func(t *testing.T) {
			t.Run("startup-cooldown-and-poststart-failure-paths", func(t *testing.T) {
				cfg := lifecycleFastConfig(t)
				cfg.Startup.ShutdownCooldownSeconds = 300
				cfg.Startup.RequirePostStartProbes = true
				cfg.Startup.PostStartProbes = []string{"https://logs.bstein.dev/health"}
				if err := state.WriteIntent(cfg.State.IntentPath, state.Intent{
					State:     state.IntentShutdownComplete,
					Reason:    "recent-shutdown",
					Source:    "test",
					UpdatedAt: time.Now().UTC(),
				}); err != nil {
					t.Fatalf("seed cooldown intent: %v", err)
				}
				orchCooldown := newLifecycleMatrixOrchestrator(t, cfg, false, nil, nil, "")
				ctxCooldown, cancelCooldown := context.WithCancel(context.Background())
				cancelCooldown()
				if err := orchCooldown.Startup(ctxCooldown, cluster.StartupOptions{Reason: "cooldown"}); err == nil || !strings.Contains(err.Error(), "startup canceled while waiting for shutdown cooldown") {
					t.Fatalf("expected startup cooldown cancel branch, got %v", err)
				}

				cfgPost := lifecycleFastConfig(t)
				cfgPost.Startup.RequirePostStartProbes = true
				cfgPost.Startup.PostStartProbes = []string{"https://logs.bstein.dev/health"}
				cfgPost.Startup.PostStartProbeWaitSeconds = 1
				cfgPost.Startup.PostStartProbePollSeconds = 1
				cfgPost.Startup.APIWaitSeconds = 1
				cfgPost.Startup.APIPollSeconds = 5
				runPost := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
					if name == "curl" {
						return "", errors.New("probe failed")
					}
					return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
				}
				orchPost := newLifecycleMatrixOrchestrator(t, cfgPost, false, runPost, runPost, "")
				if err := orchPost.Startup(context.Background(), cluster.StartupOptions{Reason: "post-start"}); err == nil || !strings.Contains(err.Error(), "post-start probes") {
					t.Fatalf("expected startup post-start probe failure branch, got %v", err)
				}
			})

			t.Run("startup-bootstrap-and-flux-branch-error-paths", func(t *testing.T) {
				cfg := lifecycleFastConfig(t)
				repo := t.TempDir()
				cfg.IACRepoPath = repo
				cfg.LocalBootstrapPaths = []string{"services/bootstrap"}
				if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil {
					t.Fatalf("mkdir .git: %v", err)
				}
				if err := os.MkdirAll(filepath.Join(repo, "services", "bootstrap"), 0o755); err != nil {
					t.Fatalf("mkdir bootstrap dir: %v", err)
				}
				readyCalls := 0
				run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
					command := name + " " + strings.Join(args, " ")
					switch {
					case name == "kubectl" && strings.Contains(command, "jsonpath={.spec.ref.branch}"):
						return "", errors.New("branch read failed")
					case name == "kubectl" && strings.Contains(command, "jsonpath={.status.conditions[?(@.type==\"Ready\")].status}"):
						readyCalls++
						if readyCalls == 1 {
							return "", nil
						}
						return "True", nil
					case name == "git":
						return "", nil
					case name == "sh" && strings.Contains(command, "kubectl kustomize"):
						return "apiVersion: v1\nkind: Namespace\nmetadata:\n  name: bootstrap\n", nil
					default:
						return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
					}
				}
				orch := newLifecycleMatrixOrchestrator(t, cfg, false, run, run, "")
				if err := orch.Startup(context.Background(), cluster.StartupOptions{Reason: "flux-branch-error"}); err == nil || !strings.Contains(err.Error(), "read flux source branch") {
					t.Fatalf("expected startup ensureFluxBranch error branch, got %v", err)
				}
			})

			t.Run("startup-intent-clear-and-auto-etcd-flux-patch-branches", func(t *testing.T) {
				t.Run("clear-stale-startup-intent-success-path", func(t *testing.T) {
					cfg := lifecycleFastConfig(t)
					if err := state.WriteIntent(cfg.State.IntentPath, state.Intent{
						State:     state.IntentStartupInProgress,
						Reason:    "stale-startup",
						Source:    "test",
						UpdatedAt: time.Now().UTC(),
					}); err != nil {
						t.Fatalf("seed stale startup intent: %v", err)
					}
					run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
						command := name + " " + strings.Join(args, " ")
						if name == "kubectl" && strings.Contains(command, "jsonpath={.spec.url}") {
							return "", errors.New("flux url read failed")
						}
						return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
					}
					orch := newLifecycleMatrixOrchestrator(t, cfg, false, run, run, "")
					if err := orch.Startup(context.Background(), cluster.StartupOptions{Reason: "stale-startup-clear"}); err == nil || !strings.Contains(err.Error(), "read flux source url") {
						t.Fatalf("expected startup flux-url guard error after stale startup clear, got %v", err)
					}
				})

				t.Run("clear-stale-shutdown-intent-success-path", func(t *testing.T) {
					cfg := lifecycleFastConfig(t)
					if err := state.WriteIntent(cfg.State.IntentPath, state.Intent{
						State:     state.IntentShuttingDown,
						Reason:    "stale-shutdown",
						Source:    "test",
						UpdatedAt: time.Now().UTC().Add(-2 * time.Hour),
					}); err != nil {
						t.Fatalf("seed stale shutdown intent: %v", err)
					}
					run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
						command := name + " " + strings.Join(args, " ")
						if name == "kubectl" && strings.Contains(command, "jsonpath={.spec.url}") {
							return "", errors.New("flux url read failed")
						}
						return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
					}
					orch := newLifecycleMatrixOrchestrator(t, cfg, false, run, run, "")
					if err := orch.Startup(context.Background(), cluster.StartupOptions{Reason: "stale-shutdown-clear"}); err == nil || !strings.Contains(err.Error(), "read flux source url") {
						t.Fatalf("expected startup flux-url guard error after stale shutdown clear, got %v", err)
					}
				})

				t.Run("auto-etcd-default-control-plane-selection", func(t *testing.T) {
					cfg := lifecycleFastConfig(t)
					cfg.Startup.AutoEtcdRestoreOnAPIFailure = true
					apiVersionCalls := 0
					run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
						command := name + " " + strings.Join(args, " ")
						switch {
						case name == "kubectl" && strings.Contains(command, "version --request-timeout=5s"):
							apiVersionCalls++
							if apiVersionCalls == 1 {
								return "", errors.New("api down")
							}
							return "v1.31.0", nil
						case name == "ssh" && strings.Contains(command, "systemctl cat k3s"):
							return "", errors.New("unit read failed")
						case name == "kubectl" && strings.Contains(command, "jsonpath={.spec.url}"):
							return "ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git", nil
						case name == "kubectl" && strings.Contains(command, "jsonpath={.spec.ref.branch}"):
							return "main", nil
						default:
							return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
						}
					}
					orch := newLifecycleMatrixOrchestrator(t, cfg, false, run, run, "")
					err := orch.Startup(context.Background(), cluster.StartupOptions{Reason: "auto-etcd-default-cp"})
					if err == nil || !strings.Contains(err.Error(), "automatic etcd restore failed") {
						t.Fatalf("expected startup auto-etcd restore failure path, got %v", err)
					}
				})

				t.Run("ensure-flux-branch-patch-error", func(t *testing.T) {
					cfg := lifecycleFastConfig(t)
					run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
						command := name + " " + strings.Join(args, " ")
						switch {
						case name == "kubectl" && strings.Contains(command, "version --request-timeout=5s"):
							return "v1.31.0", nil
						case name == "kubectl" && strings.Contains(command, "jsonpath={.spec.url}"):
							return "ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git", nil
						case name == "kubectl" && strings.Contains(command, "jsonpath={.spec.ref.branch}"):
							return "feature/sso", nil
						case name == "kubectl" && strings.Contains(command, "patch gitrepository flux-system"):
							return "", errors.New("branch patch failed")
						default:
							return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
						}
					}
					orch := newLifecycleMatrixOrchestrator(t, cfg, false, run, run, "")
					err := orch.Startup(context.Background(), cluster.StartupOptions{
						Reason:          "ensure-flux-branch-patch",
						ForceFluxBranch: "main",
					})
					if err == nil || !strings.Contains(err.Error(), "set flux source branch") {
						t.Fatalf("expected startup ensureFluxBranch patch error path, got %v", err)
					}
				})
			})

			t.Run("etcd-restore-and-shutdown-error-paths", func(t *testing.T) {
				cfg := lifecycleConfig(t)
				cfg.ControlPlanes = []string{"titan-db", "titan-24"}
				cfg.SSHManagedNodes = []string{"titan-db", "titan-24", "titan-23"}
				cfg.SSHNodeHosts["titan-24"] = "titan-24"
				cfg.SSHNodeHosts["titan-23"] = "titan-23"

				runEtcdModeErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
					command := name + " " + strings.Join(args, " ")
					if name == "ssh" && strings.Contains(command, "systemctl cat k3s") {
						return "", errors.New("unit read failed")
					}
					return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
				}
				orchEtcdModeErr := newLifecycleMatrixOrchestrator(t, cfg, false, runEtcdModeErr, runEtcdModeErr, "")
				if err := orchEtcdModeErr.EtcdRestore(context.Background(), cluster.EtcdRestoreOptions{ControlPlane: "titan-db"}); err == nil || !strings.Contains(err.Error(), "inspect k3s service on titan-db for datastore mode") {
					t.Fatalf("expected EtcdRestore datastore-mode error branch, got %v", err)
				}

				runLatestErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
					command := name + " " + strings.Join(args, " ")
					switch {
					case name == "ssh" && strings.Contains(command, "systemctl cat k3s"):
						return "ExecStart=/usr/local/bin/k3s server", nil
					case name == "ssh" && strings.Contains(command, "etcd-snapshot ls"):
						return "", errors.New("snapshot list failed")
					default:
						return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
					}
				}
				orchLatestErr := newLifecycleMatrixOrchestrator(t, cfg, false, runLatestErr, runLatestErr, "")
				if err := orchLatestErr.EtcdRestore(context.Background(), cluster.EtcdRestoreOptions{ControlPlane: "titan-db"}); err == nil || !strings.Contains(err.Error(), "resolve latest etcd snapshot") {
					t.Fatalf("expected EtcdRestore latest-snapshot error branch, got %v", err)
				}

				runVerifyErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
					command := name + " " + strings.Join(args, " ")
					switch {
					case name == "ssh" && strings.Contains(command, "systemctl cat k3s"):
						return "ExecStart=/usr/local/bin/k3s server", nil
					case name == "ssh" && strings.Contains(command, "stat -c %s"):
						return "1", nil
					default:
						return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
					}
				}
				orchVerifyErr := newLifecycleMatrixOrchestrator(t, cfg, false, runVerifyErr, runVerifyErr, "")
				if err := orchVerifyErr.EtcdRestore(context.Background(), cluster.EtcdRestoreOptions{
					ControlPlane: "titan-db",
					SnapshotPath: "/var/lib/rancher/k3s/server/db/snapshots/pre-shutdown",
				}); err == nil || !strings.Contains(err.Error(), "snapshot too small") {
					t.Fatalf("expected EtcdRestore verify-snapshot error branch, got %v", err)
				}

				runStartErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
					command := name + " " + strings.Join(args, " ")
					switch {
					case name == "ssh" && strings.Contains(command, "systemctl cat k3s"):
						return "ExecStart=/usr/local/bin/k3s server", nil
					case name == "ssh" && strings.Contains(command, "stat -c %s"):
						return "2097152", nil
					case name == "ssh" && strings.Contains(command, "sha256sum"):
						return "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", nil
					case name == "ssh" && strings.Contains(command, "etcd-snapshot ls"):
						return "/var/lib/rancher/k3s/server/db/snapshots/pre-shutdown", nil
					case name == "ssh" && strings.Contains(command, "server --cluster-reset"):
						return "", nil
					case name == "ssh" && strings.Contains(command, "sudo systemctl start k3s || true") && strings.Contains(command, "atlas@titan-db"):
						return "", errors.New("start failed")
					default:
						return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
					}
				}
				orchStartErr := newLifecycleMatrixOrchestrator(t, cfg, false, runStartErr, runStartErr, "")
				if err := orchStartErr.EtcdRestore(context.Background(), cluster.EtcdRestoreOptions{
					ControlPlane: "titan-db",
					SnapshotPath: "/var/lib/rancher/k3s/server/db/snapshots/pre-shutdown",
				}); err == nil || !strings.Contains(err.Error(), "failed to start k3s on restore control plane") {
					t.Fatalf("expected EtcdRestore restore-node start error branch, got %v", err)
				}

				cfgShutdownLock := lifecycleConfig(t)
				cfgShutdownLock.State.LockPath = t.TempDir()
				orchShutdownLock := newLifecycleMatrixOrchestrator(t, cfgShutdownLock, false, nil, nil, "")
				if err := orchShutdownLock.Shutdown(context.Background(), cluster.ShutdownOptions{Reason: "lock"}); err == nil {
					t.Fatalf("expected shutdown lock acquire failure branch")
				}

				cfgShutdownInventory := lifecycleConfig(t)
				cfgShutdownInventory.Workers = nil
				cfgShutdownInventory.SSHManagedNodes = []string{"titan-db"}
				cfgShutdownInventory.SSHNodeHosts = map[string]string{
					"titan-db": "titan-db",
				}
				runWorkersErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
					if name == "kubectl" && len(args) >= 2 && args[0] == "get" && args[1] == "nodes" {
						return "", errors.New("discover workers failed")
					}
					return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
				}
				orchWorkersErr := newLifecycleMatrixOrchestrator(t, cfgShutdownInventory, false, runWorkersErr, runWorkersErr, "")
				if err := orchWorkersErr.Shutdown(context.Background(), cluster.ShutdownOptions{Reason: "workers"}); err == nil || !strings.Contains(err.Error(), "discover workers") {
					t.Fatalf("expected shutdown effectiveWorkers error branch, got %v", err)
				}
			})
		})
	})
}