package orchestrator import ( "context" "errors" "net/http" "net/http/httptest" "strings" "testing" "time" "scm.bstein.dev/bstein/ananke/internal/cluster" "scm.bstein.dev/bstein/ananke/internal/config" "scm.bstein.dev/bstein/ananke/internal/state" ) // TestHookCoordinationLowFunctionMatrix runs one orchestration or CLI step. // Signature: TestHookCoordinationLowFunctionMatrix(t *testing.T). // Why: closes remaining coordination helper branches that are not naturally hit by startup drill tests. func TestHookCoordinationLowFunctionMatrix(t *testing.T) { old := state.Intent{UpdatedAt: time.Now().Add(-2 * time.Hour)} if got := cluster.TestHookIntentAge(state.Intent{}); got != 0 { t.Fatalf("expected zero age for zero timestamp, got %s", got) } if !cluster.TestHookIntentFresh(state.Intent{}, time.Second) { t.Fatalf("expected zero timestamp intent to be treated as fresh") } if cluster.TestHookIntentFresh(old, 5*time.Minute) { t.Fatalf("expected stale intent to be non-fresh") } cfg := lifecycleConfig(t) orch, _ := newHookOrchestrator(t, cfg, nil, nil) if _, err := orch.TestHookReadRemotePeerStatus(context.Background(), "not-managed"); err == nil { t.Fatalf("expected unmanaged peer status error") } run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") if name == "ssh" && strings.Contains(command, "systemctl cat k3s") { return "", errors.New("permission denied") } return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } orchErr, _ := newHookOrchestrator(t, cfg, run, run) if _, err := orchErr.TestHookControlPlaneUsesExternalDatastore(context.Background(), "titan-db"); err == nil { t.Fatalf("expected datastore mode inspection error") } } // TestHookFluxHealthLowFunctionMatrix runs one orchestration or CLI step. // Signature: TestHookFluxHealthLowFunctionMatrix(t *testing.T). // Why: covers immutable-job and flux parser/helper branches to lift fluxhealth file coverage. func TestHookFluxHealthLowFunctionMatrix(t *testing.T) { if !cluster.TestHookLooksLikeImmutableJobError("Job update failed: field is immutable") { t.Fatalf("expected immutable job matcher true") } if cluster.TestHookLooksLikeImmutableJobError("random failure") { t.Fatalf("expected immutable job matcher false") } if !cluster.TestHookJobLooksFluxManaged("flux-system", "job-a", map[string]string{"kustomize.toolkit.fluxcd.io/name": "services"}, nil) { t.Fatalf("expected flux-managed job by label") } if cluster.TestHookJobLooksFluxManaged("flux-system", "job-b", nil, []string{"CronJob"}) { t.Fatalf("expected cronjob-owned job to be ignored") } if cluster.TestHookJobFailed(1, 1, []string{"Failed"}, []string{"True"}) { t.Fatalf("expected succeeded job to not count as failed") } if cluster.TestHookJobFailed(1, 0, []string{"Complete"}, []string{"True"}) { t.Fatalf("expected failed job without Failed=True condition to be false") } cfg := lifecycleConfig(t) cfg.Startup.FluxHealthWaitSeconds = 1 cfg.Startup.FluxHealthPollSeconds = 1 run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"): return "{bad json", nil case name == "kubectl" && strings.Contains(command, "get jobs -A -o json"): return "{bad json", nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orch, _ := newHookOrchestrator(t, cfg, run, run) if _, _, err := orch.TestHookFluxHealthReady(context.Background()); err == nil { t.Fatalf("expected fluxHealthReady decode error") } if _, err := orch.TestHookHealImmutableFluxJobs(context.Background()); err == nil { t.Fatalf("expected healImmutableFluxJobs decode error") } cancelCtx, cancel := context.WithCancel(context.Background()) cancel() if err := orch.TestHookWaitForFluxHealth(cancelCtx); !errors.Is(err, context.Canceled) { t.Fatalf("expected waitForFluxHealth cancel branch, got %v", err) } } // TestHookServiceStabilityLowFunctionMatrix runs one orchestration or CLI step. // Signature: TestHookServiceStabilityLowFunctionMatrix(t *testing.T). // Why: drives startup-stability failure/success branches across all gated checks. func TestHookServiceStabilityLowFunctionMatrix(t *testing.T) { cfg := lifecycleConfig(t) cfg.Startup.RequireFluxHealth = true cfg.Startup.RequireWorkloadConvergence = true cfg.Startup.RequireServiceChecklist = true cfg.Startup.RequireIngressChecklist = false svc := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("ok")) })) defer svc.Close() cfg.Startup.ServiceChecklist = []config.ServiceChecklistCheck{ {Name: "svc", URL: svc.URL, AcceptedStatuses: []int{200}, TimeoutSeconds: 2}, } run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"): return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"): return `{"items":[]}`, nil case name == "kubectl" && strings.Contains(command, "get pods -A -o json"): return `{"items":[]}`, nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orch, _ := newHookOrchestrator(t, cfg, run, run) if err := orch.TestHookStartupStabilityHealthy(context.Background()); err != nil { t.Fatalf("expected stability healthy success, got %v", err) } runFluxDown := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") if name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json") { return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","message":"syncing"}]}}]}`, nil } return run(ctx, timeout, name, args...) } orchFluxDown, _ := newHookOrchestrator(t, cfg, runFluxDown, runFluxDown) if err := orchFluxDown.TestHookStartupStabilityHealthy(context.Background()); err == nil { t.Fatalf("expected flux-not-ready stability failure") } } // TestHookStorageTimesyncLowFunctionMatrix runs one orchestration or CLI step. // Signature: TestHookStorageTimesyncLowFunctionMatrix(t *testing.T). // Why: targets low parser and storage edge branches to increase readiness helper coverage. func TestHookStorageTimesyncLowFunctionMatrix(t *testing.T) { if got := cluster.TestHookParseDatastoreEndpoint("ExecStart=/usr/local/bin/k3s server --datastore-endpoint='postgres://db:5432/k3s' \\"); got != "postgres://db:5432/k3s" { t.Fatalf("unexpected parsed endpoint from single-quoted arg: %q", got) } if got := cluster.TestHookParseDatastoreEndpoint("x --datastore-endpoint = \"postgres://db:5432/k3s\" \\"); got == "" { t.Fatalf("expected non-empty parse result from spaced datastore flag") } cfg := lifecycleConfig(t) cfg.Startup.StorageCriticalPVCs = []string{"invalid-entry"} run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") if name == "kubectl" && strings.Contains(command, "nodes.longhorn.io") { return "titan-23:True:True\ntitan-24:True:True\n", nil } return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } orch, _ := newHookOrchestrator(t, cfg, run, run) if _, _, err := orch.TestHookStorageReady(context.Background()); err == nil { t.Fatalf("expected invalid critical pvc entry error") } }