ananke/testing/orchestrator/hooks_lowfunc_matrix_test.go

package orchestrator

import (
	"context"
	"errors"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"scm.bstein.dev/bstein/ananke/internal/cluster"
	"scm.bstein.dev/bstein/ananke/internal/config"
	"scm.bstein.dev/bstein/ananke/internal/state"
)

// TestHookCoordinationLowFunctionMatrix runs one orchestration or CLI step.
// Signature: TestHookCoordinationLowFunctionMatrix(t *testing.T).
// Why: closes remaining coordination helper branches that are not naturally hit by startup drill tests.
func TestHookCoordinationLowFunctionMatrix(t *testing.T) {
	old := state.Intent{UpdatedAt: time.Now().Add(-2 * time.Hour)}
	if got := cluster.TestHookIntentAge(state.Intent{}); got != 0 {
		t.Fatalf("expected zero age for zero timestamp, got %s", got)
	}
	if !cluster.TestHookIntentFresh(state.Intent{}, time.Second) {
		t.Fatalf("expected zero timestamp intent to be treated as fresh")
	}
	if cluster.TestHookIntentFresh(old, 5*time.Minute) {
		t.Fatalf("expected stale intent to be non-fresh")
	}

	cfg := lifecycleConfig(t)
	orch, _ := newHookOrchestrator(t, cfg, nil, nil)
	if _, err := orch.TestHookReadRemotePeerStatus(context.Background(), "not-managed"); err == nil {
		t.Fatalf("expected unmanaged peer status error")
	}

	run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
		command := name + " " + strings.Join(args, " ")
		if name == "ssh" && strings.Contains(command, "systemctl cat k3s") {
			return "", errors.New("permission denied")
		}
		return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
	}
	orchErr, _ := newHookOrchestrator(t, cfg, run, run)
	if _, err := orchErr.TestHookControlPlaneUsesExternalDatastore(context.Background(), "titan-db"); err == nil {
		t.Fatalf("expected datastore mode inspection error")
	}
}

// TestHookFluxHealthLowFunctionMatrix runs one orchestration or CLI step.
// Signature: TestHookFluxHealthLowFunctionMatrix(t *testing.T).
// Why: covers immutable-job and flux parser/helper branches to lift fluxhealth file coverage.
func TestHookFluxHealthLowFunctionMatrix(t *testing.T) {
	if !cluster.TestHookLooksLikeImmutableJobError("Job update failed: field is immutable") {
		t.Fatalf("expected immutable job matcher true")
	}
	if cluster.TestHookLooksLikeImmutableJobError("random failure") {
		t.Fatalf("expected immutable job matcher false")
	}

	if !cluster.TestHookJobLooksFluxManaged("flux-system", "job-a", map[string]string{"kustomize.toolkit.fluxcd.io/name": "services"}, nil) {
		t.Fatalf("expected flux-managed job by label")
	}
	if cluster.TestHookJobLooksFluxManaged("flux-system", "job-b", nil, []string{"CronJob"}) {
		t.Fatalf("expected cronjob-owned job to be ignored")
	}

	if cluster.TestHookJobFailed(1, 1, []string{"Failed"}, []string{"True"}) {
		t.Fatalf("expected succeeded job to not count as failed")
	}
	if cluster.TestHookJobFailed(1, 0, []string{"Complete"}, []string{"True"}) {
		t.Fatalf("expected failed job without Failed=True condition to be false")
	}

	cfg := lifecycleConfig(t)
	cfg.Startup.FluxHealthWaitSeconds = 1
	cfg.Startup.FluxHealthPollSeconds = 1
	run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
		command := name + " " + strings.Join(args, " ")
		switch {
		case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
			return "{bad json", nil
		case name == "kubectl" && strings.Contains(command, "get jobs -A -o json"):
			return "{bad json", nil
		default:
			return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
		}
	}
	orch, _ := newHookOrchestrator(t, cfg, run, run)
	if _, _, err := orch.TestHookFluxHealthReady(context.Background()); err == nil {
		t.Fatalf("expected fluxHealthReady decode error")
	}
	if _, err := orch.TestHookHealImmutableFluxJobs(context.Background()); err == nil {
		t.Fatalf("expected healImmutableFluxJobs decode error")
	}
	cancelCtx, cancel := context.WithCancel(context.Background())
	cancel()
	if err := orch.TestHookWaitForFluxHealth(cancelCtx); !errors.Is(err, context.Canceled) {
		t.Fatalf("expected waitForFluxHealth cancel branch, got %v", err)
	}
}

// TestHookServiceStabilityLowFunctionMatrix runs one orchestration or CLI step.
// Signature: TestHookServiceStabilityLowFunctionMatrix(t *testing.T).
// Why: drives startup-stability failure/success branches across all gated checks.
func TestHookServiceStabilityLowFunctionMatrix(t *testing.T) {
	cfg := lifecycleConfig(t)
	cfg.Startup.RequireFluxHealth = true
	cfg.Startup.RequireWorkloadConvergence = true
	cfg.Startup.RequireServiceChecklist = true
	cfg.Startup.RequireIngressChecklist = false

	svc := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write([]byte("ok"))
	}))
	defer svc.Close()
	cfg.Startup.ServiceChecklist = []config.ServiceChecklistCheck{
		{Name: "svc", URL: svc.URL, AcceptedStatuses: []int{200}, TimeoutSeconds: 2},
	}

	run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
		command := name + " " + strings.Join(args, " ")
		switch {
		case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
			return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil
		case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"):
			return `{"items":[]}`, nil
		case name == "kubectl" && strings.Contains(command, "get pods -A -o json"):
			return `{"items":[]}`, nil
		default:
			return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
		}
	}
	orch, _ := newHookOrchestrator(t, cfg, run, run)
	if err := orch.TestHookStartupStabilityHealthy(context.Background()); err != nil {
		t.Fatalf("expected stability healthy success, got %v", err)
	}

	runFluxDown := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
		command := name + " " + strings.Join(args, " ")
		if name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json") {
			return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","message":"syncing"}]}}]}`, nil
		}
		return run(ctx, timeout, name, args...)
	}
	orchFluxDown, _ := newHookOrchestrator(t, cfg, runFluxDown, runFluxDown)
	if err := orchFluxDown.TestHookStartupStabilityHealthy(context.Background()); err == nil {
		t.Fatalf("expected flux-not-ready stability failure")
	}
}

// TestHookStorageTimesyncLowFunctionMatrix runs one orchestration or CLI step.
// Signature: TestHookStorageTimesyncLowFunctionMatrix(t *testing.T).
// Why: targets low parser and storage edge branches to increase readiness helper coverage.
func TestHookStorageTimesyncLowFunctionMatrix(t *testing.T) {
	if got := cluster.TestHookParseDatastoreEndpoint("ExecStart=/usr/local/bin/k3s server --datastore-endpoint='postgres://db:5432/k3s' \\"); got != "postgres://db:5432/k3s" {
		t.Fatalf("unexpected parsed endpoint from single-quoted arg: %q", got)
	}
	if got := cluster.TestHookParseDatastoreEndpoint("x --datastore-endpoint = \"postgres://db:5432/k3s\" \\"); got == "" {
		t.Fatalf("expected non-empty parse result from spaced datastore flag")
	}

	cfg := lifecycleConfig(t)
	cfg.Startup.StorageCriticalPVCs = []string{"invalid-entry"}
	run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
		command := name + " " + strings.Join(args, " ")
		if name == "kubectl" && strings.Contains(command, "nodes.longhorn.io") {
			return "titan-23:True:True\ntitan-24:True:True\n", nil
		}
		return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
	}
	orch, _ := newHookOrchestrator(t, cfg, run, run)
	if _, _, err := orch.TestHookStorageReady(context.Background()); err == nil {
		t.Fatalf("expected invalid critical pvc entry error")
	}
}