179 lines
8.0 KiB
Go
179 lines
8.0 KiB
Go
package orchestrator
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"scm.bstein.dev/bstein/ananke/internal/cluster"
|
|
"scm.bstein.dev/bstein/ananke/internal/config"
|
|
"scm.bstein.dev/bstein/ananke/internal/state"
|
|
)
|
|
|
|
// TestHookCoordinationLowFunctionMatrix runs one orchestration or CLI step.
|
|
// Signature: TestHookCoordinationLowFunctionMatrix(t *testing.T).
|
|
// Why: closes remaining coordination helper branches that are not naturally hit by startup drill tests.
|
|
func TestHookCoordinationLowFunctionMatrix(t *testing.T) {
|
|
old := state.Intent{UpdatedAt: time.Now().Add(-2 * time.Hour)}
|
|
if got := cluster.TestHookIntentAge(state.Intent{}); got != 0 {
|
|
t.Fatalf("expected zero age for zero timestamp, got %s", got)
|
|
}
|
|
if !cluster.TestHookIntentFresh(state.Intent{}, time.Second) {
|
|
t.Fatalf("expected zero timestamp intent to be treated as fresh")
|
|
}
|
|
if cluster.TestHookIntentFresh(old, 5*time.Minute) {
|
|
t.Fatalf("expected stale intent to be non-fresh")
|
|
}
|
|
|
|
cfg := lifecycleConfig(t)
|
|
orch, _ := newHookOrchestrator(t, cfg, nil, nil)
|
|
if _, err := orch.TestHookReadRemotePeerStatus(context.Background(), "not-managed"); err == nil {
|
|
t.Fatalf("expected unmanaged peer status error")
|
|
}
|
|
|
|
run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
|
command := name + " " + strings.Join(args, " ")
|
|
if name == "ssh" && strings.Contains(command, "systemctl cat k3s") {
|
|
return "", errors.New("permission denied")
|
|
}
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
|
}
|
|
orchErr, _ := newHookOrchestrator(t, cfg, run, run)
|
|
if _, err := orchErr.TestHookControlPlaneUsesExternalDatastore(context.Background(), "titan-db"); err == nil {
|
|
t.Fatalf("expected datastore mode inspection error")
|
|
}
|
|
}
|
|
|
|
// TestHookFluxHealthLowFunctionMatrix runs one orchestration or CLI step.
|
|
// Signature: TestHookFluxHealthLowFunctionMatrix(t *testing.T).
|
|
// Why: covers immutable-job and flux parser/helper branches to lift fluxhealth file coverage.
|
|
func TestHookFluxHealthLowFunctionMatrix(t *testing.T) {
|
|
if !cluster.TestHookLooksLikeImmutableJobError("Job update failed: field is immutable") {
|
|
t.Fatalf("expected immutable job matcher true")
|
|
}
|
|
if cluster.TestHookLooksLikeImmutableJobError("random failure") {
|
|
t.Fatalf("expected immutable job matcher false")
|
|
}
|
|
|
|
if !cluster.TestHookJobLooksFluxManaged("flux-system", "job-a", map[string]string{"kustomize.toolkit.fluxcd.io/name": "services"}, nil) {
|
|
t.Fatalf("expected flux-managed job by label")
|
|
}
|
|
if cluster.TestHookJobLooksFluxManaged("flux-system", "job-b", nil, []string{"CronJob"}) {
|
|
t.Fatalf("expected cronjob-owned job to be ignored")
|
|
}
|
|
|
|
if cluster.TestHookJobFailed(1, 1, []string{"Failed"}, []string{"True"}) {
|
|
t.Fatalf("expected succeeded job to not count as failed")
|
|
}
|
|
if cluster.TestHookJobFailed(1, 0, []string{"Complete"}, []string{"True"}) {
|
|
t.Fatalf("expected failed job without Failed=True condition to be false")
|
|
}
|
|
|
|
cfg := lifecycleConfig(t)
|
|
cfg.Startup.FluxHealthWaitSeconds = 1
|
|
cfg.Startup.FluxHealthPollSeconds = 1
|
|
run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
|
command := name + " " + strings.Join(args, " ")
|
|
switch {
|
|
case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
|
|
return "{bad json", nil
|
|
case name == "kubectl" && strings.Contains(command, "get jobs -A -o json"):
|
|
return "{bad json", nil
|
|
default:
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
|
}
|
|
}
|
|
orch, _ := newHookOrchestrator(t, cfg, run, run)
|
|
if _, _, err := orch.TestHookFluxHealthReady(context.Background()); err == nil {
|
|
t.Fatalf("expected fluxHealthReady decode error")
|
|
}
|
|
if _, err := orch.TestHookHealImmutableFluxJobs(context.Background()); err == nil {
|
|
t.Fatalf("expected healImmutableFluxJobs decode error")
|
|
}
|
|
cancelCtx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if err := orch.TestHookWaitForFluxHealth(cancelCtx); !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("expected waitForFluxHealth cancel branch, got %v", err)
|
|
}
|
|
}
|
|
|
|
// TestHookServiceStabilityLowFunctionMatrix runs one orchestration or CLI step.
|
|
// Signature: TestHookServiceStabilityLowFunctionMatrix(t *testing.T).
|
|
// Why: drives startup-stability failure/success branches across all gated checks.
|
|
func TestHookServiceStabilityLowFunctionMatrix(t *testing.T) {
|
|
cfg := lifecycleConfig(t)
|
|
cfg.Startup.RequireFluxHealth = true
|
|
cfg.Startup.RequireWorkloadConvergence = true
|
|
cfg.Startup.RequireServiceChecklist = true
|
|
cfg.Startup.RequireIngressChecklist = false
|
|
|
|
svc := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write([]byte("ok"))
|
|
}))
|
|
defer svc.Close()
|
|
cfg.Startup.ServiceChecklist = []config.ServiceChecklistCheck{
|
|
{Name: "svc", URL: svc.URL, AcceptedStatuses: []int{200}, TimeoutSeconds: 2},
|
|
}
|
|
|
|
run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
|
command := name + " " + strings.Join(args, " ")
|
|
switch {
|
|
case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
|
|
return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil
|
|
case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"):
|
|
return `{"items":[]}`, nil
|
|
case name == "kubectl" && strings.Contains(command, "get pods -A -o json"):
|
|
return `{"items":[]}`, nil
|
|
default:
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
|
}
|
|
}
|
|
orch, _ := newHookOrchestrator(t, cfg, run, run)
|
|
if err := orch.TestHookStartupStabilityHealthy(context.Background()); err != nil {
|
|
t.Fatalf("expected stability healthy success, got %v", err)
|
|
}
|
|
|
|
runFluxDown := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
|
command := name + " " + strings.Join(args, " ")
|
|
if name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json") {
|
|
return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","message":"syncing"}]}}]}`, nil
|
|
}
|
|
return run(ctx, timeout, name, args...)
|
|
}
|
|
orchFluxDown, _ := newHookOrchestrator(t, cfg, runFluxDown, runFluxDown)
|
|
if err := orchFluxDown.TestHookStartupStabilityHealthy(context.Background()); err == nil {
|
|
t.Fatalf("expected flux-not-ready stability failure")
|
|
}
|
|
}
|
|
|
|
// TestHookStorageTimesyncLowFunctionMatrix runs one orchestration or CLI step.
|
|
// Signature: TestHookStorageTimesyncLowFunctionMatrix(t *testing.T).
|
|
// Why: targets low parser and storage edge branches to increase readiness helper coverage.
|
|
func TestHookStorageTimesyncLowFunctionMatrix(t *testing.T) {
|
|
if got := cluster.TestHookParseDatastoreEndpoint("ExecStart=/usr/local/bin/k3s server --datastore-endpoint='postgres://db:5432/k3s' \\"); got != "postgres://db:5432/k3s" {
|
|
t.Fatalf("unexpected parsed endpoint from single-quoted arg: %q", got)
|
|
}
|
|
if got := cluster.TestHookParseDatastoreEndpoint("x --datastore-endpoint = \"postgres://db:5432/k3s\" \\"); got == "" {
|
|
t.Fatalf("expected non-empty parse result from spaced datastore flag")
|
|
}
|
|
|
|
cfg := lifecycleConfig(t)
|
|
cfg.Startup.StorageCriticalPVCs = []string{"invalid-entry"}
|
|
run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
|
command := name + " " + strings.Join(args, " ")
|
|
if name == "kubectl" && strings.Contains(command, "nodes.longhorn.io") {
|
|
return "titan-23:True:True\ntitan-24:True:True\n", nil
|
|
}
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
|
}
|
|
orch, _ := newHookOrchestrator(t, cfg, run, run)
|
|
if _, _, err := orch.TestHookStorageReady(context.Background()); err == nil {
|
|
t.Fatalf("expected invalid critical pvc entry error")
|
|
}
|
|
}
|