From c73b8fe8fd2d451f92083510109e8cdc35b4b7db Mon Sep 17 00:00:00 2001 From: codex Date: Sat, 3 Oct 2026 02:30:50 -0500 Subject: [PATCH] test: verify repair isolation without prolonged fixture waits --- .../daemon_poststart_autorepair_test.go | 32 +++++++++++++++++++ ...ooks_vault_lifecycle_branch_matrix_test.go | 9 ++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/internal/service/daemon_poststart_autorepair_test.go b/internal/service/daemon_poststart_autorepair_test.go index 9737ed1..9d81654 100644 --- a/internal/service/daemon_poststart_autorepair_test.go +++ b/internal/service/daemon_poststart_autorepair_test.go @@ -1,7 +1,14 @@ package service import ( + "bytes" "context" + "errors" + "log" + "strings" + + "scm.bstein.dev/bstein/ananke/internal/cluster" + "scm.bstein.dev/bstein/ananke/internal/execx" "testing" "time" @@ -55,3 +62,28 @@ func TestDaemonMaybeRunPostStartAutoHeal(t *testing.T) { t.Fatalf("expected second allowed auto-heal call, got %d", calls) } } + +// TestDaemonRecoveryFailureDoesNotStopMonitoring exercises optional repair isolation. +// Signature: TestDaemonRecoveryFailureDoesNotStopMonitoring(t *testing.T). +// Why: a missing or failed repair helper must not terminate UPS monitoring. +func TestDaemonRecoveryFailureDoesNotStopMonitoring(t *testing.T) { + var messages bytes.Buffer + d := &Daemon{cfg: config.Config{Startup: config.Startup{PostStartAutoHealSeconds: 60}}, log: log.New(&messages, "", 0)} + var last time.Time + d.maybeRunPostStartAutoHeal(context.Background(), &last, false) + if !last.IsZero() { + t.Fatal("missing helper must not consume an interval") + } + if err := d.runPostStartAutoHeal(context.Background()); err != nil { + t.Fatal(err) + } + d.orch = cluster.New(config.Config{}, &execx.Runner{DryRun: true}, nil, d.log) + if err := d.runPostStartAutoHeal(context.Background()); err != nil { + t.Fatal(err) + } + d.postStartAutoHealOverride = func(context.Context) error { return errors.New("synthetic repair failure") } + d.maybeRunPostStartAutoHeal(context.Background(), &last, false) + if last.IsZero() || !strings.Contains(messages.String(), "synthetic repair failure") { + t.Fatal("failed repair must be logged and rate limited") + } +} diff --git a/testing/orchestrator/hooks_vault_lifecycle_branch_matrix_test.go b/testing/orchestrator/hooks_vault_lifecycle_branch_matrix_test.go index 88aacc1..622089d 100644 --- a/testing/orchestrator/hooks_vault_lifecycle_branch_matrix_test.go +++ b/testing/orchestrator/hooks_vault_lifecycle_branch_matrix_test.go @@ -96,7 +96,7 @@ func TestHookVaultLifecycleBranchMatrix(t *testing.T) { } }) - t.Run("workload-ready-no-value-and-ensure-error", func(t *testing.T) { + t.Run("workload-ready-no-value-and-ensure-cancel", func(t *testing.T) { cfg := lifecycleConfig(t) runNoValue := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") @@ -111,6 +111,8 @@ func TestHookVaultLifecycleBranchMatrix(t *testing.T) { t.Fatalf("expected no-value readiness branch, ready=%v err=%v", ready, err) } + waitCtx, cancelWait := context.WithCancel(context.Background()) + defer cancelWait() runEnsureErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { @@ -119,14 +121,15 @@ func TestHookVaultLifecycleBranchMatrix(t *testing.T) { case name == "kubectl" && strings.Contains(command, "get pods -o custom-columns"): return "", nil case name == "kubectl" && strings.Contains(command, "rollout status statefulset/victoria-metrics-single-server"): + cancelWait() return "", errors.New("rollout failed") default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } } orchEnsureErr, _ := newHookOrchestratorWithRunnerMode(t, cfg, false, runEnsureErr, runEnsureErr) - if err := orchEnsureErr.TestHookEnsureCriticalStartupWorkloads(context.Background()); err == nil || !strings.Contains(err.Error(), "rollout failed") { - t.Fatalf("expected ensureCriticalStartupWorkloads wait error branch, got %v", err) + if err := orchEnsureErr.TestHookEnsureCriticalStartupWorkloads(waitCtx); !errors.Is(err, context.Canceled) { + t.Fatalf("expected ensureCriticalStartupWorkloads cancellation after failed rollout, got %v", err) } })