titan/ananke
3
0
Fork 0
Code Issues 222 Pull Requests 3 Actions Packages Projects Releases Wiki Activity

test: verify repair isolation without prolonged fixture waits

Browse Source
This commit is contained in:
codex 2026-10-03 02:30:50 -05:00
parent bbcf285514
commit c73b8fe8fd
2 changed files with 38 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
internal/service/daemon_poststart_autorepair_test.go
View File

@ -1,7 +1,14 @@
package service package service
import ( import (
"bytes"
"context" "context"
"errors"
"log"
"strings"
"scm.bstein.dev/bstein/ananke/internal/cluster"
"scm.bstein.dev/bstein/ananke/internal/execx"
"testing" "testing"
"time" "time"
@ -55,3 +62,28 @@ func TestDaemonMaybeRunPostStartAutoHeal(t *testing.T) {
t.Fatalf("expected second allowed auto-heal call, got %d", calls) t.Fatalf("expected second allowed auto-heal call, got %d", calls)
} }
} }
// TestDaemonRecoveryFailureDoesNotStopMonitoring exercises optional repair isolation.
// Signature: TestDaemonRecoveryFailureDoesNotStopMonitoring(t *testing.T).
// Why: a missing or failed repair helper must not terminate UPS monitoring.
func TestDaemonRecoveryFailureDoesNotStopMonitoring(t *testing.T) {
var messages bytes.Buffer
d := &Daemon{cfg: config.Config{Startup: config.Startup{PostStartAutoHealSeconds: 60}}, log: log.New(&messages, "", 0)}
var last time.Time
d.maybeRunPostStartAutoHeal(context.Background(), &last, false)
if !last.IsZero() {
t.Fatal("missing helper must not consume an interval")
}
if err := d.runPostStartAutoHeal(context.Background()); err != nil {
t.Fatal(err)
}
d.orch = cluster.New(config.Config{}, &execx.Runner{DryRun: true}, nil, d.log)
if err := d.runPostStartAutoHeal(context.Background()); err != nil {
t.Fatal(err)
}
d.postStartAutoHealOverride = func(context.Context) error { return errors.New("synthetic repair failure") }
d.maybeRunPostStartAutoHeal(context.Background(), &last, false)
if last.IsZero() || !strings.Contains(messages.String(), "synthetic repair failure") {
t.Fatal("failed repair must be logged and rate limited")
}
}

9
testing/orchestrator/hooks_vault_lifecycle_branch_matrix_test.go
View File

@ -96,7 +96,7 @@ func TestHookVaultLifecycleBranchMatrix(t *testing.T) {
} }
}) })
t.Run("workload-ready-no-value-and-ensure-error", func(t *testing.T) { t.Run("workload-ready-no-value-and-ensure-cancel", func(t *testing.T) {
cfg := lifecycleConfig(t) cfg := lifecycleConfig(t)
runNoValue := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { runNoValue := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
command := name + " " + strings.Join(args, " ") command := name + " " + strings.Join(args, " ")
@ -111,6 +111,8 @@ func TestHookVaultLifecycleBranchMatrix(t *testing.T) {
t.Fatalf("expected no-value readiness branch, ready=%v err=%v", ready, err) t.Fatalf("expected no-value readiness branch, ready=%v err=%v", ready, err)
} }
waitCtx, cancelWait := context.WithCancel(context.Background())
defer cancelWait()
runEnsureErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { runEnsureErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
command := name + " " + strings.Join(args, " ") command := name + " " + strings.Join(args, " ")
switch { switch {
@ -119,14 +121,15 @@ func TestHookVaultLifecycleBranchMatrix(t *testing.T) {
case name == "kubectl" && strings.Contains(command, "get pods -o custom-columns"): case name == "kubectl" && strings.Contains(command, "get pods -o custom-columns"):
return "", nil return "", nil
case name == "kubectl" && strings.Contains(command, "rollout status statefulset/victoria-metrics-single-server"): case name == "kubectl" && strings.Contains(command, "rollout status statefulset/victoria-metrics-single-server"):
cancelWait()
return "", errors.New("rollout failed") return "", errors.New("rollout failed")
default: default:
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
} }
} }
orchEnsureErr, _ := newHookOrchestratorWithRunnerMode(t, cfg, false, runEnsureErr, runEnsureErr) orchEnsureErr, _ := newHookOrchestratorWithRunnerMode(t, cfg, false, runEnsureErr, runEnsureErr)
if err := orchEnsureErr.TestHookEnsureCriticalStartupWorkloads(context.Background()); err == nil || !strings.Contains(err.Error(), "rollout failed") { if err := orchEnsureErr.TestHookEnsureCriticalStartupWorkloads(waitCtx); !errors.Is(err, context.Canceled) {
t.Fatalf("expected ensureCriticalStartupWorkloads wait error branch, got %v", err) t.Fatalf("expected ensureCriticalStartupWorkloads cancellation after failed rollout, got %v", err)
} }
}) })