test(ananke): make startup cooldown coverage deterministic
This commit is contained in:
parent
aabcf83c5f
commit
bc65124db0
@ -22,12 +22,23 @@ type Intent struct {
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
var writeIntentImpl = writeIntentDefault
|
||||
var (
|
||||
readIntentImpl = readIntentDefault
|
||||
writeIntentImpl = writeIntentDefault
|
||||
)
|
||||
|
||||
// ReadIntent runs one orchestration or CLI step.
|
||||
// Signature: ReadIntent(path string) (Intent, error).
|
||||
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
|
||||
func ReadIntent(path string) (Intent, error) {
|
||||
return readIntentImpl(path)
|
||||
}
|
||||
|
||||
// readIntentDefault runs one orchestration or CLI step.
|
||||
// Signature: readIntentDefault(path string) (Intent, error).
|
||||
// Why: keeps production read behavior available while tests can override intent
|
||||
// reads deterministically without racing background file mutations.
|
||||
func readIntentDefault(path string) (Intent, error) {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
|
||||
@ -22,6 +22,34 @@ func TestHookWriteIntentDefault(path string, in Intent) error {
|
||||
return writeIntentDefault(path, in)
|
||||
}
|
||||
|
||||
// TestHookReadIntentDefault runs one orchestration or CLI step.
|
||||
// Signature: TestHookReadIntentDefault(path string) (Intent, error).
|
||||
// Why: lets top-level tests delegate to production ReadIntent behavior while
|
||||
// selectively forcing deterministic read sequences for lifecycle branches.
|
||||
func TestHookReadIntentDefault(path string) (Intent, error) {
|
||||
return readIntentDefault(path)
|
||||
}
|
||||
|
||||
// TestHookSetReadIntentOverride runs one orchestration or CLI step.
|
||||
// Signature: TestHookSetReadIntentOverride(fn func(path string) (Intent, error)) (restore func()).
|
||||
// Why: enables deterministic intent-read failure injection without sleeping
|
||||
// goroutines that race slower CI agents.
|
||||
func TestHookSetReadIntentOverride(fn func(path string) (Intent, error)) (restore func()) {
|
||||
testHookOverrideMu.Lock()
|
||||
prev := readIntentImpl
|
||||
if fn == nil {
|
||||
readIntentImpl = readIntentDefault
|
||||
} else {
|
||||
readIntentImpl = fn
|
||||
}
|
||||
testHookOverrideMu.Unlock()
|
||||
return func() {
|
||||
testHookOverrideMu.Lock()
|
||||
readIntentImpl = prev
|
||||
testHookOverrideMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// TestHookSetWriteIntentOverride runs one orchestration or CLI step.
|
||||
// Signature: TestHookSetWriteIntentOverride(fn func(path string, in Intent) error) (restore func()).
|
||||
// Why: enables deterministic intent-write failure injection from the top-level
|
||||
|
||||
@ -4,7 +4,6 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@ -127,19 +126,23 @@ func TestLifecycleDeepFailureMatrix(t *testing.T) {
|
||||
cfg := lifecycleFastConfig(t)
|
||||
cfg.Startup.RequireNodeInventoryReach = false
|
||||
cfg.Startup.ShutdownCooldownSeconds = 5
|
||||
if err := state.WriteIntent(cfg.State.IntentPath, state.Intent{
|
||||
reads := 0
|
||||
restoreRead := state.TestHookSetReadIntentOverride(func(path string) (state.Intent, error) {
|
||||
if path != cfg.State.IntentPath {
|
||||
return state.TestHookReadIntentDefault(path)
|
||||
}
|
||||
reads++
|
||||
if reads == 1 {
|
||||
return state.Intent{
|
||||
State: state.IntentShutdownComplete,
|
||||
Reason: "recent",
|
||||
Source: "test",
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed cooldown intent: %v", err)
|
||||
UpdatedAt: time.Now().UTC().Add(-4 * time.Second),
|
||||
}, nil
|
||||
}
|
||||
go func(intentPath string) {
|
||||
time.Sleep(2 * time.Second)
|
||||
_ = os.Remove(intentPath)
|
||||
_ = os.Mkdir(intentPath, 0o755)
|
||||
}(cfg.State.IntentPath)
|
||||
return state.Intent{}, errors.New("forced reread failure")
|
||||
})
|
||||
t.Cleanup(restoreRead)
|
||||
orch, _ := newHookOrchestrator(t, cfg, nil, nil)
|
||||
err := orch.Startup(context.Background(), cluster.StartupOptions{Reason: "cooldown-reread"})
|
||||
if err == nil || !strings.Contains(err.Error(), "re-read startup intent after cooldown wait") {
|
||||
@ -151,23 +154,28 @@ func TestLifecycleDeepFailureMatrix(t *testing.T) {
|
||||
cfg := lifecycleFastConfig(t)
|
||||
cfg.Startup.RequireNodeInventoryReach = false
|
||||
cfg.Startup.ShutdownCooldownSeconds = 5
|
||||
if err := state.WriteIntent(cfg.State.IntentPath, state.Intent{
|
||||
reads := 0
|
||||
restoreRead := state.TestHookSetReadIntentOverride(func(path string) (state.Intent, error) {
|
||||
if path != cfg.State.IntentPath {
|
||||
return state.TestHookReadIntentDefault(path)
|
||||
}
|
||||
reads++
|
||||
if reads == 1 {
|
||||
return state.Intent{
|
||||
State: state.IntentShutdownComplete,
|
||||
Reason: "recent",
|
||||
Source: "test",
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}); err != nil {
|
||||
t.Fatalf("seed cooldown intent: %v", err)
|
||||
UpdatedAt: time.Now().UTC().Add(-4 * time.Second),
|
||||
}, nil
|
||||
}
|
||||
go func(intentPath string) {
|
||||
time.Sleep(2 * time.Second)
|
||||
_ = state.WriteIntent(intentPath, state.Intent{
|
||||
return state.Intent{
|
||||
State: state.IntentShuttingDown,
|
||||
Reason: "peer-shutdown",
|
||||
Source: "test",
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}, nil
|
||||
})
|
||||
}(cfg.State.IntentPath)
|
||||
t.Cleanup(restoreRead)
|
||||
orch, _ := newHookOrchestrator(t, cfg, nil, nil)
|
||||
err := orch.Startup(context.Background(), cluster.StartupOptions{Reason: "cooldown-peer-active"})
|
||||
if err == nil || !strings.Contains(err.Error(), "shutdown intent became active during cooldown wait") {
|
||||
|
||||
@ -24,12 +24,36 @@ func TestStateTestHookOverrideSetters(t *testing.T) {
|
||||
}
|
||||
restoreWriteNil()
|
||||
|
||||
restoreReadNil := state.TestHookSetReadIntentOverride(nil)
|
||||
readAfterNil, err := state.ReadIntent(intentPath)
|
||||
if err != nil || readAfterNil.State != state.IntentNormal {
|
||||
t.Fatalf("expected default read intent path after nil override, got %v / %v", readAfterNil, err)
|
||||
}
|
||||
restoreReadNil()
|
||||
|
||||
readOverrideCalled := false
|
||||
restoreRead := state.TestHookSetReadIntentOverride(func(path string) (state.Intent, error) {
|
||||
readOverrideCalled = true
|
||||
return state.Intent{}, errors.New("forced read override")
|
||||
})
|
||||
_, err = state.ReadIntent(intentPath)
|
||||
if err == nil || !strings.Contains(err.Error(), "forced read override") {
|
||||
t.Fatalf("expected forced read override error, got %v", err)
|
||||
}
|
||||
if !readOverrideCalled {
|
||||
t.Fatalf("expected read override to be invoked")
|
||||
}
|
||||
restoreRead()
|
||||
if _, err := state.TestHookReadIntentDefault(intentPath); err != nil {
|
||||
t.Fatalf("expected explicit default read helper to succeed, got %v", err)
|
||||
}
|
||||
|
||||
writeOverrideCalled := false
|
||||
restoreWrite := state.TestHookSetWriteIntentOverride(func(path string, in state.Intent) error {
|
||||
writeOverrideCalled = true
|
||||
return errors.New("forced write override")
|
||||
})
|
||||
err := state.WriteIntent(intentPath, state.Intent{State: state.IntentNormal})
|
||||
err = state.WriteIntent(intentPath, state.Intent{State: state.IntentNormal})
|
||||
if err == nil || !strings.Contains(err.Error(), "forced write override") {
|
||||
t.Fatalf("expected forced write override error, got %v", err)
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user