test(ananke): make startup cooldown coverage deterministic
This commit is contained in:
parent
aabcf83c5f
commit
bc65124db0
@ -22,12 +22,23 @@ type Intent struct {
|
|||||||
UpdatedAt time.Time `json:"updated_at"`
|
UpdatedAt time.Time `json:"updated_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var writeIntentImpl = writeIntentDefault
|
var (
|
||||||
|
readIntentImpl = readIntentDefault
|
||||||
|
writeIntentImpl = writeIntentDefault
|
||||||
|
)
|
||||||
|
|
||||||
// ReadIntent runs one orchestration or CLI step.
|
// ReadIntent runs one orchestration or CLI step.
|
||||||
// Signature: ReadIntent(path string) (Intent, error).
|
// Signature: ReadIntent(path string) (Intent, error).
|
||||||
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
|
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
|
||||||
func ReadIntent(path string) (Intent, error) {
|
func ReadIntent(path string) (Intent, error) {
|
||||||
|
return readIntentImpl(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readIntentDefault runs one orchestration or CLI step.
|
||||||
|
// Signature: readIntentDefault(path string) (Intent, error).
|
||||||
|
// Why: keeps production read behavior available while tests can override intent
|
||||||
|
// reads deterministically without racing background file mutations.
|
||||||
|
func readIntentDefault(path string) (Intent, error) {
|
||||||
b, err := os.ReadFile(path)
|
b, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if os.IsNotExist(err) {
|
if os.IsNotExist(err) {
|
||||||
|
|||||||
@ -22,6 +22,34 @@ func TestHookWriteIntentDefault(path string, in Intent) error {
|
|||||||
return writeIntentDefault(path, in)
|
return writeIntentDefault(path, in)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestHookReadIntentDefault runs one orchestration or CLI step.
|
||||||
|
// Signature: TestHookReadIntentDefault(path string) (Intent, error).
|
||||||
|
// Why: lets top-level tests delegate to production ReadIntent behavior while
|
||||||
|
// selectively forcing deterministic read sequences for lifecycle branches.
|
||||||
|
func TestHookReadIntentDefault(path string) (Intent, error) {
|
||||||
|
return readIntentDefault(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHookSetReadIntentOverride runs one orchestration or CLI step.
|
||||||
|
// Signature: TestHookSetReadIntentOverride(fn func(path string) (Intent, error)) (restore func()).
|
||||||
|
// Why: enables deterministic intent-read failure injection without sleeping
|
||||||
|
// goroutines that race slower CI agents.
|
||||||
|
func TestHookSetReadIntentOverride(fn func(path string) (Intent, error)) (restore func()) {
|
||||||
|
testHookOverrideMu.Lock()
|
||||||
|
prev := readIntentImpl
|
||||||
|
if fn == nil {
|
||||||
|
readIntentImpl = readIntentDefault
|
||||||
|
} else {
|
||||||
|
readIntentImpl = fn
|
||||||
|
}
|
||||||
|
testHookOverrideMu.Unlock()
|
||||||
|
return func() {
|
||||||
|
testHookOverrideMu.Lock()
|
||||||
|
readIntentImpl = prev
|
||||||
|
testHookOverrideMu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestHookSetWriteIntentOverride runs one orchestration or CLI step.
|
// TestHookSetWriteIntentOverride runs one orchestration or CLI step.
|
||||||
// Signature: TestHookSetWriteIntentOverride(fn func(path string, in Intent) error) (restore func()).
|
// Signature: TestHookSetWriteIntentOverride(fn func(path string, in Intent) error) (restore func()).
|
||||||
// Why: enables deterministic intent-write failure injection from the top-level
|
// Why: enables deterministic intent-write failure injection from the top-level
|
||||||
|
|||||||
@ -4,7 +4,6 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"net"
|
"net"
|
||||||
"os"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@ -127,19 +126,23 @@ func TestLifecycleDeepFailureMatrix(t *testing.T) {
|
|||||||
cfg := lifecycleFastConfig(t)
|
cfg := lifecycleFastConfig(t)
|
||||||
cfg.Startup.RequireNodeInventoryReach = false
|
cfg.Startup.RequireNodeInventoryReach = false
|
||||||
cfg.Startup.ShutdownCooldownSeconds = 5
|
cfg.Startup.ShutdownCooldownSeconds = 5
|
||||||
if err := state.WriteIntent(cfg.State.IntentPath, state.Intent{
|
reads := 0
|
||||||
State: state.IntentShutdownComplete,
|
restoreRead := state.TestHookSetReadIntentOverride(func(path string) (state.Intent, error) {
|
||||||
Reason: "recent",
|
if path != cfg.State.IntentPath {
|
||||||
Source: "test",
|
return state.TestHookReadIntentDefault(path)
|
||||||
UpdatedAt: time.Now().UTC(),
|
}
|
||||||
}); err != nil {
|
reads++
|
||||||
t.Fatalf("seed cooldown intent: %v", err)
|
if reads == 1 {
|
||||||
}
|
return state.Intent{
|
||||||
go func(intentPath string) {
|
State: state.IntentShutdownComplete,
|
||||||
time.Sleep(2 * time.Second)
|
Reason: "recent",
|
||||||
_ = os.Remove(intentPath)
|
Source: "test",
|
||||||
_ = os.Mkdir(intentPath, 0o755)
|
UpdatedAt: time.Now().UTC().Add(-4 * time.Second),
|
||||||
}(cfg.State.IntentPath)
|
}, nil
|
||||||
|
}
|
||||||
|
return state.Intent{}, errors.New("forced reread failure")
|
||||||
|
})
|
||||||
|
t.Cleanup(restoreRead)
|
||||||
orch, _ := newHookOrchestrator(t, cfg, nil, nil)
|
orch, _ := newHookOrchestrator(t, cfg, nil, nil)
|
||||||
err := orch.Startup(context.Background(), cluster.StartupOptions{Reason: "cooldown-reread"})
|
err := orch.Startup(context.Background(), cluster.StartupOptions{Reason: "cooldown-reread"})
|
||||||
if err == nil || !strings.Contains(err.Error(), "re-read startup intent after cooldown wait") {
|
if err == nil || !strings.Contains(err.Error(), "re-read startup intent after cooldown wait") {
|
||||||
@ -151,23 +154,28 @@ func TestLifecycleDeepFailureMatrix(t *testing.T) {
|
|||||||
cfg := lifecycleFastConfig(t)
|
cfg := lifecycleFastConfig(t)
|
||||||
cfg.Startup.RequireNodeInventoryReach = false
|
cfg.Startup.RequireNodeInventoryReach = false
|
||||||
cfg.Startup.ShutdownCooldownSeconds = 5
|
cfg.Startup.ShutdownCooldownSeconds = 5
|
||||||
if err := state.WriteIntent(cfg.State.IntentPath, state.Intent{
|
reads := 0
|
||||||
State: state.IntentShutdownComplete,
|
restoreRead := state.TestHookSetReadIntentOverride(func(path string) (state.Intent, error) {
|
||||||
Reason: "recent",
|
if path != cfg.State.IntentPath {
|
||||||
Source: "test",
|
return state.TestHookReadIntentDefault(path)
|
||||||
UpdatedAt: time.Now().UTC(),
|
}
|
||||||
}); err != nil {
|
reads++
|
||||||
t.Fatalf("seed cooldown intent: %v", err)
|
if reads == 1 {
|
||||||
}
|
return state.Intent{
|
||||||
go func(intentPath string) {
|
State: state.IntentShutdownComplete,
|
||||||
time.Sleep(2 * time.Second)
|
Reason: "recent",
|
||||||
_ = state.WriteIntent(intentPath, state.Intent{
|
Source: "test",
|
||||||
|
UpdatedAt: time.Now().UTC().Add(-4 * time.Second),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
return state.Intent{
|
||||||
State: state.IntentShuttingDown,
|
State: state.IntentShuttingDown,
|
||||||
Reason: "peer-shutdown",
|
Reason: "peer-shutdown",
|
||||||
Source: "test",
|
Source: "test",
|
||||||
UpdatedAt: time.Now().UTC(),
|
UpdatedAt: time.Now().UTC(),
|
||||||
})
|
}, nil
|
||||||
}(cfg.State.IntentPath)
|
})
|
||||||
|
t.Cleanup(restoreRead)
|
||||||
orch, _ := newHookOrchestrator(t, cfg, nil, nil)
|
orch, _ := newHookOrchestrator(t, cfg, nil, nil)
|
||||||
err := orch.Startup(context.Background(), cluster.StartupOptions{Reason: "cooldown-peer-active"})
|
err := orch.Startup(context.Background(), cluster.StartupOptions{Reason: "cooldown-peer-active"})
|
||||||
if err == nil || !strings.Contains(err.Error(), "shutdown intent became active during cooldown wait") {
|
if err == nil || !strings.Contains(err.Error(), "shutdown intent became active during cooldown wait") {
|
||||||
|
|||||||
@ -24,12 +24,36 @@ func TestStateTestHookOverrideSetters(t *testing.T) {
|
|||||||
}
|
}
|
||||||
restoreWriteNil()
|
restoreWriteNil()
|
||||||
|
|
||||||
|
restoreReadNil := state.TestHookSetReadIntentOverride(nil)
|
||||||
|
readAfterNil, err := state.ReadIntent(intentPath)
|
||||||
|
if err != nil || readAfterNil.State != state.IntentNormal {
|
||||||
|
t.Fatalf("expected default read intent path after nil override, got %v / %v", readAfterNil, err)
|
||||||
|
}
|
||||||
|
restoreReadNil()
|
||||||
|
|
||||||
|
readOverrideCalled := false
|
||||||
|
restoreRead := state.TestHookSetReadIntentOverride(func(path string) (state.Intent, error) {
|
||||||
|
readOverrideCalled = true
|
||||||
|
return state.Intent{}, errors.New("forced read override")
|
||||||
|
})
|
||||||
|
_, err = state.ReadIntent(intentPath)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "forced read override") {
|
||||||
|
t.Fatalf("expected forced read override error, got %v", err)
|
||||||
|
}
|
||||||
|
if !readOverrideCalled {
|
||||||
|
t.Fatalf("expected read override to be invoked")
|
||||||
|
}
|
||||||
|
restoreRead()
|
||||||
|
if _, err := state.TestHookReadIntentDefault(intentPath); err != nil {
|
||||||
|
t.Fatalf("expected explicit default read helper to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
writeOverrideCalled := false
|
writeOverrideCalled := false
|
||||||
restoreWrite := state.TestHookSetWriteIntentOverride(func(path string, in state.Intent) error {
|
restoreWrite := state.TestHookSetWriteIntentOverride(func(path string, in state.Intent) error {
|
||||||
writeOverrideCalled = true
|
writeOverrideCalled = true
|
||||||
return errors.New("forced write override")
|
return errors.New("forced write override")
|
||||||
})
|
})
|
||||||
err := state.WriteIntent(intentPath, state.Intent{State: state.IntentNormal})
|
err = state.WriteIntent(intentPath, state.Intent{State: state.IntentNormal})
|
||||||
if err == nil || !strings.Contains(err.Error(), "forced write override") {
|
if err == nil || !strings.Contains(err.Error(), "forced write override") {
|
||||||
t.Fatalf("expected forced write override error, got %v", err)
|
t.Fatalf("expected forced write override error, got %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user