999 lines
44 KiB
Go
999 lines
44 KiB
Go
package cluster
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"scm.bstein.dev/bstein/ananke/internal/config"
|
|
"scm.bstein.dev/bstein/ananke/internal/execx"
|
|
"scm.bstein.dev/bstein/ananke/internal/state"
|
|
)
|
|
|
|
type commandStub struct {
|
|
match func(name string, args []string) bool
|
|
out string
|
|
err error
|
|
}
|
|
|
|
// buildOrchestratorWithStubs runs one orchestration or CLI step.
|
|
// Signature: buildOrchestratorWithStubs(t *testing.T, cfg config.Config, stubs []commandStub) *Orchestrator.
|
|
// Why: helper centralizes deterministic command dispatch for fast, isolated unit tests.
|
|
func buildOrchestratorWithStubs(t *testing.T, cfg config.Config, stubs []commandStub) *Orchestrator {
|
|
t.Helper()
|
|
if cfg.State.Dir == "" {
|
|
cfg.State.Dir = t.TempDir()
|
|
}
|
|
if cfg.State.ReportsDir == "" {
|
|
cfg.State.ReportsDir = filepath.Join(cfg.State.Dir, "reports")
|
|
}
|
|
if cfg.State.RunHistoryPath == "" {
|
|
cfg.State.RunHistoryPath = filepath.Join(cfg.State.Dir, "runs.json")
|
|
}
|
|
orch := &Orchestrator{
|
|
cfg: cfg,
|
|
runner: &execx.Runner{},
|
|
store: state.New(cfg.State.RunHistoryPath),
|
|
log: log.New(io.Discard, "", 0),
|
|
}
|
|
dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
|
|
for _, stub := range stubs {
|
|
if stub.match(name, args) {
|
|
return stub.out, stub.err
|
|
}
|
|
}
|
|
return "", nil
|
|
}
|
|
orch.runOverride = dispatch
|
|
orch.runSensitiveOverride = dispatch
|
|
return orch
|
|
}
|
|
|
|
// matchContains runs one orchestration or CLI step.
|
|
// Signature: matchContains(cmd string, parts ...string) func(string, []string) bool.
|
|
// Why: concise substring matching keeps command stubs readable across many tests.
|
|
func matchContains(cmd string, parts ...string) func(string, []string) bool {
|
|
return func(name string, args []string) bool {
|
|
if name != cmd {
|
|
return false
|
|
}
|
|
joined := strings.Join(args, " ")
|
|
for _, part := range parts {
|
|
if !strings.Contains(joined, part) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
}
|
|
|
|
// TestStartupEarlyFailureLeavesFluxSuspensionUnchanged runs one orchestration or CLI step.
|
|
// Signature: TestStartupEarlyFailureLeavesFluxSuspensionUnchanged(t *testing.T).
|
|
// Why: recovery must not release Flux when bootstrap fails before storage and
|
|
// critical workloads are ready, or Flux can re-create the same dependency loop.
|
|
func TestStartupEarlyFailureLeavesFluxSuspensionUnchanged(t *testing.T) {
|
|
tmpDir := t.TempDir()
|
|
cfg := config.Config{
|
|
SSHPort: 2277,
|
|
Startup: config.Startup{
|
|
APIWaitSeconds: 1,
|
|
APIPollSeconds: 1,
|
|
RequireNodeInventoryReach: false,
|
|
RequireTimeSync: false,
|
|
RequireNodeSSHAuth: false,
|
|
ReconcileAccessOnBoot: false,
|
|
AutoEtcdRestoreOnAPIFailure: false,
|
|
RequiredNodeLabels: map[string]map[string]string{
|
|
"titan-missing": {
|
|
"node-role.kubernetes.io/worker": "true",
|
|
},
|
|
},
|
|
},
|
|
State: config.State{
|
|
Dir: tmpDir,
|
|
ReportsDir: filepath.Join(tmpDir, "reports"),
|
|
RunHistoryPath: filepath.Join(tmpDir, "runs.json"),
|
|
LockPath: filepath.Join(tmpDir, "ananke.lock"),
|
|
IntentPath: filepath.Join(tmpDir, "intent.json"),
|
|
},
|
|
}
|
|
|
|
var mu sync.Mutex
|
|
calls := []string{}
|
|
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
mu.Lock()
|
|
calls = append(calls, name+" "+strings.Join(args, " "))
|
|
mu.Unlock()
|
|
return false
|
|
},
|
|
},
|
|
{match: matchContains("kubectl", "version", "--request-timeout=5s"), out: "ok"},
|
|
{match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"},
|
|
{
|
|
match: matchContains("kubectl", "label", "node", "titan-missing"),
|
|
err: errors.New(`nodes "titan-missing" not found`),
|
|
},
|
|
})
|
|
|
|
err := orch.Startup(context.Background(), StartupOptions{Reason: "test early failure"})
|
|
if err == nil {
|
|
t.Fatalf("expected startup to fail before flux resume")
|
|
}
|
|
if !strings.Contains(err.Error(), "ensure required node labels on titan-missing") {
|
|
t.Fatalf("expected required-label failure, got: %v", err)
|
|
}
|
|
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
for _, call := range calls {
|
|
if strings.Contains(call, `"suspend":false`) {
|
|
t.Fatalf("early failed startup unexpectedly resumed flux via call: %s", call)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestRecycleStuckControllerPodsHandlesLonghornAttachBlockedPods runs one orchestration or CLI step.
|
|
// Signature: TestRecycleStuckControllerPodsHandlesLonghornAttachBlockedPods(t *testing.T).
|
|
// Why: Pending Longhorn-backed pods on Longhorn-unready nodes should be
|
|
// rescheduled without mutating Longhorn volume, replica, or disk objects.
|
|
func TestRecycleStuckControllerPodsHandlesLonghornAttachBlockedPods(t *testing.T) {
|
|
created := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
|
|
lastSeen := time.Now().UTC().Format(time.RFC3339)
|
|
pods := `{"items":[{"metadata":{"namespace":"monitoring","name":"victoria-metrics-single-server-0","creationTimestamp":"` + created + `","ownerReferences":[{"kind":"StatefulSet","name":"victoria-metrics-single-server"}]},"spec":{"nodeName":"titan-0b"},"status":{"phase":"Pending"}}]}`
|
|
events := `{"items":[{"metadata":{"namespace":"monitoring","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"monitoring","name":"victoria-metrics-single-server-0"},"type":"Warning","reason":"FailedAttachVolume","message":"AttachVolume.Attach failed for volume \"pvc-1\" : rpc error from [http://longhorn-backend:9500/v1/volumes/pvc-1?action=attach]: unable to attach volume pvc-1 to titan-0b: node titan-0b is not ready","lastTimestamp":"` + lastSeen + `"}]}`
|
|
|
|
deleted := false
|
|
orch := buildOrchestratorWithStubs(t, config.Config{
|
|
Startup: config.Startup{StuckPodGraceSeconds: 180},
|
|
}, []commandStub{
|
|
{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: pods},
|
|
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: "titan-0b\tFalse\n"},
|
|
{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
|
|
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-0b"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if !matchContains("kubectl", "-n", "monitoring", "delete", "pod", "victoria-metrics-single-server-0", "--wait=false")(name, args) {
|
|
return false
|
|
}
|
|
deleted = true
|
|
return true
|
|
},
|
|
},
|
|
})
|
|
|
|
if err := orch.recycleStuckControllerPods(context.Background()); err != nil {
|
|
t.Fatalf("recycleStuckControllerPods failed: %v", err)
|
|
}
|
|
if !deleted {
|
|
t.Fatalf("expected longhorn attach-blocked pending pod to be recycled")
|
|
}
|
|
}
|
|
|
|
// TestRecycleStuckControllerPodsRepairsEncryptedVolumeCryptsetup runs one orchestration or CLI step.
|
|
// Signature: TestRecycleStuckControllerPodsRepairsEncryptedVolumeCryptsetup(t *testing.T).
|
|
// Why: encrypted Longhorn PVC recovery should repair missing host cryptsetup and
|
|
// then recycle the blocked pod without touching Longhorn data-plane objects.
|
|
func TestRecycleStuckControllerPodsRepairsEncryptedVolumeCryptsetup(t *testing.T) {
|
|
created := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
|
|
lastSeen := time.Now().UTC().Format(time.RFC3339)
|
|
pods := `{"items":[{"metadata":{"namespace":"finance","name":"actual-budget-abc","creationTimestamp":"` + created + `","ownerReferences":[{"kind":"ReplicaSet","name":"actual-budget"}]},"spec":{"nodeName":"titan-19"},"status":{"phase":"Pending"}}]}`
|
|
events := `{"items":[{"metadata":{"namespace":"finance","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"finance","name":"actual-budget-abc"},"type":"Warning","reason":"FailedMount","message":"MountVolume.MountDevice failed for volume \"pvc-1\" : nsenter: failed to execute cryptsetup: No such file or directory","lastTimestamp":"` + lastSeen + `"}]}`
|
|
|
|
installed := false
|
|
deleted := false
|
|
orch := buildOrchestratorWithStubs(t, config.Config{
|
|
Startup: config.Startup{StuckPodGraceSeconds: 180},
|
|
}, []commandStub{
|
|
{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: pods},
|
|
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: "titan-19\tTrue\n"},
|
|
{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
|
|
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-19"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if name != "ssh" || !strings.Contains(strings.Join(args, " "), "apt-get install -y --no-install-recommends cryptsetup-bin") {
|
|
return false
|
|
}
|
|
installed = true
|
|
return true
|
|
},
|
|
out: "__ANANKE_CRYPTSETUP_INSTALLED__",
|
|
},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if !matchContains("kubectl", "-n", "finance", "delete", "pod", "actual-budget-abc", "--wait=false")(name, args) {
|
|
return false
|
|
}
|
|
deleted = true
|
|
return true
|
|
},
|
|
},
|
|
})
|
|
|
|
if err := orch.recycleStuckControllerPods(context.Background()); err != nil {
|
|
t.Fatalf("recycleStuckControllerPods failed: %v", err)
|
|
}
|
|
if !installed {
|
|
t.Fatalf("expected missing host cryptsetup to be installed")
|
|
}
|
|
if !deleted {
|
|
t.Fatalf("expected encrypted-volume blocked pod to be recycled")
|
|
}
|
|
}
|
|
|
|
// TestRecycleStuckControllerPodsCordonsEncryptedVolumeNodeWhenRepairFails runs one orchestration or CLI step.
|
|
// Signature: TestRecycleStuckControllerPodsCordonsEncryptedVolumeNodeWhenRepairFails(t *testing.T).
|
|
// Why: when host package repair is blocked by sudo policy, Ananke should avoid
|
|
// the bad node and retry the controller-owned pod elsewhere.
|
|
func TestRecycleStuckControllerPodsCordonsEncryptedVolumeNodeWhenRepairFails(t *testing.T) {
|
|
created := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
|
|
lastSeen := time.Now().UTC().Format(time.RFC3339)
|
|
pods := `{"items":[{"metadata":{"namespace":"finance","name":"actual-budget-abc","creationTimestamp":"` + created + `","ownerReferences":[{"kind":"ReplicaSet","name":"actual-budget"}]},"spec":{"nodeName":"titan-19"},"status":{"phase":"Pending"}}]}`
|
|
events := `{"items":[{"metadata":{"namespace":"finance","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"finance","name":"actual-budget-abc"},"type":"Warning","reason":"FailedMount","message":"MountVolume.MountDevice failed for volume \"pvc-1\" : nsenter: failed to execute cryptsetup: No such file or directory","lastTimestamp":"` + lastSeen + `"}]}`
|
|
|
|
cordoned := false
|
|
deleted := false
|
|
orch := buildOrchestratorWithStubs(t, config.Config{
|
|
Startup: config.Startup{StuckPodGraceSeconds: 180},
|
|
}, []commandStub{
|
|
{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: pods},
|
|
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: "titan-19\tTrue\n"},
|
|
{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
|
|
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-19"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`},
|
|
{
|
|
match: matchContains("ssh", "apt-get install -y --no-install-recommends cryptsetup-bin"),
|
|
err: errors.New("sudo: a password is required"),
|
|
},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if !matchContains("kubectl", "cordon", "titan-19")(name, args) {
|
|
return false
|
|
}
|
|
cordoned = true
|
|
return true
|
|
},
|
|
},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if !matchContains("kubectl", "-n", "finance", "delete", "pod", "actual-budget-abc", "--wait=false")(name, args) {
|
|
return false
|
|
}
|
|
deleted = true
|
|
return true
|
|
},
|
|
},
|
|
})
|
|
|
|
if err := orch.recycleStuckControllerPods(context.Background()); err != nil {
|
|
t.Fatalf("recycleStuckControllerPods failed: %v", err)
|
|
}
|
|
if !cordoned {
|
|
t.Fatalf("expected cryptsetup-missing node to be cordoned")
|
|
}
|
|
if !deleted {
|
|
t.Fatalf("expected encrypted-volume blocked pod to be recycled")
|
|
}
|
|
}
|
|
|
|
// TestRecycleStuckControllerPodsHandlesStalePodsOnReadyNodes runs one orchestration or CLI step.
|
|
// Signature: TestRecycleStuckControllerPodsHandlesStalePodsOnReadyNodes(t *testing.T).
|
|
// Why: post-outage controller pods can remain Unknown or Failed after their
|
|
// node recovers; deletion clears stale status while force deletion stays away
|
|
// from PVC-backed storage.
|
|
func TestRecycleStuckControllerPodsHandlesStalePodsOnReadyNodes(t *testing.T) {
|
|
old := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
|
|
recent := time.Now().Add(-30 * time.Second).UTC().Format(time.RFC3339)
|
|
pods := `{"items":[` +
|
|
`{"metadata":{"namespace":"longhorn-system","name":"longhorn-vault-sync-old","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"longhorn-vault-sync"}]},"spec":{"nodeName":"titan-12"},"status":{"phase":"Unknown"}},` +
|
|
`{"metadata":{"namespace":"longhorn-system","name":"longhorn-vault-sync-failed","creationTimestamp":"` + old + `","deletionTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"longhorn-vault-sync"}]},"spec":{"nodeName":"titan-12","volumes":[{"name":"secret"}]},"status":{"phase":"Failed"}},` +
|
|
`{"metadata":{"namespace":"logging","name":"oauth2-proxy-terminating","creationTimestamp":"` + old + `","deletionTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"oauth2-proxy-logs"}]},"spec":{"nodeName":"titan-18","volumes":[{"name":"secret"}]},"status":{"phase":"Running"}},` +
|
|
`{"metadata":{"namespace":"longhorn-system","name":"pvc-backed-failed","creationTimestamp":"` + old + `","deletionTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"pvc-backed"}]},"spec":{"nodeName":"titan-12","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"data"}}]},"status":{"phase":"Failed"}},` +
|
|
`{"metadata":{"namespace":"longhorn-system","name":"longhorn-vault-sync-fresh","creationTimestamp":"` + recent + `","ownerReferences":[{"kind":"ReplicaSet","name":"longhorn-vault-sync"}]},"spec":{"nodeName":"titan-12"},"status":{"phase":"Unknown"}},` +
|
|
`{"metadata":{"namespace":"maintenance","name":"stale-on-bad-node","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"maintenance"}]},"spec":{"nodeName":"titan-22"},"status":{"phase":"Unknown"}},` +
|
|
`{"metadata":{"namespace":"default","name":"bare-pod","creationTimestamp":"` + old + `"},"spec":{"nodeName":"titan-12"},"status":{"phase":"Unknown"}}]}`
|
|
|
|
deleted := []string{}
|
|
orch := buildOrchestratorWithStubs(t, config.Config{
|
|
Startup: config.Startup{StuckPodGraceSeconds: 180},
|
|
}, []commandStub{
|
|
{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: pods},
|
|
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: "titan-12\tTrue\ntitan-22\tTrue\n"},
|
|
{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: `{"items":[]}`},
|
|
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-12"},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"Ready","status":"False"}]}}]}`},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if !matchContains("kubectl", "-n", "longhorn-system", "delete", "pod", "longhorn-vault-sync-old", "--wait=false")(name, args) {
|
|
return false
|
|
}
|
|
deleted = append(deleted, "longhorn-vault-sync-old")
|
|
return true
|
|
},
|
|
},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if !matchContains("kubectl", "-n", "longhorn-system", "delete", "pod", "longhorn-vault-sync-failed", "--wait=false", "--grace-period=0", "--force")(name, args) {
|
|
return false
|
|
}
|
|
deleted = append(deleted, "longhorn-vault-sync-failed")
|
|
return true
|
|
},
|
|
},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if !matchContains("kubectl", "-n", "logging", "delete", "pod", "oauth2-proxy-terminating", "--wait=false", "--grace-period=0", "--force")(name, args) {
|
|
return false
|
|
}
|
|
deleted = append(deleted, "oauth2-proxy-terminating")
|
|
return true
|
|
},
|
|
},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if !matchContains("kubectl", "-n", "longhorn-system", "delete", "pod", "pvc-backed-failed", "--wait=false")(name, args) {
|
|
return false
|
|
}
|
|
if strings.Contains(strings.Join(args, " "), "--force") {
|
|
t.Fatalf("pvc-backed stale pod must not be force deleted")
|
|
}
|
|
deleted = append(deleted, "pvc-backed-failed")
|
|
return true
|
|
},
|
|
},
|
|
})
|
|
|
|
if err := orch.recycleStuckControllerPods(context.Background()); err != nil {
|
|
t.Fatalf("recycleStuckControllerPods failed: %v", err)
|
|
}
|
|
if strings.Join(deleted, ",") != "longhorn-vault-sync-old,longhorn-vault-sync-failed,oauth2-proxy-terminating,pvc-backed-failed" {
|
|
t.Fatalf("expected only stale controller pods on Ready node to be recycled, got %#v", deleted)
|
|
}
|
|
}
|
|
|
|
// TestRecycleStuckControllerPodsCordonsContainerRuntimeWedgeNode runs one orchestration or CLI step.
|
|
// Signature: TestRecycleStuckControllerPodsCordonsContainerRuntimeWedgeNode(t *testing.T).
|
|
// Why: a Ready node with a wedged container runtime can trap replacement pods
|
|
// indefinitely; startup should cordon that scheduler target without draining it
|
|
// or touching Longhorn data-plane objects.
|
|
func TestRecycleStuckControllerPodsCordonsContainerRuntimeWedgeNode(t *testing.T) {
|
|
old := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
|
|
lastSeen := time.Now().UTC().Format(time.RFC3339)
|
|
pods := `{"items":[` +
|
|
`{"metadata":{"namespace":"logging","name":"oauth2-proxy-bad","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"oauth2-proxy"}]},"spec":{"nodeName":"titan-18","volumes":[{"name":"scratch"}]},"status":{"phase":"Pending","containerStatuses":[{"name":"oauth2-proxy","state":{"waiting":{"reason":"CreateContainerError"}}}]}},` +
|
|
`{"metadata":{"namespace":"monitoring","name":"suite-probe-bad","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"Job","name":"suite-probe"}]},"spec":{"nodeName":"titan-18","volumes":[{"name":"scratch"}]},"status":{"phase":"Pending","containerStatuses":[{"name":"probe","state":{"waiting":{"reason":"CreateContainerError"}}}]}},` +
|
|
`{"metadata":{"namespace":"sso","name":"secret-ensure-bad","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"Job","name":"secret-ensure"}]},"spec":{"nodeName":"titan-18","volumes":[{"name":"scratch"}]},"status":{"phase":"Pending","initContainerStatuses":[{"name":"init","state":{"waiting":{"reason":"CreateContainerError"}}}]}},` +
|
|
`{"metadata":{"namespace":"finance","name":"single-node-bad","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"single"}]},"spec":{"nodeName":"titan-19","volumes":[{"name":"scratch"}]},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CreateContainerError"}}}]}}]}`
|
|
events := `{"items":[` +
|
|
`{"metadata":{"namespace":"logging","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"logging","name":"oauth2-proxy-bad"},"type":"Warning","reason":"Failed","message":"spec.containers{oauth2-proxy}: Error: failed to reserve container name oauth2-proxy_logging","lastTimestamp":"` + lastSeen + `"},` +
|
|
`{"metadata":{"namespace":"monitoring","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"monitoring","name":"suite-probe-bad"},"type":"Warning","reason":"Failed","message":"spec.containers{probe}: Error: context deadline exceeded","lastTimestamp":"` + lastSeen + `"},` +
|
|
`{"metadata":{"namespace":"sso","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"sso","name":"secret-ensure-bad"},"type":"Warning","reason":"Failed","message":"spec.initContainers{init}: Error: failed to reserve container name init_sso","lastTimestamp":"` + lastSeen + `"},` +
|
|
`{"metadata":{"namespace":"finance","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"finance","name":"single-node-bad"},"type":"Warning","reason":"Failed","message":"spec.containers{app}: Error: failed to reserve container name app_finance","lastTimestamp":"` + lastSeen + `"}]}`
|
|
|
|
cordoned := []string{}
|
|
deleted := []string{}
|
|
orch := buildOrchestratorWithStubs(t, config.Config{
|
|
Startup: config.Startup{StuckPodGraceSeconds: 180},
|
|
}, []commandStub{
|
|
{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: pods},
|
|
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: ""},
|
|
{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
|
|
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-18"},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"titan-19"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if !matchContains("kubectl", "cordon")(name, args) {
|
|
return false
|
|
}
|
|
cordoned = append(cordoned, args[len(args)-1])
|
|
return true
|
|
},
|
|
},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if !matchContains("kubectl", "delete", "pod", "--wait=false")(name, args) {
|
|
return false
|
|
}
|
|
joined := strings.Join(args, " ")
|
|
if strings.Contains(joined, "--force") {
|
|
t.Fatalf("container-runtime wedge recycle must not force-delete fresh pods")
|
|
}
|
|
if len(args) >= 5 {
|
|
deleted = append(deleted, args[4])
|
|
}
|
|
return true
|
|
},
|
|
},
|
|
})
|
|
|
|
if err := orch.recycleStuckControllerPods(context.Background()); err != nil {
|
|
t.Fatalf("recycleStuckControllerPods failed: %v", err)
|
|
}
|
|
if strings.Join(cordoned, ",") != "titan-18" {
|
|
t.Fatalf("expected only titan-18 to be cordoned, got %#v", cordoned)
|
|
}
|
|
if strings.Join(deleted, ",") != "oauth2-proxy-bad,suite-probe-bad,secret-ensure-bad,single-node-bad" {
|
|
t.Fatalf("expected runtime-wedged pods to be recycled, got %#v", deleted)
|
|
}
|
|
}
|
|
|
|
// TestEffectiveWorkersFiltersIgnoredUnavailableNodes runs one orchestration or CLI step.
|
|
// Signature: TestEffectiveWorkersFiltersIgnoredUnavailableNodes(t *testing.T).
|
|
// Why: ignored unavailable nodes should be excluded before startup tries SSH,
|
|
// k3s-agent start, or uncordon operations against intentionally absent hosts.
|
|
func TestEffectiveWorkersFiltersIgnoredUnavailableNodes(t *testing.T) {
|
|
cfg := config.Config{
|
|
Workers: []string{" titan-08 ", "titan-09", "titan-10", "titan-11"},
|
|
Startup: config.Startup{
|
|
IgnoreUnavailableNodes: []string{"titan-09", "titan-10"},
|
|
},
|
|
}
|
|
orch := buildOrchestratorWithStubs(t, cfg, nil)
|
|
got, err := orch.effectiveWorkers(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("effectiveWorkers failed: %v", err)
|
|
}
|
|
want := []string{"titan-08", "titan-11"}
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Fatalf("effectiveWorkers mismatch got=%v want=%v", got, want)
|
|
}
|
|
}
|
|
|
|
// TestEnsureLonghornEncryptedHostPrereqsFiltersUnsafeWorkers runs one orchestration or CLI step.
|
|
// Signature: TestEnsureLonghornEncryptedHostPrereqsFiltersUnsafeWorkers(t *testing.T).
|
|
// Why: startup must not uncordon Longhorn workers that cannot mount encrypted
|
|
// PVCs; cordoning those nodes is safe and avoids repeating the post-outage
|
|
// mount deadlock.
|
|
func TestEnsureLonghornEncryptedHostPrereqsFiltersUnsafeWorkers(t *testing.T) {
|
|
cordoned := []string{}
|
|
orch := buildOrchestratorWithStubs(t, config.Config{
|
|
SSHManagedNodes: []string{"titan-04", "titan-19"},
|
|
}, []commandStub{
|
|
{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: "titan-04\ntitan-19\ntitan-23\n"},
|
|
{
|
|
match: matchContains("ssh", "titan-04", "command -v cryptsetup"),
|
|
out: "__ANANKE_CRYPTSETUP_PRESENT__",
|
|
},
|
|
{
|
|
match: matchContains("ssh", "titan-19", "apt-get install -y --no-install-recommends cryptsetup-bin"),
|
|
err: errors.New("sudo: a password is required"),
|
|
},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if name != "kubectl" || len(args) == 0 || args[0] != "cordon" {
|
|
return false
|
|
}
|
|
if len(args) > 1 {
|
|
cordoned = append(cordoned, args[len(args)-1])
|
|
}
|
|
return true
|
|
},
|
|
},
|
|
})
|
|
|
|
got, err := orch.ensureLonghornEncryptedHostPrereqs(context.Background(), []string{"titan-04", "titan-19", "titan-20"})
|
|
if err != nil {
|
|
t.Fatalf("ensureLonghornEncryptedHostPrereqs failed: %v", err)
|
|
}
|
|
want := []string{"titan-04", "titan-20"}
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Fatalf("guarded workers mismatch got=%v want=%v", got, want)
|
|
}
|
|
if strings.Join(cordoned, ",") != "titan-19,titan-23" {
|
|
t.Fatalf("expected unsafe longhorn hosts to be cordoned, got %v", cordoned)
|
|
}
|
|
}
|
|
|
|
// TestLonghornCryptsetupExemptNodesAreNotQuarantined runs one orchestration or CLI step.
|
|
// Signature: TestLonghornCryptsetupExemptNodesAreNotQuarantined(t *testing.T).
|
|
// Why: Veles/Oceanus uses titan-23 as a Longhorn host for unencrypted local
|
|
// volumes; startup should uncordon that policy-exempt node without requiring
|
|
// host SSH or weakening encrypted-volume safety on other workers.
|
|
func TestLonghornCryptsetupExemptNodesAreNotQuarantined(t *testing.T) {
|
|
cordoned := []string{}
|
|
uncordoned := []string{}
|
|
sshTitan23 := false
|
|
orch := buildOrchestratorWithStubs(t, config.Config{
|
|
SSHManagedNodes: []string{"titan-04"},
|
|
Startup: config.Startup{
|
|
LonghornCryptsetupExemptNodes: []string{"titan-23"},
|
|
},
|
|
}, []commandStub{
|
|
{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: "titan-04\ntitan-23\n"},
|
|
{
|
|
match: matchContains("ssh", "titan-04", "command -v cryptsetup"),
|
|
out: "__ANANKE_CRYPTSETUP_PRESENT__",
|
|
},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if name == "ssh" && strings.Contains(strings.Join(args, " "), "titan-23") {
|
|
sshTitan23 = true
|
|
return true
|
|
}
|
|
return false
|
|
},
|
|
},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if name != "kubectl" || len(args) == 0 || args[0] != "cordon" {
|
|
return false
|
|
}
|
|
if len(args) > 1 {
|
|
cordoned = append(cordoned, args[len(args)-1])
|
|
}
|
|
return true
|
|
},
|
|
},
|
|
{
|
|
match: func(name string, args []string) bool {
|
|
if !matchContains("kubectl", "uncordon")(name, args) {
|
|
return false
|
|
}
|
|
if len(args) > 1 {
|
|
uncordoned = append(uncordoned, args[len(args)-1])
|
|
}
|
|
return true
|
|
},
|
|
},
|
|
})
|
|
|
|
got, err := orch.ensureLonghornEncryptedHostPrereqs(context.Background(), []string{"titan-04"})
|
|
if err != nil {
|
|
t.Fatalf("ensureLonghornEncryptedHostPrereqs failed: %v", err)
|
|
}
|
|
if strings.Join(got, ",") != "titan-04" {
|
|
t.Fatalf("guarded workers mismatch got=%v", got)
|
|
}
|
|
if err := orch.uncordonLonghornCryptsetupExemptNodes(context.Background()); err != nil {
|
|
t.Fatalf("uncordonLonghornCryptsetupExemptNodes failed: %v", err)
|
|
}
|
|
if sshTitan23 {
|
|
t.Fatalf("did not expect cryptsetup SSH check for exempt titan-23")
|
|
}
|
|
if len(cordoned) != 0 {
|
|
t.Fatalf("did not expect exempt node to be cordoned, got %v", cordoned)
|
|
}
|
|
if strings.Join(uncordoned, ",") != "titan-23" {
|
|
t.Fatalf("expected exempt titan-23 to be uncordoned, got %v", uncordoned)
|
|
}
|
|
}
|
|
|
|
// TestLonghornHostNodesFallsBackToConfiguredLabels runs one orchestration or CLI step.
|
|
// Signature: TestLonghornHostNodesFallsBackToConfiguredLabels(t *testing.T).
|
|
// Why: bootstrap caches or minimal test clusters can lack live labels; the
|
|
// static startup inventory should still protect configured storage workers.
|
|
func TestLonghornHostNodesFallsBackToConfiguredLabels(t *testing.T) {
|
|
orch := buildOrchestratorWithStubs(t, config.Config{
|
|
Startup: config.Startup{
|
|
RequiredNodeLabels: map[string]map[string]string{
|
|
"titan-04": {"longhorn-host": "true"},
|
|
"titan-20": {"node-role.kubernetes.io/worker": "true"},
|
|
},
|
|
},
|
|
}, []commandStub{
|
|
{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: ""},
|
|
})
|
|
|
|
got, err := orch.longhornHostNodes(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("longhornHostNodes failed: %v", err)
|
|
}
|
|
if _, ok := got["titan-04"]; !ok || len(got) != 1 {
|
|
t.Fatalf("expected configured longhorn host fallback, got %v", got)
|
|
}
|
|
}
|
|
|
|
// TestNewConstructsOrchestrator runs one orchestration or CLI step.
|
|
// Signature: TestNewConstructsOrchestrator(t *testing.T).
|
|
// Why: covers constructor path in orchestrator core module.
|
|
func TestNewConstructsOrchestrator(t *testing.T) {
|
|
cfg := config.Config{State: config.State{RunHistoryPath: filepath.Join(t.TempDir(), "runs.json")}}
|
|
r := &execx.Runner{}
|
|
s := state.New(cfg.State.RunHistoryPath)
|
|
orch := New(cfg, r, s, log.New(io.Discard, "", 0))
|
|
if orch == nil || orch.runner != r || orch.store != s {
|
|
t.Fatalf("constructor returned unexpected orchestrator: %#v", orch)
|
|
}
|
|
}
|
|
|
|
// TestParseSnapshotPathFromEtcdSnapshotList runs one orchestration or CLI step.
|
|
// Signature: TestParseSnapshotPathFromEtcdSnapshotList(t *testing.T).
|
|
// Why: covers snapshot-path parser branches including header skip and no-match.
|
|
func TestParseSnapshotPathFromEtcdSnapshotList(t *testing.T) {
|
|
out := strings.Join([]string{
|
|
"Name Size Created Location",
|
|
`pre-shutdown 4.2M now "file:///var/lib/rancher/k3s/server/db/snapshots/pre-shutdown"`,
|
|
}, "\n")
|
|
got := parseSnapshotPathFromEtcdSnapshotList(out)
|
|
if got != "/var/lib/rancher/k3s/server/db/snapshots/pre-shutdown" {
|
|
t.Fatalf("unexpected snapshot path: %q", got)
|
|
}
|
|
if parseSnapshotPathFromEtcdSnapshotList("no snapshots") != "" {
|
|
t.Fatalf("expected no snapshot path")
|
|
}
|
|
}
|
|
|
|
// TestFluxSourceHelpers runs one orchestration or CLI step.
|
|
// Signature: TestFluxSourceHelpers(t *testing.T).
|
|
// Why: covers flux source readiness/guard/branch patch helper flows.
|
|
func TestFluxSourceHelpers(t *testing.T) {
|
|
cfg := config.Config{
|
|
ExpectedFluxSource: "ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git",
|
|
}
|
|
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{match: matchContains("kubectl", "jsonpath={.status.conditions"), out: "True"},
|
|
{match: matchContains("kubectl", "jsonpath={.spec.url}"), out: "ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git"},
|
|
{match: matchContains("kubectl", "jsonpath={.spec.ref.branch}"), out: "main"},
|
|
{match: matchContains("kubectl", "patch", "gitrepository"), out: ""},
|
|
})
|
|
|
|
ready, err := orch.fluxSourceReady(context.Background())
|
|
if err != nil || !ready {
|
|
t.Fatalf("expected flux source ready, got ready=%v err=%v", ready, err)
|
|
}
|
|
if err := orch.guardFluxSourceDrift(context.Background(), "main", false); err != nil {
|
|
t.Fatalf("guardFluxSourceDrift failed: %v", err)
|
|
}
|
|
if err := orch.ensureFluxBranch(context.Background(), "main", false); err != nil {
|
|
t.Fatalf("ensureFluxBranch no-op failed: %v", err)
|
|
}
|
|
if got := normalizeGitURL(" SSH://Git@Host/Repo.git/ "); got != "ssh://git@host/repo" {
|
|
t.Fatalf("unexpected normalized url: %q", got)
|
|
}
|
|
}
|
|
|
|
// TestCoordinationHelpers runs one orchestration or CLI step.
|
|
// Signature: TestCoordinationHelpers(t *testing.T).
|
|
// Why: covers intent-age helpers, shell quoting, and peer selection logic.
|
|
func TestCoordinationHelpers(t *testing.T) {
|
|
in := state.Intent{UpdatedAt: time.Now().Add(-10 * time.Second)}
|
|
if intentAge(in) <= 0 {
|
|
t.Fatalf("expected positive age")
|
|
}
|
|
if !intentFresh(state.Intent{}, time.Second) {
|
|
t.Fatalf("zero timestamp should be fresh")
|
|
}
|
|
if shellQuote("a'b") != `'a'"'"'b'` {
|
|
t.Fatalf("unexpected shell quote output")
|
|
}
|
|
|
|
orch := buildOrchestratorWithStubs(t, config.Config{
|
|
Coordination: config.Coordination{
|
|
PeerHosts: []string{"titan-24", "titan-24", "titan-db"},
|
|
ForwardShutdownHost: "titan-db",
|
|
},
|
|
}, nil)
|
|
peers := orch.coordinationPeers()
|
|
if len(peers) != 2 {
|
|
t.Fatalf("expected deduped peers, got %v", peers)
|
|
}
|
|
}
|
|
|
|
// TestVerifyEtcdSnapshotAndRunSudoK3S runs one orchestration or CLI step.
|
|
// Signature: TestVerifyEtcdSnapshotAndRunSudoK3S(t *testing.T).
|
|
// Why: covers k3s command fallback and snapshot verification happy path.
|
|
func TestVerifyEtcdSnapshotAndRunSudoK3S(t *testing.T) {
|
|
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
|
|
{match: matchContains("ssh", "stat -c %s"), out: "2097152"},
|
|
{match: matchContains("ssh", "k3s etcd-snapshot ls"), out: "/var/lib/rancher/k3s/server/db/snapshots/pre-shutdown"},
|
|
{match: matchContains("ssh", "sha256sum"), out: strings.Repeat("a", 64)},
|
|
})
|
|
if err := orch.verifyEtcdSnapshot(context.Background(), "titan-0a", "/var/lib/rancher/k3s/server/db/snapshots/pre-shutdown"); err != nil {
|
|
t.Fatalf("verifyEtcdSnapshot failed: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestScalingHelpers runs one orchestration or CLI step.
|
|
// Signature: TestScalingHelpers(t *testing.T).
|
|
// Why: covers workload discovery, snapshot IO, and scale command orchestration.
|
|
func TestScalingHelpers(t *testing.T) {
|
|
cfg := config.Config{
|
|
ExcludedNamespaces: []string{"kube-system"},
|
|
State: config.State{Dir: t.TempDir()},
|
|
}
|
|
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{
|
|
match: matchContains("kubectl", "get deployment", "jsonpath"),
|
|
out: strings.Join([]string{
|
|
"default\tgrafana\t1",
|
|
"kube-system\tcoredns\t2",
|
|
"",
|
|
}, "\n"),
|
|
},
|
|
{
|
|
match: matchContains("kubectl", "get statefulset", "jsonpath"),
|
|
out: "vault\tvault\t1\n",
|
|
},
|
|
{match: matchContains("kubectl", "scale", "deployment", "grafana"), out: ""},
|
|
{match: matchContains("kubectl", "scale", "statefulset", "vault"), out: ""},
|
|
})
|
|
|
|
entries, err := orch.listScalableWorkloads(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("listScalableWorkloads failed: %v", err)
|
|
}
|
|
if len(entries) != 2 {
|
|
t.Fatalf("expected 2 scalable entries, got %d (%v)", len(entries), entries)
|
|
}
|
|
if err := orch.writeScaledWorkloadSnapshot(entries); err != nil {
|
|
t.Fatalf("writeScaledWorkloadSnapshot failed: %v", err)
|
|
}
|
|
snapshot, err := orch.readScaledWorkloadSnapshot()
|
|
if err != nil || snapshot == nil || len(snapshot.Entries) != 2 {
|
|
t.Fatalf("readScaledWorkloadSnapshot failed snapshot=%v err=%v", snapshot, err)
|
|
}
|
|
if err := orch.scaleWorkloads(context.Background(), entries, -1, 2); err != nil {
|
|
t.Fatalf("scaleWorkloads failed: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestStorageReadyAndWorkloadHelpers runs one orchestration or CLI step.
|
|
// Signature: TestStorageReadyAndWorkloadHelpers(t *testing.T).
|
|
// Why: covers storage readiness checks and workload helper utilities.
|
|
func TestStorageReadyAndWorkloadHelpers(t *testing.T) {
|
|
cfg := config.Config{
|
|
Startup: config.Startup{
|
|
StorageMinReadyNodes: 1,
|
|
StorageCriticalPVCs: []string{"vault/data-vault-0"},
|
|
},
|
|
}
|
|
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{match: matchContains("kubectl", "nodes.longhorn.io"), out: "titan-23:True:True\n"},
|
|
{match: matchContains("kubectl", "get pvc data-vault-0"), out: "Bound"},
|
|
})
|
|
ok, reason, err := orch.storageReady(context.Background())
|
|
if err != nil || !ok {
|
|
t.Fatalf("expected storageReady true, got ok=%v reason=%q err=%v", ok, reason, err)
|
|
}
|
|
}
|
|
|
|
// TestIngressAndServiceHelpers runs one orchestration or CLI step.
|
|
// Signature: TestIngressAndServiceHelpers(t *testing.T).
|
|
// Why: covers ingress host discovery helpers and URL parsing helpers.
|
|
func TestIngressAndServiceHelpers(t *testing.T) {
|
|
cfg := config.Config{
|
|
Startup: config.Startup{
|
|
IngressChecklistIgnoreHosts: []string{"ignore.bstein.dev"},
|
|
},
|
|
}
|
|
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{match: matchContains("kubectl", "get ingress", "-A", "-o", "json"), out: `{"items":[{"metadata":{"namespace":"gitea"},"spec":{"rules":[{"host":"scm.bstein.dev"}]}},{"metadata":{"namespace":"x"},"spec":{"rules":[{"host":"ignore.bstein.dev"}]}}]}`},
|
|
})
|
|
hosts, err := orch.discoverIngressHosts(context.Background())
|
|
if err != nil || len(hosts) != 1 || hosts[0] != "scm.bstein.dev" {
|
|
t.Fatalf("discoverIngressHosts unexpected hosts=%v err=%v", hosts, err)
|
|
}
|
|
if got := hostFromURL("https://metrics.bstein.dev/api/health"); got != "metrics.bstein.dev" {
|
|
t.Fatalf("unexpected hostFromURL value: %q", got)
|
|
}
|
|
if !isLikelyHostname("metrics.bstein.dev") || isLikelyHostname("bad path/value") {
|
|
t.Fatalf("isLikelyHostname classification mismatch")
|
|
}
|
|
}
|
|
|
|
// TestWorkloadConvergenceHelpers runs one orchestration or CLI step.
|
|
// Signature: TestWorkloadConvergenceHelpers(t *testing.T).
|
|
// Why: covers controller readiness helpers and stuck-pod heuristics.
|
|
func TestWorkloadConvergenceHelpers(t *testing.T) {
|
|
replicas := int32(2)
|
|
item := workloadResource{Kind: "deployment"}
|
|
item.Spec.Replicas = &replicas
|
|
item.Status.ReadyReplicas = 1
|
|
desired, ready, ok := desiredReady(item)
|
|
if !ok || desired != 2 || ready != 1 {
|
|
t.Fatalf("desiredReady mismatch desired=%d ready=%d ok=%v", desired, ready, ok)
|
|
}
|
|
var pod podResource
|
|
pod.Metadata.OwnerReferences = []ownerReference{{Kind: "ReplicaSet"}}
|
|
if !podControllerOwned(pod) {
|
|
t.Fatalf("expected podControllerOwned=true")
|
|
}
|
|
pod.Status.ContainerStatuses = []podContainerStatus{{State: podContainerState{Waiting: &podContainerWaitingState{Reason: "CrashLoopBackOff"}}}}
|
|
reason := stuckContainerReason(pod, map[string]struct{}{"CrashLoopBackOff": struct{}{}})
|
|
if reason != "CrashLoopBackOff" {
|
|
t.Fatalf("unexpected stuck reason: %q", reason)
|
|
}
|
|
}
|
|
|
|
// TestDrainAndK3SHelpers runs one orchestration or CLI step.
|
|
// Signature: TestDrainAndK3SHelpers(t *testing.T).
|
|
// Why: covers node drain diagnostics and k3s snapshot selection flow.
|
|
func TestDrainAndK3SHelpers(t *testing.T) {
|
|
cfg := config.Config{
|
|
SSHManagedNodes: []string{"titan-0a"},
|
|
}
|
|
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{match: matchContains("kubectl", "get pods", "--field-selector", "spec.nodeName=titan-22"), out: "vault vault-0 Running StatefulSet\n"},
|
|
{match: matchContains("ssh", "k3s etcd-snapshot ls"), out: "pre-shutdown /var/lib/rancher/k3s/server/db/snapshots/pre-shutdown"},
|
|
})
|
|
diag := orch.drainNodeDiagnostics(context.Background(), "titan-22")
|
|
if !strings.Contains(diag, "vault/vault-0") {
|
|
t.Fatalf("unexpected diagnostics output: %q", diag)
|
|
}
|
|
snapshot, err := orch.latestEtcdSnapshotPath(context.Background(), "titan-0a")
|
|
if err != nil || snapshot == "" {
|
|
t.Fatalf("latestEtcdSnapshotPath failed snapshot=%q err=%v", snapshot, err)
|
|
}
|
|
}
|
|
|
|
// TestTimesyncAndInventoryHelpers runs one orchestration or CLI step.
|
|
// Signature: TestTimesyncAndInventoryHelpers(t *testing.T).
|
|
// Why: covers time sync helpers, datastore endpoint parsing, and inventory assembly.
|
|
func TestTimesyncAndInventoryHelpers(t *testing.T) {
|
|
cfg := config.Config{
|
|
ControlPlanes: []string{"titan-0a"},
|
|
Workers: []string{"titan-22"},
|
|
SSHManagedNodes: []string{"titan-0a", "titan-22"},
|
|
SSHNodeHosts: map[string]string{
|
|
"titan-db": "10.0.0.10",
|
|
},
|
|
Coordination: config.Coordination{
|
|
PeerHosts: []string{"titan-24"},
|
|
ForwardShutdownHost: "titan-db",
|
|
},
|
|
}
|
|
orch := buildOrchestratorWithStubs(t, cfg, nil)
|
|
nodes := orch.inventoryNodesForValidation()
|
|
if len(nodes) < 3 {
|
|
t.Fatalf("expected combined inventory nodes, got %v", nodes)
|
|
}
|
|
if parseDatastoreEndpoint("ExecStart=/usr/local/bin/k3s server --datastore-endpoint=postgres://x") == "" {
|
|
t.Fatalf("expected datastore endpoint parse")
|
|
}
|
|
if !isTimeSynced("YES") || isTimeSynced("no") {
|
|
t.Fatalf("unexpected isTimeSynced behavior")
|
|
}
|
|
|
|
ln, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("listen failed: %v", err)
|
|
}
|
|
defer ln.Close()
|
|
if !orch.tcpReachable(ln.Addr().String(), 500*time.Millisecond) {
|
|
t.Fatalf("expected tcpReachable=true for open listener")
|
|
}
|
|
}
|
|
|
|
// TestShutdownModeValidation runs one orchestration or CLI step.
|
|
// Signature: TestShutdownModeValidation(t *testing.T).
|
|
// Why: covers removed poweroff mode and invalid-mode errors.
|
|
func TestShutdownModeValidation(t *testing.T) {
|
|
if mode, err := normalizeShutdownMode("cluster-only"); err != nil || mode != "cluster-only" {
|
|
t.Fatalf("expected cluster-only mode, got mode=%q err=%v", mode, err)
|
|
}
|
|
if _, err := normalizeShutdownMode("bogus"); err == nil {
|
|
t.Fatalf("expected invalid mode error")
|
|
}
|
|
}
|
|
|
|
// TestWaitForAPIDryRunShortCircuit runs one orchestration or CLI step.
|
|
// Signature: TestWaitForAPIDryRunShortCircuit(t *testing.T).
|
|
// Why: covers dry-run short-circuit branch for api readiness wait.
|
|
func TestWaitForAPIDryRunShortCircuit(t *testing.T) {
|
|
orch := &Orchestrator{runner: &execx.Runner{DryRun: true}}
|
|
if err := orch.waitForAPI(context.Background(), 1, time.Millisecond); err != nil {
|
|
t.Fatalf("expected dry-run waitForAPI to pass: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestGuardFluxSourceDriftMismatch runs one orchestration or CLI step.
|
|
// Signature: TestGuardFluxSourceDriftMismatch(t *testing.T).
|
|
// Why: covers url-drift and branch-drift error branches.
|
|
func TestGuardFluxSourceDriftMismatch(t *testing.T) {
|
|
cfg := config.Config{ExpectedFluxSource: "ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git"}
|
|
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{match: matchContains("kubectl", "jsonpath={.spec.url}"), out: "ssh://git@scm.bstein.dev:2242/bstein/wrong.git"},
|
|
})
|
|
if err := orch.guardFluxSourceDrift(context.Background(), "main", false); err == nil {
|
|
t.Fatalf("expected guardFluxSourceDrift mismatch error")
|
|
}
|
|
|
|
orch = buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{match: matchContains("kubectl", "jsonpath={.spec.url}"), out: "ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git"},
|
|
{match: matchContains("kubectl", "jsonpath={.spec.ref.branch}"), out: "atlasbot"},
|
|
})
|
|
if err := orch.guardFluxSourceDrift(context.Background(), "main", false); err == nil {
|
|
t.Fatalf("expected branch drift error")
|
|
}
|
|
}
|
|
|
|
// TestRunSudoK3SFailsWhenAllCandidatesFail runs one orchestration or CLI step.
|
|
// Signature: TestRunSudoK3SFailsWhenAllCandidatesFail(t *testing.T).
|
|
// Why: covers fallback failure return in runSudoK3S.
|
|
func TestRunSudoK3SFailsWhenAllCandidatesFail(t *testing.T) {
|
|
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
|
|
{match: matchContains("ssh", "k3s"), err: errors.New("no binary")},
|
|
})
|
|
if _, err := orch.runSudoK3S(context.Background(), "titan-0a", "server"); err == nil {
|
|
t.Fatalf("expected runSudoK3S failure when all candidates fail")
|
|
}
|
|
}
|
|
|
|
// TestCriticalEndpointHelpers runs one orchestration or CLI step.
|
|
// Signature: TestCriticalEndpointHelpers(t *testing.T).
|
|
// Why: covers critical endpoint parsing and readiness checks that gate startup completion.
|
|
func TestCriticalEndpointHelpers(t *testing.T) {
|
|
cfg := config.Config{
|
|
Startup: config.Startup{
|
|
CriticalServiceEndpoints: []string{"monitoring/victoria-metrics-single-server"},
|
|
},
|
|
}
|
|
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{match: matchContains("kubectl", "get endpoints victoria-metrics-single-server"), out: "10.42.0.10\n10.42.0.11\n"},
|
|
})
|
|
ok, detail, ns, svc, err := orch.criticalServiceEndpointsReady(context.Background())
|
|
if err != nil || !ok {
|
|
t.Fatalf("expected criticalServiceEndpointsReady success, got ok=%v detail=%q ns=%q svc=%q err=%v", ok, detail, ns, svc, err)
|
|
}
|
|
if detail != "services=1" {
|
|
t.Fatalf("unexpected readiness detail: %q", detail)
|
|
}
|
|
gotNS, gotSvc, err := parseCriticalServiceEndpoint("monitoring/victoria-metrics-single-server")
|
|
if err != nil || gotNS != "monitoring" || gotSvc != "victoria-metrics-single-server" {
|
|
t.Fatalf("unexpected parse result ns=%q svc=%q err=%v", gotNS, gotSvc, err)
|
|
}
|
|
if _, _, err := parseCriticalServiceEndpoint("invalid"); err == nil {
|
|
t.Fatalf("expected parseCriticalServiceEndpoint error")
|
|
}
|
|
}
|
|
|
|
// TestCriticalEndpointAutoHealWorkflow runs one orchestration or CLI step.
|
|
// Signature: TestCriticalEndpointAutoHealWorkflow(t *testing.T).
|
|
// Why: covers endpoint-zero recovery where startup heals workload replicas before succeeding.
|
|
func TestCriticalEndpointAutoHealWorkflow(t *testing.T) {
|
|
cfg := config.Config{
|
|
Startup: config.Startup{
|
|
CriticalServiceEndpointWaitSec: 2,
|
|
CriticalServiceEndpointPollSec: 1,
|
|
CriticalServiceEndpoints: []string{"monitoring/victoria-metrics-single-server"},
|
|
},
|
|
State: config.State{
|
|
Dir: t.TempDir(),
|
|
ReportsDir: filepath.Join(t.TempDir(), "reports"),
|
|
RunHistoryPath: filepath.Join(t.TempDir(), "runs.json"),
|
|
},
|
|
}
|
|
orch := &Orchestrator{
|
|
cfg: cfg,
|
|
runner: &execx.Runner{},
|
|
store: state.New(cfg.State.RunHistoryPath),
|
|
log: log.New(io.Discard, "", 0),
|
|
}
|
|
|
|
endpointChecks := 0
|
|
dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
|
|
joined := name + " " + strings.Join(args, " ")
|
|
if strings.Contains(joined, "get endpoints victoria-metrics-single-server") {
|
|
endpointChecks++
|
|
if endpointChecks == 1 {
|
|
return "", nil
|
|
}
|
|
return "10.42.0.10\n", nil
|
|
}
|
|
if strings.Contains(joined, "scale deployment victoria-metrics-single-server") {
|
|
return "", errors.New(`Error from server (NotFound): deployments.apps "victoria-metrics-single-server" not found`)
|
|
}
|
|
if strings.Contains(joined, "scale statefulset victoria-metrics-single-server") {
|
|
return "", nil
|
|
}
|
|
if strings.Contains(joined, "rollout status statefulset/victoria-metrics-single-server") {
|
|
return "statefulset rolled out", nil
|
|
}
|
|
return "", nil
|
|
}
|
|
orch.runOverride = dispatch
|
|
orch.runSensitiveOverride = dispatch
|
|
|
|
if err := orch.waitForCriticalServiceEndpoints(context.Background()); err != nil {
|
|
t.Fatalf("waitForCriticalServiceEndpoints failed: %v", err)
|
|
}
|
|
if endpointChecks < 2 {
|
|
t.Fatalf("expected repeated endpoint checks, got %d", endpointChecks)
|
|
}
|
|
}
|