From 76876ef895bf48a8bf975cead2f4f7a962056373 Mon Sep 17 00:00:00 2001 From: codex Date: Thu, 16 Jul 2026 02:27:44 -0300 Subject: [PATCH] support terraform-owned bootstrap inputs --- README.md | 8 + cmd/ananke/builder.go | 46 +- cmd/ananke/builder_test.go | 107 ++++ cmd/ananke/command_handlers_autoheal_test.go | 30 ++ configs/ananke.example.yaml | 1 + configs/ananke.tethys.yaml | 1 + configs/ananke.titan-db.yaml | 1 + internal/cluster/orchestrator_autorepair.go | 88 ---- .../orchestrator_autorepair_cleanup.go | 97 ++++ internal/cluster/orchestrator_drain.go | 5 + internal/cluster/orchestrator_ingress.go | 80 +++ internal/cluster/orchestrator_lifecycle.go | 4 +- ...orchestrator_quality_gate_closeout_test.go | 442 ++++++++++++++++ ...trator_quality_gate_final_closeout_test.go | 212 ++++++++ ...strator_quality_gate_more_closeout_test.go | 480 ++++++++++++++++++ ...tor_quality_gate_recovery_closeout_test.go | 370 ++++++++++++++ ...ator_quality_gate_storage_closeout_test.go | 429 ++++++++++++++++ internal/config/apply_defaults.go | 4 + internal/config/defaults.go | 1 + internal/config/load.go | 85 ++++ internal/config/load_additional_test.go | 183 +++++++ .../startup_service_catalog_closeout_test.go | 25 + internal/config/types.go | 1 + internal/config/validate.go | 3 + internal/config/validate_matrix_test.go | 4 + scripts/install-config-migration.sh | 12 + scripts/install.sh | 2 + testing/hygiene/in_tree_test_allowlist.txt | 6 + .../hooks_ingress_service_matrix_test.go | 62 +++ 29 files changed, 2699 insertions(+), 90 deletions(-) create mode 100644 internal/cluster/orchestrator_autorepair_cleanup.go create mode 100644 internal/cluster/orchestrator_quality_gate_closeout_test.go create mode 100644 internal/cluster/orchestrator_quality_gate_final_closeout_test.go create mode 100644 internal/cluster/orchestrator_quality_gate_more_closeout_test.go create mode 100644 internal/cluster/orchestrator_quality_gate_recovery_closeout_test.go create mode 100644 internal/cluster/orchestrator_quality_gate_storage_closeout_test.go create mode 100644 internal/config/startup_service_catalog_closeout_test.go diff --git a/README.md b/README.md index 841b356..1f8442a 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,14 @@ Host files: - `/var/lib/ananke/last-shutdown-report.json` - `/var/log/ananke/update.log` +## Terraform-Generated Inventory + +Ananke can load a generated inventory fragment from `/etc/ananke/ananke.inventory.yaml` alongside `/etc/ananke/ananke.yaml`. The installer also accepts `ANANKE_INVENTORY_FRAGMENT=/path/to/ananke.inventory.yaml` and copies it into place. + +That fragment is for non-secret declarative inputs only: node hosts, managed nodes, control planes, workers, required node labels, and ignored unavailable nodes. When Terraform owns node labels, the fragment should set `startup.required_node_labels_mode: validate`; Ananke will report label drift but will not apply `kubectl label` during normal startup or post-start auto-heal. + +Keep recovery behavior in Ananke: UPS shutdown, SSH repair, k3s restart/reboot, Longhorn/runtime recovery, cordon/uncordon, pod recycling, and Flux suspend/resume remain imperative Ananke actions. + ## Development Local testing check before installing: diff --git a/cmd/ananke/builder.go b/cmd/ananke/builder.go index 9d4c2ed..378bb3f 100644 --- a/cmd/ananke/builder.go +++ b/cmd/ananke/builder.go @@ -2,6 +2,9 @@ package main import ( "log" + "os" + "path/filepath" + "strings" "scm.bstein.dev/bstein/ananke/internal/cluster" "scm.bstein.dev/bstein/ananke/internal/config" @@ -13,7 +16,7 @@ import ( // Signature: buildOrchestrator(logger *log.Logger, cfgPath string, dryRun bool) (config.Config, *cluster.Orchestrator, error). // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve. func buildOrchestrator(logger *log.Logger, cfgPath string, dryRun bool) (config.Config, *cluster.Orchestrator, error) { - cfg, err := config.Load(cfgPath) + cfg, err := config.LoadWithFragments(cfgPath, configFragmentPaths(cfgPath)...) if err != nil { return config.Config{}, nil, err } @@ -32,3 +35,44 @@ func buildOrchestrator(logger *log.Logger, cfgPath string, dryRun bool) (config. orch := cluster.New(cfg, runner, store, logger) return cfg, orch, nil } + +// configFragmentPaths runs one orchestration or CLI step. +// Signature: configFragmentPaths(cfgPath string) []string. +// Why: keeps generated inventory overlays discoverable without changing every +// systemd command line or recovery script invocation. +func configFragmentPaths(cfgPath string) []string { + paths := []string{} + if env := strings.TrimSpace(os.Getenv("ANANKE_CONFIG_FRAGMENTS")); env != "" { + for _, path := range filepath.SplitList(env) { + path = strings.TrimSpace(path) + if path != "" { + paths = append(paths, path) + } + } + } + defaultPath := defaultInventoryFragmentPath(cfgPath) + if defaultPath != "" { + if _, err := os.Stat(defaultPath); err == nil { + paths = append(paths, defaultPath) + } + } + return paths +} + +// defaultInventoryFragmentPath runs one orchestration or CLI step. +// Signature: defaultInventoryFragmentPath(cfgPath string) string. +// Why: pairs ananke.yaml with an adjacent Terraform-generated inventory +// fragment using a deterministic host-local convention. +func defaultInventoryFragmentPath(cfgPath string) string { + dir := filepath.Dir(cfgPath) + base := filepath.Base(cfgPath) + ext := filepath.Ext(base) + name := strings.TrimSuffix(base, ext) + if name == "" { + return "" + } + if ext == "" { + return filepath.Join(dir, name+".inventory.yaml") + } + return filepath.Join(dir, name+".inventory"+ext) +} diff --git a/cmd/ananke/builder_test.go b/cmd/ananke/builder_test.go index 2711d56..43be9e9 100644 --- a/cmd/ananke/builder_test.go +++ b/cmd/ananke/builder_test.go @@ -33,6 +33,57 @@ func TestBuildOrchestratorCreatesStateDirs(t *testing.T) { } } +// TestBuildOrchestratorLoadsAdjacentInventoryFragment runs one orchestration or CLI step. +// Signature: TestBuildOrchestratorLoadsAdjacentInventoryFragment(t *testing.T). +// Why: the systemd default path must consume Terraform-generated inventory +// without requiring extra command-line flags. +func TestBuildOrchestratorLoadsAdjacentInventoryFragment(t *testing.T) { + cfgPath := writeTestConfig(t) + fragmentPath := filepath.Join(filepath.Dir(cfgPath), "ananke.inventory.yaml") + raw := ` +ssh_node_hosts: + titan-0a: 192.168.22.11 + titan-05: 192.168.22.31 +ssh_managed_nodes: [titan-0a, titan-05] +control_planes: [titan-0a] +workers: [titan-05] +startup: + required_node_labels_mode: validate + required_node_labels: {} + ignore_unavailable_nodes: [] +` + if err := os.WriteFile(fragmentPath, []byte(raw), 0o644); err != nil { + t.Fatalf("write inventory fragment: %v", err) + } + + cfg, orch, err := buildOrchestrator(log.New(io.Discard, "", 0), cfgPath, true) + if err != nil { + t.Fatalf("buildOrchestrator failed: %v", err) + } + if orch == nil { + t.Fatalf("expected orchestrator") + } + if len(cfg.ControlPlanes) != 1 || cfg.ControlPlanes[0] != "titan-0a" { + t.Fatalf("expected adjacent fragment control plane, got %v", cfg.ControlPlanes) + } + if len(cfg.Workers) != 1 || cfg.Workers[0] != "titan-05" { + t.Fatalf("expected adjacent fragment workers, got %v", cfg.Workers) + } + if cfg.Startup.RequiredNodeLabelsMode != "validate" { + t.Fatalf("expected adjacent fragment label mode validate, got %q", cfg.Startup.RequiredNodeLabelsMode) + } +} + +// TestBuildOrchestratorPropagatesLoadError runs one orchestration or CLI step. +// Signature: TestBuildOrchestratorPropagatesLoadError(t *testing.T). +// Why: missing host configs must fail before any recovery runner is built. +func TestBuildOrchestratorPropagatesLoadError(t *testing.T) { + missing := filepath.Join(t.TempDir(), "missing.yaml") + if _, orch, err := buildOrchestrator(log.New(io.Discard, "", 0), missing, true); err == nil || orch != nil { + t.Fatalf("expected missing config to fail without orchestrator, orch=%v err=%v", orch, err) + } +} + // TestBuildOrchestratorFailsWhenStateDirIsFile runs one orchestration or CLI step. // Signature: TestBuildOrchestratorFailsWhenStateDirIsFile(t *testing.T). // Why: covers EnsureDir error path in orchestrator builder. @@ -116,3 +167,59 @@ state: t.Fatalf("expected buildOrchestrator failure when reports dir is a file") } } + +// TestConfigFragmentPathsUsesEnvAndAdjacentInventory runs one orchestration or CLI step. +// Signature: TestConfigFragmentPathsUsesEnvAndAdjacentInventory(t *testing.T). +// Why: host installs should be able to consume Terraform-generated inventory +// without changing the systemd command line. +func TestConfigFragmentPathsUsesEnvAndAdjacentInventory(t *testing.T) { + tmp := t.TempDir() + cfgPath := filepath.Join(tmp, "ananke.yaml") + defaultFragment := filepath.Join(tmp, "ananke.inventory.yaml") + envFragment := filepath.Join(tmp, "env-fragment.yaml") + if err := os.WriteFile(defaultFragment, []byte("ssh_managed_nodes: []\n"), 0o644); err != nil { + t.Fatalf("write default fragment: %v", err) + } + t.Setenv("ANANKE_CONFIG_FRAGMENTS", envFragment) + + paths := configFragmentPaths(cfgPath) + if len(paths) != 2 { + t.Fatalf("expected env and adjacent fragments, got %v", paths) + } + if paths[0] != envFragment || paths[1] != defaultFragment { + t.Fatalf("unexpected fragment order: %v", paths) + } +} + +// TestConfigFragmentPathsSkipsMissingAndBlankEntries runs one orchestration or CLI step. +// Signature: TestConfigFragmentPathsSkipsMissingAndBlankEntries(t *testing.T). +// Why: optional generated fragments should be discovered deterministically without +// turning an absent adjacent file into a startup blocker. +func TestConfigFragmentPathsSkipsMissingAndBlankEntries(t *testing.T) { + tmp := t.TempDir() + envFragment := filepath.Join(tmp, "env.yaml") + t.Setenv("ANANKE_CONFIG_FRAGMENTS", string(os.PathListSeparator)+envFragment+string(os.PathListSeparator)+" ") + + paths := configFragmentPaths(filepath.Join(tmp, "ananke.yaml")) + if len(paths) != 1 || paths[0] != envFragment { + t.Fatalf("expected only env fragment, got %v", paths) + } +} + +// TestDefaultInventoryFragmentPathVariants runs one orchestration or CLI step. +// Signature: TestDefaultInventoryFragmentPathVariants(t *testing.T). +// Why: generated inventory naming should stay stable for config paths with and +// without a YAML extension. +func TestDefaultInventoryFragmentPathVariants(t *testing.T) { + tmp := t.TempDir() + cases := map[string]string{ + filepath.Join(tmp, "ananke.yaml"): filepath.Join(tmp, "ananke.inventory.yaml"), + filepath.Join(tmp, "ananke"): filepath.Join(tmp, "ananke.inventory.yaml"), + ".": "", + } + for in, want := range cases { + if got := defaultInventoryFragmentPath(in); got != want { + t.Fatalf("defaultInventoryFragmentPath(%q)=%q want %q", in, got, want) + } + } +} diff --git a/cmd/ananke/command_handlers_autoheal_test.go b/cmd/ananke/command_handlers_autoheal_test.go index 92e9185..fe050a6 100644 --- a/cmd/ananke/command_handlers_autoheal_test.go +++ b/cmd/ananke/command_handlers_autoheal_test.go @@ -2,8 +2,10 @@ package main import ( "context" + "errors" "io" "log" + "strings" "testing" "scm.bstein.dev/bstein/ananke/internal/cluster" @@ -38,3 +40,31 @@ func TestRunAutoHealInvokesPostStartRepair(t *testing.T) { t.Fatalf("expected auto-heal orchestrator hook to be called") } } + +// TestDefaultAutoHealWrapperHonorsDryRunOrchestrator runs one orchestration or CLI step. +// Signature: TestDefaultAutoHealWrapperHonorsDryRunOrchestrator(t *testing.T). +// Why: the default command hook should keep pointing at the orchestrator method, +// not only at test-injected replacements. +func TestDefaultAutoHealWrapperHonorsDryRunOrchestrator(t *testing.T) { + cfg := minimalHandlerConfig(t) + orch := newTestOrchestrator(cfg, true) + if err := autoHealOrchestratorCommand(context.Background(), orch); err != nil { + t.Fatalf("default auto-heal wrapper failed: %v", err) + } +} + +// TestRunAutoHealBuildError runs one orchestration or CLI step. +// Signature: TestRunAutoHealBuildError(t *testing.T). +// Why: command handlers should return config/build failures before invoking +// repair actions. +func TestRunAutoHealBuildError(t *testing.T) { + restore := stubCommandHandlerHooks() + defer restore() + buildOrchestratorCommand = func(_ *log.Logger, _ string, _ bool) (config.Config, *cluster.Orchestrator, error) { + return config.Config{}, nil, errors.New("build failed") + } + err := runAutoHeal(log.New(io.Discard, "", 0), []string{"--config", "/bad.yaml"}) + if err == nil || !strings.Contains(err.Error(), "build failed") { + t.Fatalf("expected build failure, got %v", err) + } +} diff --git a/configs/ananke.example.yaml b/configs/ananke.example.yaml index 059b27d..f1059d2 100644 --- a/configs/ananke.example.yaml +++ b/configs/ananke.example.yaml @@ -56,6 +56,7 @@ startup: required_node_labels: titan-09: ananke.bstein.dev/harbor-bootstrap: "true" + required_node_labels_mode: enforce require_time_sync: true time_sync_wait_seconds: 240 time_sync_poll_seconds: 5 diff --git a/configs/ananke.tethys.yaml b/configs/ananke.tethys.yaml index 2fc00fc..520d75e 100644 --- a/configs/ananke.tethys.yaml +++ b/configs/ananke.tethys.yaml @@ -166,6 +166,7 @@ startup: titan-09: node-role.kubernetes.io/worker: "true" ananke.bstein.dev/harbor-bootstrap: "true" + required_node_labels_mode: enforce require_time_sync: true time_sync_wait_seconds: 240 time_sync_poll_seconds: 5 diff --git a/configs/ananke.titan-db.yaml b/configs/ananke.titan-db.yaml index 02972c9..a2c7d7f 100644 --- a/configs/ananke.titan-db.yaml +++ b/configs/ananke.titan-db.yaml @@ -166,6 +166,7 @@ startup: titan-09: node-role.kubernetes.io/worker: "true" ananke.bstein.dev/harbor-bootstrap: "true" + required_node_labels_mode: enforce require_time_sync: true time_sync_wait_seconds: 240 time_sync_poll_seconds: 5 diff --git a/internal/cluster/orchestrator_autorepair.go b/internal/cluster/orchestrator_autorepair.go index 1920ce2..f4fbf94 100644 --- a/internal/cluster/orchestrator_autorepair.go +++ b/internal/cluster/orchestrator_autorepair.go @@ -205,65 +205,6 @@ func (o *Orchestrator) rerunVaultK8sAuthConfigJob(ctx context.Context) error { return nil } -// cleanupTerminatingPodsOnUnavailableNodes runs one orchestration or CLI step. -// Signature: (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error). -// Why: dead nodes can strand terminating pods indefinitely, so the daemon should -// clear only that narrow failure class instead of leaving garbage behind forever. -func (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error) { - if o.runner.DryRun { - return 0, nil - } - - unavailable, err := o.unavailableNodeSet(ctx) - if err != nil { - return 0, err - } - if len(unavailable) == 0 { - return 0, nil - } - - out, err := o.kubectl(ctx, 30*time.Second, "get", "pods", "-A", "-o", "json") - if err != nil { - return 0, fmt.Errorf("query pods: %w", err) - } - var pods podDeleteList - if err := json.Unmarshal([]byte(out), &pods); err != nil { - return 0, fmt.Errorf("decode pods: %w", err) - } - - grace := time.Duration(o.cfg.Startup.DeadNodeCleanupGraceSeconds) * time.Second - now := time.Now() - count := 0 - for _, item := range pods.Items { - if item.Metadata.DeletionTimestamp == nil || item.Spec.NodeName == "" { - continue - } - if _, badNode := unavailable[item.Spec.NodeName]; !badNode { - continue - } - if now.Sub(*item.Metadata.DeletionTimestamp) < grace { - continue - } - o.log.Printf("warning: force deleting terminating pod %s/%s on unavailable node %s", item.Metadata.Namespace, item.Metadata.Name, item.Spec.NodeName) - if _, err := o.kubectl( - ctx, - 20*time.Second, - "-n", item.Metadata.Namespace, - "delete", "pod", item.Metadata.Name, - "--grace-period=0", - "--force", - "--wait=false", - ); err != nil && !isNotFoundErr(err) { - return count, fmt.Errorf("delete pod %s/%s: %w", item.Metadata.Namespace, item.Metadata.Name, err) - } - count++ - } - if count > 0 { - o.log.Printf("post-start auto-heal cleaned %d terminating pod(s) from unavailable nodes", count) - } - return count, nil -} - // repairBrokenKubeletProxies runs one orchestration or CLI step. // Signature: (o *Orchestrator) repairBrokenKubeletProxies(ctx context.Context) (int, error). // Why: a Ready node can still have a dead kubelet tunnel, which breaks Jenkins @@ -422,35 +363,6 @@ func isRepairableKubeletProxyErr(err error) bool { return false } -// unavailableNodeSet runs one orchestration or CLI step. -// Signature: (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error). -// Why: isolates Ready-condition parsing so dead-node cleanup stays targeted. -func (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error) { - out, err := o.kubectl(ctx, 20*time.Second, "get", "nodes", "-o", "json") - if err != nil { - return nil, fmt.Errorf("query nodes: %w", err) - } - var nodes nodeReadyList - if err := json.Unmarshal([]byte(out), &nodes); err != nil { - return nil, fmt.Errorf("decode nodes: %w", err) - } - - unavailable := map[string]struct{}{} - for _, item := range nodes.Items { - ready := "" - for _, cond := range item.Status.Conditions { - if strings.EqualFold(strings.TrimSpace(cond.Type), "Ready") { - ready = strings.TrimSpace(cond.Status) - break - } - } - if ready != "True" { - unavailable[item.Metadata.Name] = struct{}{} - } - } - return unavailable, nil -} - // requestFluxReconcile runs one orchestration or CLI step. // Signature: (o *Orchestrator) requestFluxReconcile(ctx context.Context) error. // Why: post-start repairs need a lightweight way to refresh GitOps health diff --git a/internal/cluster/orchestrator_autorepair_cleanup.go b/internal/cluster/orchestrator_autorepair_cleanup.go new file mode 100644 index 0000000..279d660 --- /dev/null +++ b/internal/cluster/orchestrator_autorepair_cleanup.go @@ -0,0 +1,97 @@ +package cluster + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" +) + +// cleanupTerminatingPodsOnUnavailableNodes runs one orchestration or CLI step. +// Signature: (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error). +// Why: dead nodes can strand terminating pods indefinitely, so the daemon should +// clear only that narrow failure class instead of leaving garbage behind forever. +func (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error) { + if o.runner.DryRun { + return 0, nil + } + + unavailable, err := o.unavailableNodeSet(ctx) + if err != nil { + return 0, err + } + if len(unavailable) == 0 { + return 0, nil + } + + out, err := o.kubectl(ctx, 30*time.Second, "get", "pods", "-A", "-o", "json") + if err != nil { + return 0, fmt.Errorf("query pods: %w", err) + } + var pods podDeleteList + if err := json.Unmarshal([]byte(out), &pods); err != nil { + return 0, fmt.Errorf("decode pods: %w", err) + } + + grace := time.Duration(o.cfg.Startup.DeadNodeCleanupGraceSeconds) * time.Second + now := time.Now() + count := 0 + for _, item := range pods.Items { + if item.Metadata.DeletionTimestamp == nil || item.Spec.NodeName == "" { + continue + } + if _, badNode := unavailable[item.Spec.NodeName]; !badNode { + continue + } + if now.Sub(*item.Metadata.DeletionTimestamp) < grace { + continue + } + o.log.Printf("warning: force deleting terminating pod %s/%s on unavailable node %s", item.Metadata.Namespace, item.Metadata.Name, item.Spec.NodeName) + if _, err := o.kubectl( + ctx, + 20*time.Second, + "-n", item.Metadata.Namespace, + "delete", "pod", item.Metadata.Name, + "--grace-period=0", + "--force", + "--wait=false", + ); err != nil && !isNotFoundErr(err) { + return count, fmt.Errorf("delete pod %s/%s: %w", item.Metadata.Namespace, item.Metadata.Name, err) + } + count++ + } + if count > 0 { + o.log.Printf("post-start auto-heal cleaned %d terminating pod(s) from unavailable nodes", count) + } + return count, nil +} + +// unavailableNodeSet runs one orchestration or CLI step. +// Signature: (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error). +// Why: isolates Ready-condition parsing so dead-node cleanup stays targeted. +func (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error) { + out, err := o.kubectl(ctx, 20*time.Second, "get", "nodes", "-o", "json") + if err != nil { + return nil, fmt.Errorf("query nodes: %w", err) + } + var nodes nodeReadyList + if err := json.Unmarshal([]byte(out), &nodes); err != nil { + return nil, fmt.Errorf("decode nodes: %w", err) + } + + unavailable := map[string]struct{}{} + for _, item := range nodes.Items { + ready := "" + for _, cond := range item.Status.Conditions { + if strings.EqualFold(strings.TrimSpace(cond.Type), "Ready") { + ready = strings.TrimSpace(cond.Status) + break + } + } + if ready != "True" { + unavailable[item.Metadata.Name] = struct{}{} + } + } + return unavailable, nil +} diff --git a/internal/cluster/orchestrator_drain.go b/internal/cluster/orchestrator_drain.go index c4dff26..c0db317 100644 --- a/internal/cluster/orchestrator_drain.go +++ b/internal/cluster/orchestrator_drain.go @@ -164,7 +164,12 @@ func (o *Orchestrator) ensureLonghornEncryptedHostPrereqs(ctx context.Context, w exempt := makeStringSet(o.cfg.Startup.LonghornCryptsetupExemptNodes) unsafe := map[string]struct{}{} var errs []string + nodes := make([]string, 0, len(longhornHosts)) for node := range longhornHosts { + nodes = append(nodes, node) + } + sort.Strings(nodes) + for _, node := range nodes { if _, skip := ignored[node]; skip { continue } diff --git a/internal/cluster/orchestrator_ingress.go b/internal/cluster/orchestrator_ingress.go index 134c383..0bae2a6 100644 --- a/internal/cluster/orchestrator_ingress.go +++ b/internal/cluster/orchestrator_ingress.go @@ -19,6 +19,9 @@ func (o *Orchestrator) ensureRequiredNodeLabels(ctx context.Context) error { if o.runner.DryRun || len(o.cfg.Startup.RequiredNodeLabels) == 0 { return nil } + if o.cfg.Startup.RequiredNodeLabelsMode == "validate" { + return o.validateRequiredNodeLabels(ctx) + } ignored := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes) nodes := make([]string, 0, len(o.cfg.Startup.RequiredNodeLabels)) for node := range o.cfg.Startup.RequiredNodeLabels { @@ -72,6 +75,83 @@ func (o *Orchestrator) ensureRequiredNodeLabels(ctx context.Context) error { return nil } +type nodeLabelItem struct { + Metadata struct { + Labels map[string]string `json:"labels"` + } `json:"metadata"` +} + +// validateRequiredNodeLabels runs one orchestration or CLI step. +// Signature: (o *Orchestrator) validateRequiredNodeLabels(ctx context.Context) error. +// Why: lets Terraform own steady-state labels while Ananke still reports drift +// during startup and post-start health checks. +func (o *Orchestrator) validateRequiredNodeLabels(ctx context.Context) error { + ignored := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes) + nodes := make([]string, 0, len(o.cfg.Startup.RequiredNodeLabels)) + for node := range o.cfg.Startup.RequiredNodeLabels { + node = strings.TrimSpace(node) + if node != "" { + nodes = append(nodes, node) + } + } + sort.Strings(nodes) + + drift := []string{} + for _, node := range nodes { + if _, skip := ignored[node]; skip { + o.log.Printf("skipping required node label validation for ignored unavailable node %s", node) + continue + } + required := o.cfg.Startup.RequiredNodeLabels[node] + if len(required) == 0 { + continue + } + keys := make([]string, 0, len(required)) + for key := range required { + key = strings.TrimSpace(key) + if key != "" { + keys = append(keys, key) + } + } + sort.Strings(keys) + if len(keys) == 0 { + continue + } + + out, err := o.kubectl(ctx, 25*time.Second, "get", "node", node, "-o", "json") + if err != nil { + if isNotFoundErr(err) && !o.startupNodeStrictlyRequired(node) { + o.log.Printf("warning: skipping required label validation for absent non-core node %s: %v", node, err) + o.noteStartupAutoHeal(fmt.Sprintf("skipped required node label validation for absent non-core node %s", node)) + continue + } + return fmt.Errorf("validate required node labels on %s: %w", node, err) + } + var item nodeLabelItem + if err := json.Unmarshal([]byte(out), &item); err != nil { + return fmt.Errorf("decode node labels for %s: %w", node, err) + } + for _, key := range keys { + want := strings.TrimSpace(required[key]) + if want == "" { + continue + } + got := strings.TrimSpace(item.Metadata.Labels[key]) + if got != want { + if got == "" { + got = "" + } + drift = append(drift, fmt.Sprintf("%s %s want=%s got=%s", node, key, want, got)) + } + } + } + if len(drift) > 0 { + return fmt.Errorf("required node label drift: %s", strings.Join(drift, "; ")) + } + o.log.Printf("validated required node labels on %d node(s)", len(nodes)) + return nil +} + // waitForStartupConvergence runs one orchestration or CLI step. // Signature: (o *Orchestrator) waitForStartupConvergence(ctx context.Context) error. // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve. diff --git a/internal/cluster/orchestrator_lifecycle.go b/internal/cluster/orchestrator_lifecycle.go index 05a58a6..b23185a 100644 --- a/internal/cluster/orchestrator_lifecycle.go +++ b/internal/cluster/orchestrator_lifecycle.go @@ -11,6 +11,8 @@ import ( "scm.bstein.dev/bstein/ananke/internal/state" ) +var etcdRestorePostStartDelay = 10 * time.Second + // Startup runs one orchestration or CLI step. // Signature: (o *Orchestrator) Startup(ctx context.Context, opts StartupOptions) (err error). // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve. @@ -433,7 +435,7 @@ func (o *Orchestrator) EtcdRestore(ctx context.Context, opts EtcdRestoreOptions) if _, err := o.ssh(ctx, controlPlane, "sudo systemctl start k3s || true"); err != nil { return fmt.Errorf("failed to start k3s on restore control plane %s: %w", controlPlane, err) } - time.Sleep(10 * time.Second) + time.Sleep(etcdRestorePostStartDelay) for _, cp := range o.cfg.ControlPlanes { cp := cp if cp == controlPlane { diff --git a/internal/cluster/orchestrator_quality_gate_closeout_test.go b/internal/cluster/orchestrator_quality_gate_closeout_test.go new file mode 100644 index 0000000..30a8558 --- /dev/null +++ b/internal/cluster/orchestrator_quality_gate_closeout_test.go @@ -0,0 +1,442 @@ +package cluster + +import ( + "bufio" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "log" + "net" + "strings" + "testing" + "time" + + "scm.bstein.dev/bstein/ananke/internal/config" + "scm.bstein.dev/bstein/ananke/internal/execx" + "scm.bstein.dev/bstein/ananke/internal/state" +) + +// TestTCPServiceChecklistAndBackendHealBranches runs one orchestration or CLI step. +// Signature: TestTCPServiceChecklistAndBackendHealBranches(t *testing.T). +// Why: TCP health checks drive Mailu recovery paths and need branch coverage for +// banner matching, unnamed checks, duplicate backend suppression, and heal notes. +func TestTCPServiceChecklistAndBackendHealBranches(t *testing.T) { + addr := startTCPFixture(t, "220 ready\r\n", "250 pong\r\n") + host, portText, err := net.SplitHostPort(addr) + if err != nil { + t.Fatalf("split tcp fixture address: %v", err) + } + var port int + if _, err := fmt.Sscanf(portText, "%d", &port); err != nil { + t.Fatalf("parse tcp fixture port: %v", err) + } + + cfg := config.Config{ + Startup: config.Startup{ + TCPServiceChecklist: []config.TCPServiceChecklistCheck{ + {Name: "smtp", Host: host, Port: port, Send: "PING\r\n", ExpectContains: "pong", Namespace: "mail", Service: "mailu"}, + }, + }, + } + orch := buildOrchestratorWithStubs(t, cfg, nil) + ok, detail := orch.tcpServiceChecklistReady(context.Background()) + if !ok || detail != "tcp-checks=1" { + t.Fatalf("expected tcp checklist success, ok=%v detail=%q", ok, detail) + } + + unnamed := config.TCPServiceChecklistCheck{Host: host, Port: port, ExpectContains: "missing"} + ok, detail = orch.tcpServiceCheckReady(context.Background(), unnamed) + if ok || !strings.Contains(detail, `banner missing expected marker "missing"`) { + t.Fatalf("expected missing marker detail, ok=%v detail=%q", ok, detail) + } + orch.cfg.Startup.TCPServiceChecklist = []config.TCPServiceChecklistCheck{unnamed} + ok, detail = orch.tcpServiceChecklistReady(context.Background()) + if ok || !strings.Contains(detail, host+":"+portText) { + t.Fatalf("expected unnamed tcp check to use host:port detail, ok=%v detail=%q", ok, detail) + } + + healAddr := startTCPFixture(t, "220 ready\r\n", "250 pong\r\n") + healHost, healPortText, _ := net.SplitHostPort(healAddr) + var healPort int + _, _ = fmt.Sscanf(healPortText, "%d", &healPort) + cfg.Startup.TCPServiceChecklist = []config.TCPServiceChecklistCheck{ + {Host: healHost, Port: healPort, Namespace: "", Service: "skip"}, + {Name: "ok", Host: healHost, Port: healPort, ExpectContains: "ready", Namespace: "mail", Service: "mailu"}, + {Name: "bad", Host: healHost, Port: healPort, ExpectContains: "absent", Namespace: "mail", Service: "mailu"}, + {Name: "dup", Host: healHost, Port: healPort, ExpectContains: "absent", Namespace: "mail", Service: "mailu"}, + } + var scaled, rolled bool + orch = buildOrchestratorWithStubs(t, cfg, []commandStub{ + {match: func(name string, args []string) bool { + joined := strings.Join(args, " ") + if name == "kubectl" && strings.Contains(joined, "-n mail scale deployment mailu --replicas=1") { + scaled = true + return true + } + if name == "kubectl" && strings.Contains(joined, "-n mail rollout status deployment/mailu") { + rolled = true + return true + } + return false + }}, + {match: matchContains("kubectl", "-n", "mail", "scale", "statefulset", "mailu"), err: errors.New(`Error from server (NotFound): statefulsets.apps "mailu" not found`)}, + }) + healed, err := orch.healFailedTCPServiceBackends(context.Background()) + if err != nil { + t.Fatalf("healFailedTCPServiceBackends: %v", err) + } + if len(healed) != 1 || healed[0] != "mail/deployment/mailu" || !scaled || !rolled { + t.Fatalf("unexpected tcp backend heal result healed=%v scaled=%v rolled=%v", healed, scaled, rolled) + } + lastAttempt := time.Now().Add(-time.Minute) + orch.maybeAutoHealTCPServiceBackends(context.Background(), &lastAttempt) +} + +// TestHostPrivilegeFailureAndSecretBranches runs one orchestration or CLI step. +// Signature: TestHostPrivilegeFailureAndSecretBranches(t *testing.T). +// Why: host repair depends on stable allowlist, secret, sudo, and failure +// classes before Ananke can safely restart or reboot managed nodes. +func TestHostPrivilegeFailureAndSecretBranches(t *testing.T) { + cfg := config.Config{ + SSHManagedNodes: []string{"titan-23"}, + SSHUser: "atlas", + Startup: config.Startup{ + HostSudoSecretNamespace: "host-secrets", + HostSudoSecretNameTemplate: "sudo-{node}", + HostSudoSecretPasswordKey: "password", + }, + } + orch := buildOrchestratorWithStubs(t, cfg, nil) + if _, err := orch.runHostPrivilegedAction(context.Background(), " ", hostActionSudoPreflight, 0); err == nil { + t.Fatalf("expected empty node privilege error") + } + if _, err := orch.runHostPrivilegedAction(context.Background(), "unmanaged", hostActionSudoPreflight, 0); err == nil { + t.Fatalf("expected unmanaged node privilege error") + } + if _, err := orch.runHostPrivilegedAction(context.Background(), "titan-23", hostPrivilegedAction("bad-action"), 0); err == nil { + t.Fatalf("expected bad action privilege error") + } + if got := orch.hostPrivilegedCommandTimeout(); got != 90*time.Second { + t.Fatalf("expected default privileged timeout, got %s", got) + } + orch.cfg.Startup.HostPrivilegedCommandTimeoutSec = 7 + if got := orch.hostPrivilegedCommandTimeout(); got != 7*time.Second { + t.Fatalf("expected configured privileged timeout, got %s", got) + } + + actions := []struct { + action hostPrivilegedAction + args []string + want string + }{ + {hostActionHostReboot, nil, "systemctl reboot"}, + {hostActionModprobeDMCrypt, nil, "modprobe dm_crypt"}, + {hostActionCrictlPods, nil, "crictl pods -o json"}, + {hostActionCrictlPs, nil, "crictl ps -a -o json"}, + {hostActionCrictlInspect, []string{"abc123"}, "crictl inspect abc123"}, + {hostActionCrictlStop, []string{"abc123"}, "crictl stop abc123"}, + } + for _, tc := range actions { + got, err := hostPrivilegedCommandArgs(tc.action, tc.args...) + if err != nil { + t.Fatalf("hostPrivilegedCommandArgs(%s): %v", tc.action, err) + } + if strings.Join(got, " ") != tc.want { + t.Fatalf("hostPrivilegedCommandArgs(%s)=%q want %q", tc.action, strings.Join(got, " "), tc.want) + } + } + for _, tc := range []struct { + action hostPrivilegedAction + arg string + }{ + {hostActionCrictlInspect, "bad;id"}, + {hostActionCrictlStop, ""}, + } { + if _, err := hostPrivilegedCommandArgs(tc.action, tc.arg); err == nil { + t.Fatalf("expected unsafe arg rejection for %s", tc.action) + } + } + + errs := []error{ + classifyHostPrivilegeFailure("n", hostActionSudoPreflight, context.DeadlineExceeded, ""), + classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("try again"), ""), + classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("permission denied publickey"), ""), + classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("exit 1"), "line one\nline two"), + } + wantClasses := []string{"host-command-timeout", "host-privilege-auth-failed", "host-ssh-unavailable", "host-command-failed"} + for i, err := range errs { + if !strings.Contains(err.Error(), wantClasses[i]) { + t.Fatalf("expected %s in %q", wantClasses[i], err) + } + } + + secretJSON := `{"data":{"password":"` + base64.StdEncoding.EncodeToString([]byte("pw\n")) + `"}}` + orch = buildOrchestratorWithStubs(t, cfg, []commandStub{ + {match: matchContains("ssh", "sudo -n /usr/bin/systemctl --version"), out: "sudo: a password is required", err: errors.New("sudo failed")}, + {match: matchContains("kubectl", "-n", "host-secrets", "get", "secret", "sudo-titan-23"), out: secretJSON}, + }) + orch.SetSSHInputOverride(func(_ context.Context, _ time.Duration, node, command, input string) (string, error) { + if node != "titan-23" || !strings.Contains(command, "sudo -S") || input != "pw\n" { + t.Fatalf("unexpected password-backed ssh node=%q command=%q input=%q", node, command, input) + } + return "incorrect password", errors.New("try again") + }) + if _, err := orch.runHostPrivilegedAction(context.Background(), "titan-23", hostActionSudoPreflight, 0); err == nil || !strings.Contains(err.Error(), "host-privilege-auth-failed") { + t.Fatalf("expected password-backed auth failure, got %v", err) + } + + secretCases := []struct { + name string + cfg config.Config + node string + out string + err error + class string + }{ + {name: "unconfigured", cfg: config.Config{}, class: "secret-lookup-unconfigured"}, + {name: "empty-name", cfg: config.Config{Startup: config.Startup{HostSudoSecretNamespace: "ns", HostSudoSecretNameTemplate: "{node}"}}, node: " ", class: "secret-lookup-invalid"}, + {name: "query-error", cfg: cfg, err: errors.New("denied"), class: "secret-lookup-failed"}, + {name: "bad-json", cfg: cfg, out: "{", class: "secret-decode-failed"}, + {name: "missing-key", cfg: cfg, out: `{"data":{"other":"cHc="}}`, class: "secret-key-missing"}, + {name: "bad-base64", cfg: cfg, out: `{"data":{"password":"%"}}`, class: "secret-decode-failed"}, + {name: "empty-password", cfg: cfg, out: `{"data":{"password":"Cg=="}}`, class: "secret-empty"}, + } + for _, tc := range secretCases { + orch := buildOrchestratorWithStubs(t, tc.cfg, []commandStub{ + {match: matchContains("kubectl", "get", "secret"), out: tc.out, err: tc.err}, + }) + node := tc.node + if node == "" { + node = "titan-23" + } + _, class, err := orch.hostSudoPassword(context.Background(), node) + if err == nil || class != tc.class { + t.Fatalf("%s: expected class %q error, got class=%q err=%v", tc.name, tc.class, class, err) + } + } +} + +// TestImagePullDNSAndCredentialBranches runs one orchestration or CLI step. +// Signature: TestImagePullDNSAndCredentialBranches(t *testing.T). +// Why: image-pull blockers should be classified instead of blindly recycling +// pods when DNS or registry credentials are the real outage cause. +func TestImagePullDNSAndCredentialBranches(t *testing.T) { + eventJSON := `{"items":[ +{"type":"Normal","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"skip","name":"normal"},"message":"lookup registry.bstein.dev no such host"}, +{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Service","namespace":"skip","name":"svc"},"message":"lookup registry.bstein.dev no such host"}, +{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"","name":""},"message":"lookup registry.bstein.dev no such host"}, +{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"media","name":"app"},"message":"failed to pull image: lookup registry.bstein.dev: no such host"}, +{"type":"Warning","reason":"FailedPull","metadata":{"namespace":"finance"},"involvedObject":{"kind":"Pod","name":"budget"},"message":"unauthorized: authentication required"}, +{"type":"Warning","reason":"FailedToRetrieveImagePullSecret","involvedObject":{"kind":"Pod","namespace":"sso","name":"keycloak"},"message":"unable to retrieve some image pull secrets"} +]}` + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), out: eventJSON}, + }) + dnsReasons, err := orch.imagePullDNSBlockerReasons(context.Background()) + if err != nil { + t.Fatalf("imagePullDNSBlockerReasons: %v", err) + } + if dnsReasons["media/app"] != "ImagePullDNSBlocker:registry.bstein.dev" { + t.Fatalf("unexpected DNS reasons: %#v", dnsReasons) + } + credentialReasons, err := orch.imagePullCredentialBlockerReasons(context.Background()) + if err != nil { + t.Fatalf("imagePullCredentialBlockerReasons: %v", err) + } + if credentialReasons["finance/budget"] != "ImagePullCredentialBlocker:unauthorized" || credentialReasons["sso/keycloak"] != "ImagePullCredentialBlocker:missing-pull-secret" { + t.Fatalf("unexpected credential reasons: %#v", credentialReasons) + } + for _, message := range []string{ + "invalid username/password", + "403 forbidden", + "pull access denied", + "registry returned something strange", + } { + if got := imagePullCredentialFailureClass("Failed", message); got == "" { + t.Fatalf("expected credential class for %q", message) + } + } + if !vaultSyncDeployment("worker", map[string]string{"app": "secret-vault-sync"}) { + t.Fatalf("expected vault-sync label value to identify sync deployment") + } + for _, message := range []string{"no resolver marker here", "lookup ", "lookup []"} { + if got := imagePullRegistryHost(message); got != "unknown-registry" { + t.Fatalf("expected unknown registry for %q, got %q", message, got) + } + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "finance", "get", "deployment"), out: `{"items":[{"metadata":{"name":"finance-vault-sync","labels":{}}},{"metadata":{"name":"app","labels":{}}}]}`}, + {match: matchContains("kubectl", "-n", "finance", "rollout", "restart", "deployment", "finance-vault-sync"), out: ""}, + {match: matchContains("kubectl", "-n", "finance", "rollout", "status", "deployment/finance-vault-sync"), out: ""}, + }) + repaired, err := orch.restartVaultSyncDeployments(context.Background(), "finance") + if err != nil || len(repaired) != 1 || repaired[0] != "finance/deployment/finance-vault-sync" { + t.Fatalf("unexpected vault-sync repair repaired=%v err=%v", repaired, err) + } + for _, tc := range []struct { + name string + out string + err error + }{ + {name: "query", err: errors.New("api down")}, + {name: "decode", out: "{"}, + {name: "missing", out: `{"items":[{"metadata":{"name":"app","labels":{}}}]}`}, + {name: "restart", out: `{"items":[{"metadata":{"name":"vault-sync","labels":{}}}]}`, err: errors.New("restart denied")}, + } { + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "ns", "get", "deployment"), out: tc.out, err: mapRestartErr(tc.name, "query", tc.err)}, + {match: matchContains("kubectl", "-n", "ns", "rollout", "restart"), err: mapRestartErr(tc.name, "restart", tc.err)}, + }) + if _, err := orch.restartVaultSyncDeployments(context.Background(), "ns"); err == nil { + t.Fatalf("%s: expected restartVaultSyncDeployments error", tc.name) + } + } +} + +// TestLonghornSelectorAndRuntimeRecoveryBranches runs one orchestration or CLI step. +// Signature: TestLonghornSelectorAndRuntimeRecoveryBranches(t *testing.T). +// Why: Longhorn manager drift and kubelet proxy recovery are safety-critical +// repair paths that should stay covered without a live cluster. +func TestLonghornSelectorAndRuntimeRecoveryBranches(t *testing.T) { + if selectorLabelString(map[string]string{"b": "2", "a": "1"}) != "a=1,b=2" { + t.Fatalf("selector labels should be sorted") + } + if podMatchesLabels(nil, map[string]string{"app": "longhorn"}) { + t.Fatalf("nil pod labels must not match selector") + } + if len(missingSelectorLabels(map[string]string{"app": "old"}, map[string]string{"app": "longhorn"})) != 1 { + t.Fatalf("expected missing selector label") + } + + cfg := config.Config{} + var labeled bool + orch := buildOrchestratorWithStubs(t, cfg, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[ +{"metadata":{"name":""}}, +{"metadata":{"name":"missing-kube"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}, +{"metadata":{"name":"has-manager"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}, +{"metadata":{"name":"ready"},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ManagerPodMissing"}]}}, +{"metadata":{"name":"other-reason"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"DiskPressure"}]}}, +{"metadata":{"name":"already-labelled"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}, +{"metadata":{"name":"repair"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}} +]}`}, + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true","skip":""}}}}`}, + {match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[ +{"metadata":{"name":"has-manager","labels":{}}}, +{"metadata":{"name":"ready","labels":{}}}, +{"metadata":{"name":"other-reason","labels":{}}}, +{"metadata":{"name":"already-labelled","labels":{"longhorn-host":"true"}}}, +{"metadata":{"name":"repair","labels":{}}} +]}`}, + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json", "-l", "longhorn-host=true"), out: `{"items":[{"metadata":{"labels":{"longhorn-host":"true"}},"spec":{"nodeName":"has-manager"}},{"metadata":{"labels":{"longhorn-host":"false"}},"spec":{"nodeName":"repair"}},{"metadata":{"labels":{"longhorn-host":"true"}},"spec":{"nodeName":""}}]}`}, + {match: func(name string, args []string) bool { + if name == "kubectl" && strings.Contains(strings.Join(args, " "), "label node repair --overwrite longhorn-host=true") { + labeled = true + return true + } + return false + }, out: ""}, + }) + repaired, err := orch.reconcileLonghornKubernetesReadiness(context.Background()) + if err != nil || repaired != 1 || !labeled { + t.Fatalf("unexpected longhorn reconcile repaired=%d labeled=%v err=%v", repaired, labeled, err) + } + + errOrch := buildOrchestratorWithStubs(t, cfg, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[{"metadata":{"name":"repair"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`}, + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true"}}}}`}, + {match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"repair","labels":{}}}]}`}, + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json"), out: `{"items":[]}`}, + {match: matchContains("kubectl", "label", "node", "repair"), err: errors.New("label denied")}, + }) + if repaired, err := errOrch.reconcileLonghornKubernetesReadiness(context.Background()); err == nil || repaired != 0 { + t.Fatalf("expected longhorn label error, repaired=%d err=%v", repaired, err) + } + + runtimeCfg := config.Config{ + SSHManagedNodes: []string{"titan-23"}, + SSHUser: "atlas", + Startup: config.Startup{ + NodeRuntimeRestartWaitSeconds: 1, + NodeRuntimeRebootWaitSeconds: 1, + HostRepairAllowReboot: true, + }, + } + orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{ + {match: matchContains("ssh", "sudo -n systemctl --no-block restart k3s-agent"), out: ""}, + {match: matchContains("kubectl", "wait", "node/titan-23"), err: errors.New("not ready")}, + {match: matchContains("ssh", "sudo -n systemctl show k3s-agent"), out: "ActiveState=deactivating\nSubState=stop-sigterm"}, + {match: matchContains("ssh", "sudo -n systemctl reboot"), out: ""}, + }) + if err := orch.recoverManagedNodeRuntime(context.Background(), "titan-23", true, "proxy broken"); err == nil || !strings.Contains(err.Error(), "node did not recover after controlled reboot") { + t.Fatalf("expected reboot wait failure, got %v", err) + } + if err := orch.recoverManagedNodeRuntime(context.Background(), " ", true, ""); err == nil { + t.Fatalf("expected empty runtime node error") + } + if !k3sAgentStateRequiresReboot("Result=timeout") || k3sAgentStateRequiresReboot("ActiveState=active") { + t.Fatalf("unexpected k3s-agent reboot predicate") + } + if summarizeSystemctlShow("") != "unknown" || !strings.Contains(summarizeSystemctlShow("ActiveState=activating\nSubState=start"), "ActiveState") { + t.Fatalf("unexpected systemctl summary") + } +} + +// startTCPFixture runs one orchestration or CLI step. +// Signature: startTCPFixture(t *testing.T, banner string, reply string) string. +// Why: TCP checklist tests need a local loopback endpoint that emits the same +// banner/response shape as SMTP-like protocol probes. +func startTCPFixture(t *testing.T, banner string, reply string) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen tcp fixture: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + _, _ = conn.Write([]byte(banner)) + _ = conn.SetDeadline(time.Now().Add(2 * time.Second)) + _, _ = bufio.NewReader(conn).ReadString('\n') + _, _ = conn.Write([]byte(reply)) + }() + } + }() + return listener.Addr().String() +} + +// mapRestartErr runs one orchestration or CLI step. +// Signature: mapRestartErr(caseName, target string, err error) error. +// Why: table-driven deployment restart tests need concise error injection +// without repeating one-off closures for each command phase. +func mapRestartErr(caseName, target string, err error) error { + if caseName == target { + return err + } + return nil +} + +// TestLocalOrchestratorConstructorUsesQuietLogger runs one orchestration or CLI step. +// Signature: TestLocalOrchestratorConstructorUsesQuietLogger(t *testing.T). +// Why: keep imports in this closeout file anchored to real construction helpers +// while preserving package-local test readability. +func TestLocalOrchestratorConstructorUsesQuietLogger(t *testing.T) { + orch := &Orchestrator{ + cfg: config.Config{}, + runner: &execx.Runner{}, + store: state.New(t.TempDir() + "/runs.json"), + log: log.New(io.Discard, "", 0), + } + if orch.log == nil || orch.runner == nil || orch.store == nil { + t.Fatalf("expected constructed orchestrator test fixture") + } +} diff --git a/internal/cluster/orchestrator_quality_gate_final_closeout_test.go b/internal/cluster/orchestrator_quality_gate_final_closeout_test.go new file mode 100644 index 0000000..f8df2f2 --- /dev/null +++ b/internal/cluster/orchestrator_quality_gate_final_closeout_test.go @@ -0,0 +1,212 @@ +package cluster + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "scm.bstein.dev/bstein/ananke/internal/config" +) + +// TestFinalCoverageLeaseDrainAndSSHBranches runs one orchestration or CLI step. +// Signature: TestFinalCoverageLeaseDrainAndSSHBranches(t *testing.T). +// Why: close the last local-only safety branches for cordon release, Longhorn +// host preflight errors, and stdin-backed SSH known-host retry. +func TestFinalCoverageLeaseDrainAndSSHBranches(t *testing.T) { + ctx := context.Background() + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), err: errors.New("api down")}, + }) + if _, err := orch.ensureLonghornEncryptedHostPrereqs(ctx, []string{"worker-a"}); err == nil || !strings.Contains(err.Error(), "api down") { + t.Fatalf("expected longhorn host query error, got %v", err) + } + + future := time.Now().UTC().Add(time.Hour).Format(time.RFC3339) + nodesJSON := fmt.Sprintf(`{"items":[ +{"metadata":{"name":"clear-fails","annotations":{%q:"ananke"}},"spec":{"unschedulable":false}}, +{"metadata":{"name":"crypt-pending","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}} +]}`, + anankeCordonOwnerAnnotation, + anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonMissingCryptsetup, anankeCordonDeadlineAnnotation, future) + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: nodesJSON}, + {match: matchContains("kubectl", "annotate", "node", "clear-fails"), err: errors.New("clear denied")}, + {match: matchContains("ssh", "crypt-pending", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")}, + }) + if released, err := orch.enforceRecoveryCordonLeases(ctx); released != 0 || err == nil || !strings.Contains(err.Error(), "clear denied") { + t.Fatalf("expected clear-cordon error and pending repair log, released=%d err=%v", released, err) + } + + releaseErrOrch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("ssh", "crypt-release", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_PRESENT__"}, + {match: matchContains("kubectl", "get", "pods", "-A"), out: `{"items":[]}`}, + {match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`}, + {match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/proxy-release/proxy/healthz"), out: "ok"}, + {match: matchContains("kubectl", "uncordon"), err: errors.New("uncordon denied")}, + }) + for _, tc := range []struct { + node string + reason string + }{ + {"crypt-release", cordonReasonMissingCryptsetup}, + {"runtime-release", cordonReasonRuntimeWedge}, + {"proxy-release", cordonReasonKubeletProxy}, + } { + if recovered, err := releaseErrOrch.recoverLeasedCordon(ctx, tc.node, map[string]string{anankeCordonReasonAnnotation: tc.reason}); recovered || err == nil || !strings.Contains(err.Error(), "uncordon denied") { + t.Fatalf("expected %s uncordon denial, recovered=%v err=%v", tc.reason, recovered, err) + } + } + + binDir := t.TempDir() + countPath := filepath.Join(t.TempDir(), "ssh-count") + writeTestExecutable(t, filepath.Join(binDir, "ssh"), `#!/bin/sh +count="${ANANKE_FAKE_SSH_COUNT:?}" +n=0 +if [ -f "$count" ]; then n="$(cat "$count")"; fi +n=$((n + 1)) +printf '%s\n' "$n" > "$count" +if [ "$n" -eq 1 ]; then + printf '%s\n' 'Host key verification failed.' >&2 + exit 255 +fi +cat >/dev/null +printf '%s\n' 'stdin-ok' +`) + writeTestExecutable(t, filepath.Join(binDir, "ssh-keygen"), "#!/bin/sh\nexit 0\n") + t.Setenv("ANANKE_FAKE_SSH_COUNT", countPath) + t.Setenv("HOME", t.TempDir()) + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + sshDir := t.TempDir() + orch = buildOrchestratorWithStubs(t, config.Config{ + SSHPort: 2222, + SSHUser: "atlas", + SSHConfigFile: filepath.Join(sshDir, "config"), + SSHIdentityFile: filepath.Join(sshDir, "id_ed25519"), + SSHNodeHosts: map[string]string{"node-a": "node-a.local"}, + }, nil) + out, err := orch.sshWithInput(ctx, "node-a", "sudo -S true", "pw\n", time.Second) + if err != nil || out != "stdin-ok" { + t.Fatalf("expected known-host retry success, out=%q err=%v", out, err) + } + if raw, err := os.ReadFile(countPath); err != nil || strings.TrimSpace(string(raw)) != "2" { + t.Fatalf("expected exactly two fake ssh attempts, count=%q err=%v", strings.TrimSpace(string(raw)), err) + } +} + +// TestPostStartAutoHealRequestsReconcileAfterImagePullRepair runs one orchestration or CLI step. +// Signature: TestPostStartAutoHealRequestsReconcileAfterImagePullRepair(t *testing.T). +// Why: a successful image-pull credential repair should request a Flux reconcile +// without requiring any real Kubernetes or Flux command. +func TestPostStartAutoHealRequestsReconcileAfterImagePullRepair(t *testing.T) { + ctx := context.Background() + eventsJSON := `{"items":[{"type":"Warning","reason":"FailedPull","message":"unauthorized: authentication required","involvedObject":{"kind":"Pod","namespace":"apps","name":"web"}}]}` + deploymentsJSON := `{"items":[{"metadata":{"name":"vault-sync","labels":{"app":"vault-sync"}}}]}` + sawFluxSourceReconcile := false + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[]}`}, + {match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[]}`}, + {match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"}, + {match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: eventsJSON}, + {match: matchContains("kubectl", "-n", "apps", "get", "deployment", "-o", "json"), out: deploymentsJSON}, + {match: matchContains("kubectl", "-n", "apps", "rollout", "restart", "deployment", "vault-sync"), out: ""}, + {match: matchContains("kubectl", "-n", "apps", "rollout", "status", "deployment/vault-sync"), out: ""}, + { + match: func(name string, args []string) bool { + if !matchContains("kubectl", "-n", "flux-system", "annotate", "gitrepository", "flux-system")(name, args) { + return false + } + sawFluxSourceReconcile = true + return true + }, + }, + {match: matchContains("kubectl", "annotate"), out: ""}, + }) + if err := orch.postStartAutoHeal(ctx); err != nil { + t.Fatalf("expected post-start image-pull repair success, got %v", err) + } + if !sawFluxSourceReconcile { + t.Fatalf("expected post-start auto-heal to request flux source reconcile") + } +} + +// TestStartupQuarantinesRuntimeWedgeBeforeSSHAuthGate runs one orchestration or CLI step. +// Signature: TestStartupQuarantinesRuntimeWedgeBeforeSSHAuthGate(t *testing.T). +// Why: startup should quarantine proven runtime-wedged workers before later SSH +// gates so bad schedulers are removed from the active worker set. +func TestStartupQuarantinesRuntimeWedgeBeforeSSHAuthGate(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + old := time.Now().Add(-10 * time.Minute).Format(time.RFC3339) + now := time.Now().Format(time.RFC3339) + podsJSON := fmt.Sprintf(`{"items":[ +{"metadata":{"namespace":"apps","name":"wedged-a","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"worker-bad"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CreateContainerError"}}}]}}, +{"metadata":{"namespace":"apps","name":"wedged-b","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"worker-bad"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"RunContainerError"}}}]}} +]}`, old, old) + eventsJSON := fmt.Sprintf(`{"items":[ +{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"wedged-a"},"message":"context deadline exceeded while creating container"}, +{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"wedged-b"},"message":"failed to reserve container name"} +]}`, now, now) + cfg := config.Config{ + Workers: []string{"worker-good", "worker-bad"}, + SSHManagedNodes: []string{"worker-good", "worker-bad"}, + SSHNodeHosts: map[string]string{ + "worker-good": "worker-good.local", + "worker-bad": "worker-bad.local", + }, + SSHUser: "atlas", + SSHPort: 22, + Startup: config.Startup{ + APIWaitSeconds: 1, + APIPollSeconds: 1, + RequireTimeSync: false, + RequireNodeInventoryReach: false, + RequireNodeSSHAuth: true, + ReconcileAccessOnBoot: false, + AutoEtcdRestoreOnAPIFailure: false, + StuckPodGraceSeconds: 1, + }, + State: config.State{ + Dir: tmpDir, + ReportsDir: filepath.Join(tmpDir, "reports"), + RunHistoryPath: filepath.Join(tmpDir, "runs.json"), + LockPath: filepath.Join(tmpDir, "ananke.lock"), + IntentPath: filepath.Join(tmpDir, "intent.json"), + }, + } + orch := buildOrchestratorWithStubs(t, cfg, []commandStub{ + {match: matchContains("kubectl", "version", "--request-timeout=5s"), out: "ok"}, + {match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"}, + {match: matchContains("kubectl", "-n", "flux-system", "get", "gitrepository", "flux-system", "-o", "jsonpath={.spec.url}"), out: ""}, + {match: matchContains("kubectl", "-n", "flux-system", "get", "gitrepository", "flux-system", "-o", "jsonpath={.spec.ref.branch}"), out: ""}, + {match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: podsJSON}, + {match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: eventsJSON}, + {match: matchContains("kubectl", "annotate", "node", "worker-bad"), out: ""}, + {match: matchContains("kubectl", "cordon", "worker-bad"), out: ""}, + {match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: ""}, + {match: matchContains("ssh", "worker-good", "echo __ANANKE_SSH_AUTH_OK__"), err: errors.New("permission denied publickey")}, + }) + err := orch.Startup(ctx, StartupOptions{Reason: "runtime quarantine coverage"}) + if err == nil || !strings.Contains(err.Error(), "ssh auth gate failed") { + t.Fatalf("expected controlled ssh auth gate failure after quarantine, got %v", err) + } + if !containsNode(orch.cfg.Startup.IgnoreUnavailableNodes, "worker-bad") { + t.Fatalf("expected runtime-wedged worker to be added to ignore_unavailable_nodes, got %v", orch.cfg.Startup.IgnoreUnavailableNodes) + } +} + +// writeTestExecutable writes a temporary command fixture for tests. +// Signature: writeTestExecutable(t *testing.T, path string, body string). +// Why: fake ssh/ssh-keygen binaries let retry paths run without touching host +// SSH state or invoking the real cluster operator tooling. +func writeTestExecutable(t *testing.T, path string, body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatalf("write test executable %s: %v", path, err) + } +} diff --git a/internal/cluster/orchestrator_quality_gate_more_closeout_test.go b/internal/cluster/orchestrator_quality_gate_more_closeout_test.go new file mode 100644 index 0000000..86793ad --- /dev/null +++ b/internal/cluster/orchestrator_quality_gate_more_closeout_test.go @@ -0,0 +1,480 @@ +package cluster + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "scm.bstein.dev/bstein/ananke/internal/config" +) + +// TestCordonLeaseEnforcementRecoveryCloseoutBranches runs one orchestration or CLI step. +// Signature: TestCordonLeaseEnforcementRecoveryCloseoutBranches(t *testing.T). +// Why: lease enforcement must release recovered Ananke cordons while escalating +// stale or unknown cordons without overriding operator-owned state. +func TestCordonLeaseEnforcementRecoveryCloseoutBranches(t *testing.T) { + now := time.Now().UTC() + future := now.Add(time.Hour).Format(time.RFC3339) + past := now.Add(-time.Hour).Format(time.RFC3339) + oldTaint := now.Add(-2 * time.Hour).Format(time.RFC3339) + nodesJSON := fmt.Sprintf(`{"items":[ +{"metadata":{"name":"","annotations":{}}}, +{"metadata":{"name":"uncordoned","annotations":{%q:"ananke"}},"spec":{"unschedulable":false}}, +{"metadata":{"name":"manual","annotations":{}},"spec":{"unschedulable":true,"taints":[{"key":"node.kubernetes.io/unschedulable","timeAdded":%q}]}}, +{"metadata":{"name":"crypt","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}}, +{"metadata":{"name":"runtime","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}}, +{"metadata":{"name":"proxy","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}}, +{"metadata":{"name":"unknown","annotations":{%q:"ananke",%q:"mystery",%q:%q}},"spec":{"unschedulable":true}} +]}`, + anankeCordonOwnerAnnotation, + oldTaint, + anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonMissingCryptsetup, anankeCordonDeadlineAnnotation, future, + anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonRuntimeWedge, anankeCordonDeadlineAnnotation, future, + anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonKubeletProxy, anankeCordonDeadlineAnnotation, future, + anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, anankeCordonDeadlineAnnotation, past) + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: nodesJSON}, + {match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_PRESENT__"}, + {match: matchContains("kubectl", "get", "pods", "-A"), out: `{"items":[]}`}, + {match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`}, + {match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/proxy/proxy/healthz"), out: "ok"}, + {match: matchContains("kubectl", "uncordon"), out: ""}, + {match: matchContains("kubectl", "annotate", "node"), out: ""}, + }) + released, err := orch.enforceRecoveryCordonLeases(context.Background()) + if err == nil || released != 3 { + t.Fatalf("expected three recovered cordons plus manual-action error, released=%d err=%v", released, err) + } + if !strings.Contains(err.Error(), "manual action required") || !strings.Contains(err.Error(), "unknown") { + t.Fatalf("expected manual action details, got %v", err) + } +} + +// TestCriticalEndpointServiceHealCloseoutBranches runs one orchestration or CLI step. +// Signature: TestCriticalEndpointServiceHealCloseoutBranches(t *testing.T). +// Why: configured service backend repair should distinguish healthy endpoints, +// missing endpoints, backend scale failures, and startup-probe repair success. +func TestCriticalEndpointServiceHealCloseoutBranches(t *testing.T) { + workloadJSON := `{"spec":{"template":{"spec":{"containers":[{"name":"app","livenessProbe":{"httpGet":{"path":"/healthz"}}}]}}}}` + cfg := config.Config{Startup: config.Startup{ + CriticalServiceEndpoints: []string{"bad-entry", "ready/svc", "missing/svc", "zero/svc", "probe/svc", "error/svc"}, + CriticalServiceStartupProbeRepair: true, + }} + orch := buildOrchestratorWithStubs(t, cfg, []commandStub{ + {match: matchContains("kubectl", "-n", "ready", "get", "endpoints", "svc"), out: `{"subsets":[{"addresses":[{"ip":"10.0.0.10"}]}]}`}, + {match: matchContains("kubectl", "-n", "missing", "get", "endpoints", "svc"), err: errors.New(`Error from server (NotFound): endpoints "svc" not found`)}, + {match: matchContains("kubectl", "-n", "zero", "get", "endpoints", "svc"), out: `{"subsets":[]}`}, + {match: matchContains("kubectl", "-n", "probe", "get", "endpoints", "svc"), out: `{"subsets":[]}`}, + {match: matchContains("kubectl", "-n", "error", "get", "endpoints", "svc"), err: errors.New("api down")}, + {match: matchContains("kubectl", "-n", "zero", "scale", "deployment", "svc"), out: ""}, + {match: matchContains("kubectl", "-n", "zero", "rollout", "status", "deployment/svc"), out: ""}, + {match: matchContains("kubectl", "-n", "zero", "scale", "statefulset", "svc"), err: errors.New(`Error from server (NotFound): statefulsets.apps "svc" not found`)}, + {match: matchContains("kubectl", "-n", "missing", "scale"), err: errors.New(`Error from server (NotFound): deployments.apps "svc" not found`)}, + {match: matchContains("kubectl", "-n", "probe", "scale", "deployment", "svc"), err: errors.New("scale denied")}, + {match: matchContains("kubectl", "-n", "probe", "get", "deployment", "svc"), out: workloadJSON}, + {match: matchContains("kubectl", "-n", "probe", "patch", "deployment", "svc"), out: ""}, + {match: matchContains("kubectl", "-n", "probe", "get", "statefulset", "svc"), err: errors.New(`Error from server (NotFound): statefulsets.apps "svc" not found`)}, + }) + healed, err := orch.healUnreadyConfiguredServiceBackends(context.Background()) + if err == nil { + t.Fatalf("expected aggregated critical endpoint errors") + } + joined := strings.Join(healed, ",") + if !strings.Contains(joined, "zero/deployment/svc") || !strings.Contains(joined, "probe/deployment/svc") { + t.Fatalf("expected zero and probe repairs, healed=%v", healed) + } + if !strings.Contains(err.Error(), "bad-entry") || !strings.Contains(err.Error(), "error/svc") { + t.Fatalf("expected invalid and query errors, got %v", err) + } + if _, _, err := parseCriticalServiceEndpoint(" /svc"); err == nil { + t.Fatalf("expected empty namespace endpoint parse error") + } +} + +// TestLonghornAndStaleRWOAdditionalCloseoutBranches runs one orchestration or CLI step. +// Signature: TestLonghornAndStaleRWOAdditionalCloseoutBranches(t *testing.T). +// Why: Longhorn parser errors and stale RWO helper predicates protect storage +// repair from false positives and should be covered without live objects. +func TestLonghornAndStaleRWOAdditionalCloseoutBranches(t *testing.T) { + ctx := context.Background() + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), err: errors.New(`Error from server (NotFound): nodes.longhorn.io not found`)}, + }) + if nodes, err := orch.queryLonghornNodes(ctx); err != nil || len(nodes.Items) != 0 { + t.Fatalf("expected missing Longhorn CRD to be empty, nodes=%v err=%v", nodes, err) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: "{"}, + }) + if _, err := orch.queryLonghornNodes(ctx); err == nil { + t.Fatalf("expected longhorn node decode error") + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: "{"}, + }) + if _, err := orch.queryLonghornManagerDaemonSet(ctx); err == nil { + t.Fatalf("expected daemonset decode error") + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json"), out: "{"}, + }) + if _, err := orch.longhornManagerPodsByNode(ctx, map[string]string{"app": "longhorn"}); err == nil { + t.Fatalf("expected manager pod decode error") + } + if cond := longhornConditionByType(longhornNode{}, "Ready"); cond != nil { + t.Fatalf("expected missing condition to return nil") + } + + checkCalls := 0 + orch = buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"titan-23"}}, nil) + orch.runOverride = func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) { + joined := strings.Join(args, " ") + if name == "ssh" && strings.Contains(joined, "command -v cryptsetup") { + checkCalls++ + if checkCalls == 1 { + return "__ANANKE_CRYPTSETUP_MISSING__", errors.New("exit 41") + } + return "__ANANKE_CRYPTSETUP_PRESENT__", nil + } + if name == "ssh" && strings.Contains(joined, "apt-get update") { + return "installed but no marker", nil + } + return "", nil + } + orch.runSensitiveOverride = orch.runOverride + if err := orch.ensureHostCryptsetup(ctx, "titan-23"); err != nil { + t.Fatalf("expected cryptsetup verify success after install, got %v", err) + } + if checkCalls < 2 { + t.Fatalf("expected install verification check, calls=%d", checkCalls) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "pvc", "-A"), out: "{"}, + }) + if _, err := orch.queryPVCs(ctx); err == nil { + t.Fatalf("expected pvc decode error") + } + pvc := persistentVolumeClaimResource{} + pvc.Metadata.Namespace = "apps" + pvc.Metadata.Name = "data" + pvc.Spec.AccessModes = []string{"ReadWriteOncePod"} + pod := closeoutPendingControllerPod("apps", "app", "node", "ReplicaSet", "ContainerCreating", time.Now().Add(-time.Hour)) + if names := podRWOPVCVolumeNames(pod, map[string]persistentVolumeClaimResource{"apps/data": pvc}); names["data"] != "data" { + t.Fatalf("expected RWO pod volume mapping, got %v", names) + } + initPod := podResource{} + initPod.Status.InitContainerStatuses = []podContainerStatus{{Name: "init", State: podContainerState{Waiting: &podContainerWaitingState{Reason: "PodInitializing"}}}} + if !replacementPodPendingForAttach(initPod) { + t.Fatalf("expected init waiting pod to count as attach-pending") + } + event := eventList{Items: []eventResource{{Type: "Warning", Reason: "SomethingElse", Message: "volume is already used by pod"}}} + event.Items[0].InvolvedObject.Kind = "Pod" + event.Items[0].InvolvedObject.Namespace = "apps" + event.Items[0].InvolvedObject.Name = "app" + pod.Metadata.Namespace = "apps" + pod.Metadata.Name = "app" + if !podHasBlockedAttachEvent(pod, event) { + t.Fatalf("expected blocked attach message to match") + } + if podsShareAnyClaim(podResource{}, pod, map[string]string{"vol": "other"}) { + t.Fatalf("unrelated claims should not match") + } + unsafePod := podResource{} + unsafePod.Status.ContainerStatuses = []podContainerStatus{{Name: "ghost", State: podContainerState{Running: &podContainerRunningState{StartedAt: time.Now()}}}} + if !runningContainersMountAnyVolume(unsafePod, map[string]string{"data": "data"}) { + t.Fatalf("unknown running container should be treated unsafe") + } +} + +// TestTCPDrainAndPostStartAdditionalCloseoutBranches runs one orchestration or CLI step. +// Signature: TestTCPDrainAndPostStartAdditionalCloseoutBranches(t *testing.T). +// Why: near-threshold TCP, drain, and post-start helpers need local coverage for +// TLS probing, error throttling, SSH fallback, and diagnostics summaries. +func TestTCPDrainAndPostStartAdditionalCloseoutBranches(t *testing.T) { + ctx := context.Background() + certServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("ok")) + })) + cert := certServer.TLS.Certificates[0] + certServer.Close() + listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}}) + if err != nil { + t.Fatalf("listen TLS fixture: %v", err) + } + t.Cleanup(func() { _ = listener.Close() }) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + _, _ = conn.Write([]byte("220 ready\r\n")) + _ = conn.SetDeadline(time.Now().Add(2 * time.Second)) + buf := make([]byte, 32) + _, _ = conn.Read(buf) + _, _ = conn.Write([]byte("250 tls-pong\r\n")) + }() + } + }() + host, portText, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatalf("split TLS server address: %v", err) + } + var port int + if _, err := fmt.Sscanf(portText, "%d", &port); err != nil { + t.Fatalf("parse TLS server port: %v", err) + } + ok, detail := buildOrchestratorWithStubs(t, config.Config{}, nil).tcpServiceCheckReady(ctx, config.TCPServiceChecklistCheck{ + Host: host, Port: port, TLS: true, InsecureSkipTLS: true, Send: "PING\r\n", ExpectContains: "tls-pong", + }) + if !ok || detail != "connected" { + t.Fatalf("expected TLS tcp check success, ok=%v detail=%q", ok, detail) + } + orch := buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{TCPServiceChecklist: []config.TCPServiceChecklistCheck{{Host: "127.0.0.1", Port: 1, Namespace: "mail", Service: "smtp"}}}}, []commandStub{ + {match: matchContains("kubectl", "-n", "mail", "scale", "deployment", "smtp"), err: errors.New("scale denied")}, + }) + var zero time.Time + orch.maybeAutoHealTCPServiceBackends(ctx, &zero) + if zero.IsZero() { + t.Fatalf("expected tcp heal attempt timestamp") + } + dry := buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{TCPServiceChecklist: []config.TCPServiceChecklistCheck{{Host: "127.0.0.1", Port: 1}}}}, nil) + dry.runner.DryRun = true + dry.maybeAutoHealTCPServiceBackends(ctx, nil) + + tmpConfig := t.TempDir() + "/ssh_config" + tmpIdentity := t.TempDir() + "/id_ed25519" + orch = buildOrchestratorWithStubs(t, config.Config{SSHConfigFile: tmpConfig, SSHIdentityFile: tmpIdentity}, nil) + if orch.resolveSSHConfigFile() != tmpConfig || orch.resolveSSHIdentityFile() != tmpIdentity { + t.Fatalf("expected explicit ssh config and identity paths") + } + orch.runSensitiveOverride = nil + if out, err := orch.runSensitive(ctx, time.Second, "sh", "-c", "echo noisy; exit 4"); err == nil || out != "noisy" { + t.Fatalf("expected runSensitive output-backed error, out=%q err=%v", out, err) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "cordon", "worker-a"), err: errors.New("cordon warning")}, + {match: matchContains("kubectl", "drain", "worker-a"), err: errors.New("drain denied")}, + {match: matchContains("kubectl", "get", "pods", "-A"), out: "apps api Running ReplicaSet\nkube ds Running DaemonSet\nold done Succeeded Job"}, + }) + if err := orch.drainWorkers(ctx, []string{"worker-a"}); err == nil || !strings.Contains(err.Error(), "apps/api") { + t.Fatalf("expected drain diagnostics in error, got %v", err) + } + if diag := orch.drainNodeDiagnostics(ctx, "worker-a"); !strings.Contains(diag, "apps/api") { + t.Fatalf("expected blocking pod diagnostic, got %q", diag) + } +} + +// TestWorkloadConvergenceAndAutoHealCloseoutBranches runs one orchestration or CLI step. +// Signature: TestWorkloadConvergenceAndAutoHealCloseoutBranches(t *testing.T). +// Why: workload convergence and post-start auto-heal are broad coordinators, so +// focused fixtures keep their skip, warning, and no-op paths from going dark. +func TestWorkloadConvergenceAndAutoHealCloseoutBranches(t *testing.T) { + ctx := context.Background() + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), err: errors.New("api down")}, + }) + if _, _, err := orch.workloadConvergenceReady(ctx); err == nil { + t.Fatalf("expected workload controller query error") + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: "{"}, + }) + if _, _, err := orch.workloadConvergenceReady(ctx); err == nil { + t.Fatalf("expected workload controller decode error") + } + workloads := `{"items":[ +{"kind":"","metadata":{"namespace":"apps","name":"skip"}}, +{"kind":"Deployment","metadata":{"namespace":"apps","name":"api"},"spec":{"replicas":2},"status":{"readyReplicas":1}}, +{"kind":"StatefulSet","metadata":{"namespace":"ignored","name":"db"},"spec":{"replicas":1},"status":{"readyReplicas":0}}, +{"kind":"DaemonSet","metadata":{"namespace":"apps","name":"agent"},"status":{"desiredNumberScheduled":2,"numberReady":1}} +]}` + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{ + IgnoreWorkloadNamespaces: []string{"ignored"}, + IgnoreUnavailableNodes: []string{"down-a"}, + }}, []commandStub{ + {match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: workloads}, + }) + ready, detail, err := orch.workloadConvergenceReady(ctx) + if err != nil || ready || !strings.Contains(detail, "apps/deployment/api") { + t.Fatalf("expected pending deployment detail, ready=%v detail=%q err=%v", ready, detail, err) + } + if _, _, ok := desiredReady(workloadResource{Kind: "Job"}); ok { + t.Fatalf("unsupported workload kind should not report desired/ready") + } + + old := time.Now().Add(-10 * time.Minute) + podA := closeoutPendingControllerPod("apps", "wedge-a", "bad-node", "ReplicaSet", "CreateContainerError", old) + podB := closeoutPendingControllerPod("apps", "wedge-b", "bad-node", "ReplicaSet", "RunContainerError", old) + podA.Spec.Volumes = nil + podB.Spec.Volumes = nil + reasons := map[string]string{"apps/wedge-a": "ContainerRuntimeWedge:bad-node", "apps/wedge-b": "ContainerRuntimeWedge:bad-node"} + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "annotate", "node", "bad-node"), out: ""}, + {match: matchContains("kubectl", "cordon", "bad-node"), out: ""}, + }) + if nodes := orch.quarantineContainerRuntimeWedgeNodes(ctx, podList{Items: []podResource{podA, podB}}, reasons, time.Second, nil, nil, nil); len(nodes) != 1 || nodes[0] != "bad-node" { + t.Fatalf("expected wedged node quarantine, nodes=%v", nodes) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "annotate", "node", "bad-node"), out: ""}, + {match: matchContains("kubectl", "cordon", "bad-node"), err: errors.New("cordon denied")}, + }) + if nodes := orch.quarantineContainerRuntimeWedgeNodes(ctx, podList{Items: []podResource{podA, podB}}, reasons, time.Second, nil, nil, nil); len(nodes) != 0 { + t.Fatalf("cordon failure should not report quarantined nodes, nodes=%v", nodes) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "pods", "-A"), out: "{"}, + }) + if _, err := orch.quarantineContainerRuntimeWedgeNodesFromCluster(ctx); err == nil { + t.Fatalf("expected runtime wedge cluster decode error") + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), err: errors.New("events down")}, + }) + if _, err := orch.containerRuntimeWedgePodReasons(ctx, podList{Items: []podResource{podA}}, time.Second); err == nil { + t.Fatalf("expected runtime wedge event query error") + } + invalidPod := closeoutPendingControllerPod("", "invalid", "node-a", "ReplicaSet", "CreateContainerError", old) + runningPod := closeoutPendingControllerPod("apps", "running", "node-a", "ReplicaSet", "CreateContainerError", old) + runningPod.Status.Phase = "Running" + noOwnerPod := closeoutPendingControllerPod("apps", "no-owner", "node-a", "", "CreateContainerError", old) + noOwnerPod.Metadata.OwnerReferences = nil + newPod := closeoutPendingControllerPod("apps", "new", "node-a", "ReplicaSet", "CreateContainerError", time.Now()) + runtimePod := closeoutPendingControllerPod("apps", "runtime", "node-a", "ReplicaSet", "CreateContainerError", old) + runtimeEvents := eventList{Items: []eventResource{ + {Type: "Normal", Reason: "Failed", Message: "failed to reserve container name"}, + {Type: "Warning", Reason: "Other", Message: "failed to reserve container name"}, + {Type: "Warning", Reason: "Failed", Message: "failed to reserve container name"}, + {Type: "Warning", Reason: "Failed", Message: "failed to reserve container name"}, + {Type: "Warning", Reason: "Failed", EventTime: old.Add(-time.Minute), Message: "failed to reserve container name"}, + {Type: "Warning", Reason: "Failed", Message: "ordinary failure"}, + }} + for i := range runtimeEvents.Items { + runtimeEvents.Items[i].InvolvedObject.Kind = "Pod" + runtimeEvents.Items[i].InvolvedObject.Namespace = "apps" + runtimeEvents.Items[i].InvolvedObject.Name = "runtime" + } + runtimeEvents.Items[2].InvolvedObject.Kind = "Service" + runtimeEvents.Items[3].InvolvedObject.Name = "missing" + runtimeRaw, err := json.Marshal(runtimeEvents) + if err != nil { + t.Fatalf("marshal runtime events: %v", err) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), out: string(runtimeRaw)}, + }) + if reasons, err := orch.containerRuntimeWedgePodReasons(ctx, podList{Items: []podResource{invalidPod, runningPod, noOwnerPod, newPod, runtimePod}}, time.Second); err != nil || len(reasons) != 0 { + t.Fatalf("expected runtime skip-only reasons, got reasons=%v err=%v", reasons, err) + } + ignoredPod := closeoutPendingControllerPod("ignored", "pod", "node-a", "ReplicaSet", "CreateContainerError", old) + rulePod := closeoutPendingControllerPod("apps", "skip-me", "node-a", "ReplicaSet", "CreateContainerError", old) + ignoredNodePod := closeoutPendingControllerPod("apps", "node-skip", "node-skip", "ReplicaSet", "CreateContainerError", old) + pvcPod := closeoutPendingControllerPod("apps", "pvc", "node-a", "ReplicaSet", "CreateContainerError", old) + noReasonPod := closeoutPendingControllerPod("apps", "no-reason", "node-a", "ReplicaSet", "CreateContainerError", old) + reasonMap := map[string]string{ + "ignored/pod": "ContainerRuntimeWedge:node-a", + "apps/skip-me": "ContainerRuntimeWedge:node-a", + "apps/node-skip": "ContainerRuntimeWedge:node-skip", + "apps/no-owner": "ContainerRuntimeWedge:node-a", + "apps/new": "ContainerRuntimeWedge:node-a", + "apps/pvc": "ContainerRuntimeWedge:node-a", + } + if nodes := orch.quarantineContainerRuntimeWedgeNodes(ctx, podList{Items: []podResource{invalidPod, noReasonPod, ignoredPod, rulePod, ignoredNodePod, noOwnerPod, newPod, pvcPod}}, reasonMap, time.Second, map[string]struct{}{"ignored": struct{}{}}, map[string]struct{}{"node-skip": struct{}{}}, []workloadIgnoreRule{{Namespace: "apps", Name: "skip-me"}}); len(nodes) != 0 { + t.Fatalf("skip-only quarantine should return no nodes, got %v", nodes) + } + + stuck := closeoutPendingControllerPod("apps", "crashy", "node-a", "ReplicaSet", "CrashLoopBackOff", old) + stuck.Spec.Volumes = nil + podsJSON := fmt.Sprintf(`{"items":[{"metadata":{"namespace":"apps","name":"crashy","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"node-a"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CrashLoopBackOff"}}}]}}]}`, old.Format(time.RFC3339)) + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{ + {match: matchContains("kubectl", "get", "pods", "-A"), out: podsJSON}, + {match: matchContains("kubectl", "get", "events", "-A"), err: errors.New("events down")}, + {match: matchContains("kubectl", "get", "pvc", "-A"), err: errors.New("pvc down")}, + {match: matchContains("kubectl", "-n", "apps", "delete", "pod", "crashy"), out: ""}, + }) + if err := orch.recycleStuckControllerPods(ctx); err != nil { + t.Fatalf("expected stuck pod recycle despite scan warnings: %v", err) + } + + auto := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: ""}, + {match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[]}`}, + {match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"}, + {match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`}, + }) + if err := auto.postStartAutoHeal(ctx); err != nil { + t.Fatalf("expected post-start auto-heal no-op success, got %v", err) + } +} + +// TestStartupWaitLoopCloseoutBranches runs one orchestration or CLI step. +// Signature: TestStartupWaitLoopCloseoutBranches(t *testing.T). +// Why: startup wait loops should cover immediate success and cancellation paths +// without sleeping or requiring any live Kubernetes objects. +func TestStartupWaitLoopCloseoutBranches(t *testing.T) { + ctx := context.Background() + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: `{"items":[]}`}, + }) + if err := orch.waitForWorkloadConvergence(ctx); err != nil { + t.Fatalf("expected workload convergence success: %v", err) + } + canceled, cancel := context.WithCancel(ctx) + cancel() + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{ + WorkloadConvergenceWaitSeconds: 1, + WorkloadConvergencePollSeconds: 1, + }}, []commandStub{ + {match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: `{"items":[{"kind":"Deployment","metadata":{"namespace":"apps","name":"api"},"spec":{"replicas":1},"status":{"readyReplicas":0}}]}`}, + }) + if err := orch.waitForWorkloadConvergence(canceled); !errors.Is(err, context.Canceled) { + t.Fatalf("expected canceled workload convergence, got %v", err) + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, nil) + if err := orch.waitForCriticalServiceEndpoints(ctx); err != nil { + t.Fatalf("expected critical endpoint success: %v", err) + } + canceled, cancel = context.WithCancel(ctx) + cancel() + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{ + CriticalServiceEndpoints: []string{"apps/api"}, + CriticalServiceEndpointWaitSec: 1, + CriticalServiceEndpointPollSec: 1, + }}, []commandStub{ + {match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), out: `{"subsets":[]}`}, + }) + orch.runner.DryRun = true + if err := orch.waitForCriticalServiceEndpoints(canceled); !errors.Is(err, context.Canceled) { + t.Fatalf("expected canceled critical endpoint wait, got %v", err) + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, nil) + orch.runner.DryRun = true + if err := orch.waitForPostStartProbes(ctx); err != nil { + t.Fatalf("dry-run post-start probes should pass: %v", err) + } + canceled, cancel = context.WithCancel(ctx) + cancel() + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{ + PostStartProbes: []string{"https://example.invalid"}, + PostStartProbeWaitSeconds: 1, + PostStartProbePollSeconds: 1, + }}, []commandStub{ + {match: matchContains("curl", "https://example.invalid"), err: errors.New("curl down")}, + }) + if err := orch.waitForPostStartProbes(canceled); !errors.Is(err, context.Canceled) { + t.Fatalf("expected canceled post-start probe wait, got %v", err) + } +} diff --git a/internal/cluster/orchestrator_quality_gate_recovery_closeout_test.go b/internal/cluster/orchestrator_quality_gate_recovery_closeout_test.go new file mode 100644 index 0000000..1db5a04 --- /dev/null +++ b/internal/cluster/orchestrator_quality_gate_recovery_closeout_test.go @@ -0,0 +1,370 @@ +package cluster + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "scm.bstein.dev/bstein/ananke/internal/config" +) + +// TestCordonLeaseCriticalEndpointAndProbeCloseoutBranches runs one orchestration or CLI step. +// Signature: TestCordonLeaseCriticalEndpointAndProbeCloseoutBranches(t *testing.T). +// Why: recovery cordons and endpoint probe repair carry the cluster-safety +// contract for automatic startup repairs, so their failure paths need coverage. +func TestCordonLeaseCriticalEndpointAndProbeCloseoutBranches(t *testing.T) { + ctx := context.Background() + orch := buildOrchestratorWithStubs(t, config.Config{}, nil) + if err := orch.cordonNodeWithLease(ctx, " ", "", ""); err == nil { + t.Fatalf("expected empty cordon node to fail") + } + if ok := manualActionAlreadyMarked(nil, "action"); ok { + t.Fatalf("nil annotations should not already mark action") + } + if expired, detail := cordonLeaseExpired(map[string]string{}, time.Now()); !expired || detail != "missing deadline" { + t.Fatalf("expected missing deadline to expire, expired=%v detail=%q", expired, detail) + } + if expired, detail := cordonLeaseExpired(map[string]string{anankeCordonDeadlineAnnotation: "bad-time"}, time.Now()); !expired || detail != "bad-time" { + t.Fatalf("expected malformed deadline to expire, expired=%v detail=%q", expired, detail) + } + if since := nodeUnschedulableSince(nodeReadyItem{}); !since.IsZero() { + t.Fatalf("expected zero unschedulable timestamp, got %s", since) + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "annotate", "node", "titan-23"), err: errors.New("annotate denied")}, + }) + if err := orch.cordonNodeWithLease(ctx, "titan-23", "repair", "detail"); err == nil || !strings.Contains(err.Error(), "annotate cordon lease") { + t.Fatalf("expected annotate cordon lease error, got %v", err) + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "cordon", "titan-23"), err: errors.New("cordon denied")}, + {match: matchContains("kubectl", "annotate", "node", "titan-23"), out: ""}, + }) + if err := orch.cordonNodeWithLease(ctx, "titan-23", "repair", "detail"); err == nil || !strings.Contains(err.Error(), "cordon denied") { + t.Fatalf("expected cordon failure, got %v", err) + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "annotate", "node", "gone"), err: errors.New(`Error from server (NotFound): nodes "gone" not found`)}, + {match: matchContains("kubectl", "annotate", "node", "blocked"), err: errors.New("api denied")}, + }) + if err := orch.clearCordonLease(ctx, "gone"); err != nil { + t.Fatalf("not-found clear should be ignored: %v", err) + } + if err := orch.clearCordonLease(ctx, "blocked"); err == nil || !strings.Contains(err.Error(), "clear cordon lease") { + t.Fatalf("expected clear cordon lease error, got %v", err) + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "uncordon", "titan-23"), out: ""}, + {match: matchContains("kubectl", "annotate", "node", "titan-23"), err: errors.New("cleanup denied")}, + }) + if err := orch.uncordonAndClearCordonLease(ctx, "titan-23", "test"); err != nil { + t.Fatalf("lease cleanup warning should not fail uncordon: %v", err) + } + if _, err := orch.recoverLeasedCordon(ctx, "titan-23", map[string]string{anankeCordonReasonAnnotation: "mystery"}); err == nil { + t.Fatalf("expected unknown cordon reason error") + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/titan-23/proxy/healthz"), err: errors.New("proxy down")}, + }) + if recovered, err := orch.recoverLeasedCordon(ctx, "titan-23", map[string]string{anankeCordonReasonAnnotation: cordonReasonKubeletProxy}); err == nil || recovered { + t.Fatalf("expected unhealthy proxy to keep cordon, recovered=%v err=%v", recovered, err) + } + + old := time.Now().Add(-10 * time.Minute) + podJSON := fmt.Sprintf(`{"items":[{"metadata":{"namespace":"apps","name":"stuck","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"titan-23"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CreateContainerError"}}}]}}]}`, old.Format(time.RFC3339)) + eventJSON := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"stuck"},"message":"failed to reserve container name for container app"}]}`, time.Now().Format(time.RFC3339)) + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{ + {match: matchContains("kubectl", "get", "pods", "-A"), out: podJSON}, + {match: matchContains("kubectl", "get", "events", "-A"), out: eventJSON}, + }) + if wedged, err := orch.nodeStillHasRuntimeWedge(ctx, "titan-23"); err != nil || !wedged { + t.Fatalf("expected runtime wedge evidence, wedged=%v err=%v", wedged, err) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "pods", "-A"), out: "{"}, + }) + if _, err := orch.nodeStillHasRuntimeWedge(ctx, "titan-23"); err == nil { + t.Fatalf("expected bad pod json error") + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: "{"}, + }) + if _, err := orch.queryReadyNodes(ctx); err == nil { + t.Fatalf("expected bad node json error") + } + + endpointsJSON := `{"subsets":[{"addresses":[{"ip":"10.0.0.1"},{"ip":"10.0.0.2"}]},{"addresses":[{"ip":""}]}]}` + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{ + {match: matchContains("kubectl", "-n", "grafana", "get", "endpoints", "grafana"), out: endpointsJSON}, + {match: matchContains("kubectl", "-n", "bad", "get", "endpoints", "svc"), out: "{", err: nil}, + }) + if count, err := orch.endpointAddressCount(ctx, "grafana", "grafana"); err != nil || count != 2 { + t.Fatalf("unexpected endpoint count=%d err=%v", count, err) + } + if _, err := orch.endpointAddressCount(ctx, "bad", "svc"); err == nil { + t.Fatalf("expected endpoint decode error") + } + if ready, detail, _, _, err := orch.criticalServiceEndpointsReady(ctx); err != nil || !ready || detail != "no critical service endpoints configured" { + t.Fatalf("expected empty critical endpoint checklist ready, ready=%v detail=%q err=%v", ready, detail, err) + } + + workloadJSON := `{"spec":{"template":{"spec":{"containers":[{"name":"app","livenessProbe":{"httpGet":{"path":"/healthz"}}},{"name":"","livenessProbe":{"tcpSocket":{"port":8080}}},{"name":"has-startup","livenessProbe":{"httpGet":{"path":"/ready"}},"startupProbe":{"httpGet":{"path":"/ready"}}}]}}}}` + var patched bool + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{ + {match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: workloadJSON}, + {match: func(name string, args []string) bool { + if name == "kubectl" && strings.Contains(strings.Join(args, " "), "-n apps patch deployment api") { + patched = true + return true + } + return false + }, out: ""}, + }) + repaired, err := orch.repairWorkloadStartupProbes(ctx, "apps", "deployment", "api") + if err != nil || len(repaired) != 1 || repaired[0] != "apps/deployment/api" || !patched { + t.Fatalf("unexpected startup probe repair repaired=%v patched=%v err=%v", repaired, patched, err) + } + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{ + {match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: "{", err: nil}, + }) + if _, err := orch.repairWorkloadStartupProbes(ctx, "apps", "deployment", "api"); err == nil { + t.Fatalf("expected workload decode error") + } + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{ + {match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: workloadJSON}, + {match: matchContains("kubectl", "-n", "apps", "patch", "deployment", "api"), err: errors.New("patch denied")}, + }) + if _, err := orch.repairWorkloadStartupProbes(ctx, "apps", "deployment", "api"); err == nil { + t.Fatalf("expected workload patch error") + } + if repaired, err := orch.maybeRepairCriticalBackendStartupProbes(ctx, " ", "api"); err != nil || repaired != nil { + t.Fatalf("blank critical backend repair should no-op, repaired=%v err=%v", repaired, err) + } +} + +// TestLonghornRuntimeStaleRWOAndPostStartCloseoutBranches runs one orchestration or CLI step. +// Signature: TestLonghornRuntimeStaleRWOAndPostStartCloseoutBranches(t *testing.T). +// Why: storage handoff, host prerequisite repair, runtime readiness, and +// post-start shell helpers are high-risk automation seams that should stay +// covered by local fixtures. +func TestLonghornRuntimeStaleRWOAndPostStartCloseoutBranches(t *testing.T) { + ctx := context.Background() + old := time.Now().Add(-10 * time.Minute) + pod := closeoutPendingControllerPod("apps", "crypt", "titan-23", "ReplicaSet", "CreateContainerError", old) + events := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"crypt"},"message":"MountVolume.SetUp failed: cryptsetup: no such file or directory"}]}`, time.Now().Format(time.RFC3339)) + cryptsetupCalls := 0 + orch := buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"titan-23"}}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), out: events}, + {match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")}, + {match: matchContains("kubectl", "annotate", "node", "titan-23"), out: ""}, + {match: matchContains("kubectl", "cordon", "titan-23"), out: ""}, + }) + reasons, err := orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{pod}}, time.Second) + if err != nil || reasons["apps/crypt"] != "EncryptedVolumeCryptsetupNodeCordoned:titan-23" { + t.Fatalf("unexpected encrypted mount repair reasons=%v err=%v", reasons, err) + } + orch = buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"titan-23"}}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), out: events}, + {match: func(name string, args []string) bool { + if name == "ssh" && strings.Contains(strings.Join(args, " "), "apt-get update") { + cryptsetupCalls++ + return true + } + return false + }, out: "__ANANKE_CRYPTSETUP_INSTALLED__"}, + {match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_MISSING__", err: errors.New("exit 41")}, + }) + reasons, err = orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{pod, pod}}, time.Second) + if err != nil || reasons["apps/crypt"] != "EncryptedVolumeCryptsetupRepaired:titan-23" || cryptsetupCalls != 1 { + t.Fatalf("unexpected repaired cryptsetup reasons=%v calls=%d err=%v", reasons, cryptsetupCalls, err) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), out: "{"}, + }) + if _, err := orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{pod}}, time.Second); err == nil { + t.Fatalf("expected encrypted mount event decode error") + } + + longhornNodes := "titan-23\tFalse\nready\tTrue\npartial\n" + attachEvents := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc-1: node titan-23 is not ready: longhorn-backend unavailable"}]}`, time.Now().Format(time.RFC3339)) + attachPod := closeoutPendingControllerPod("apps", "attach", "titan-23", "ReplicaSet", "ContainerCreating", old) + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: longhornNodes}, + {match: matchContains("kubectl", "get", "events", "-A"), out: attachEvents}, + }) + blocked, err := orch.longhornAttachBlockedPodReasons(ctx, podList{Items: []podResource{attachPod}}, time.Second) + if err != nil || blocked["apps/attach"] != "LonghornAttachBlockedOnUnreadyNode:titan-23" { + t.Fatalf("unexpected attach-blocked reasons=%v err=%v", blocked, err) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), err: errors.New("api down")}, + }) + if _, err := orch.longhornAttachBlockedPodReasons(ctx, podList{Items: []podResource{attachPod}}, time.Second); err == nil { + t.Fatalf("expected longhorn node query error") + } + + pvcJSON := `{"items":[{"metadata":{"namespace":"apps","name":"data"},"spec":{"accessModes":["ReadWriteOnce"],"volumeName":"pv-data"}}]}` + replacement := closeoutPendingControllerPod("apps", "app-new", "titan-24", "ReplicaSet", "ContainerCreating", old) + oldOwner := closeoutDeletingRWOPod("apps", "app-old", "titan-23", old, false) + blockEvents := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"app-new"},"message":"Multi-Attach error for volume pv-data"}]}`, time.Now().Format(time.RFC3339)) + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), out: blockEvents}, + {match: matchContains("kubectl", "get", "pvc", "-A"), out: pvcJSON}, + }) + decisions, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{Items: []podResource{oldOwner, replacement}}, time.Second) + if err != nil || !decisions["apps/app-old"].ForceDelete || decisions["apps/app-old"].Unsafe { + t.Fatalf("unexpected safe stale RWO decision=%v err=%v", decisions, err) + } + oldUnsafe := closeoutDeletingRWOPod("apps", "app-old", "titan-23", old, true) + decisions, err = orch.staleRWOPVCOwnerDecisions(ctx, podList{Items: []podResource{oldUnsafe, replacement}}, time.Second) + if err != nil || !decisions["apps/app-old"].Unsafe { + t.Fatalf("unexpected unsafe stale RWO decision=%v err=%v", decisions, err) + } + if pvcSingleWriter(persistentVolumeClaimResource{}) { + t.Fatalf("empty pvc should not be single-writer") + } + if replacementPodPendingForAttach(podResource{}) { + t.Fatalf("empty pod should not be pending for attach") + } + if podKey(podResource{}) != "" { + t.Fatalf("empty pod should not have key") + } + + runtimeCfg := config.Config{SSHManagedNodes: []string{"titan-23"}, Startup: config.Startup{NodeRuntimeRestartWaitSeconds: 1}} + orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{ + {match: matchContains("ssh", "systemctl --no-block restart k3s-agent"), out: ""}, + {match: matchContains("kubectl", "wait", "node/titan-23"), out: ""}, + {match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/titan-23/proxy/healthz"), out: "ok"}, + {match: matchContains("ssh", "systemctl is-active k3s-agent"), out: "active"}, + }) + if err := orch.recoverManagedNodeRuntime(ctx, "titan-23", true, "already cordoned"); err != nil { + t.Fatalf("expected runtime recovery success, got %v", err) + } + orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{ + {match: matchContains("kubectl", "wait", "node/titan-23"), out: ""}, + {match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/titan-23/proxy/healthz"), out: "ok"}, + {match: matchContains("ssh", "systemctl is-active k3s-agent"), out: "inactive"}, + }) + if err := orch.waitForManagedNodeRuntimeReady(ctx, "titan-23", 0); err == nil || !strings.Contains(err.Error(), "inactive") { + t.Fatalf("expected inactive k3s-agent readiness failure, got %v", err) + } + if got := orch.nodeRuntimeRebootWait(); got != 420*time.Second { + t.Fatalf("expected default reboot wait, got %s", got) + } + + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{PostStartProbes: []string{" https://ok ", " "}}}, []commandStub{ + {match: matchContains("curl", "https://ok"), out: "403"}, + }) + if ok, detail := orch.postStartProbesReady(ctx); !ok || detail != "all probes successful" { + t.Fatalf("expected post-start probe success, ok=%v detail=%q", ok, detail) + } + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{PostStartProbes: []string{"https://bad"}}}, []commandStub{ + {match: matchContains("curl", "https://bad"), out: "oops"}, + }) + if ok, detail := orch.postStartProbesReady(ctx); ok || !strings.Contains(detail, "parse http status") { + t.Fatalf("expected post-start parse failure, ok=%v detail=%q", ok, detail) + } + if probeStatusAccepted("x", 500) { + t.Fatalf("500 should not be an accepted probe status") + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "--raw", "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health"), err: errors.New("transport closed")}, + }) + if _, err := orch.vaultSealedViaHTTP(ctx); err == nil || !strings.Contains(err.Error(), "vault HTTP health check failed") { + t.Fatalf("expected empty-output vault HTTP error, got %v", err) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "--raw", "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health"), out: "{", err: errors.New("partial")}, + }) + if _, err := orch.vaultSealedViaHTTP(ctx); err == nil || !strings.Contains(err.Error(), "parse") { + t.Fatalf("expected vault HTTP parse error with transport failure, got %v", err) + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, nil) + orch.runner.Kubeconfig = filepath.Join(t.TempDir(), "kubeconfig") + if out, err := orch.runSensitiveWithInput(ctx, time.Second, "secret", "sh", "-c", "cat"); err != nil || out != "secret" { + t.Fatalf("expected runSensitiveWithInput echo, out=%q err=%v", out, err) + } + if out, err := orch.runSensitiveWithInput(ctx, time.Second, "", "sh", "-c", "exit 3"); err == nil || out != "" { + t.Fatalf("expected empty-output command failure, out=%q err=%v", out, err) + } + + binDir := t.TempDir() + fakeSSH := filepath.Join(binDir, "ssh") + if err := os.WriteFile(fakeSSH, []byte("#!/bin/sh\ncase \" $* \" in *\" -J \"*) echo jump failed; exit 7;; esac\ncat >/dev/null\necho direct-ok\n"), 0o755); err != nil { + t.Fatalf("write fake ssh: %v", err) + } + fakeFlux := filepath.Join(binDir, "flux") + if err := os.WriteFile(fakeFlux, []byte("#!/bin/sh\necho flux-ok\n"), 0o755); err != nil { + t.Fatalf("write fake flux: %v", err) + } + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + orch = buildOrchestratorWithStubs(t, config.Config{SSHJumpHost: "bastion", SSHJumpUser: "jump", SSHUser: "atlas", SSHNodeHosts: map[string]string{"titan-23": "10.0.0.23"}}, nil) + out, err := orch.sshWithInput(ctx, "titan-23", "sudo -S true", "pw\n", time.Second) + if err != nil || out != "direct-ok" { + t.Fatalf("expected fake ssh direct fallback success, out=%q err=%v", out, err) + } + var sawFlux bool + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "patch"), out: ""}, + {match: matchContains("kubectl", "annotate"), out: ""}, + {match: func(name string, args []string) bool { + if name == "flux" && strings.Contains(strings.Join(args, " "), "reconcile source git flux-system") { + sawFlux = true + return true + } + return false + }, out: ""}, + }) + if err := orch.resumeFluxAndReconcile(ctx); err != nil || !sawFlux { + t.Fatalf("expected flux reconcile path, sawFlux=%v err=%v", sawFlux, err) + } +} + +// closeoutPendingControllerPod runs one orchestration or CLI step. +// Signature: closeoutPendingControllerPod(ns, name, node, ownerKind, waitReason string, created time.Time) podResource. +// Why: coverage closeout tests need compact pod fixtures for storage and runtime +// classifiers without repeating nested Kubernetes JSON structs. +func closeoutPendingControllerPod(ns, name, node, ownerKind, waitReason string, created time.Time) podResource { + var pod podResource + pod.Metadata.Namespace = ns + pod.Metadata.Name = name + pod.Metadata.CreationTimestamp = created + pod.Metadata.OwnerReferences = []ownerReference{{Kind: ownerKind, Name: "owner"}} + pod.Spec.NodeName = node + pod.Spec.Volumes = []podVolume{{Name: "data", PersistentVolumeClaim: &podPersistentVolumeClaim{ClaimName: "data"}}} + pod.Status.Phase = "Pending" + pod.Status.ContainerStatuses = []podContainerStatus{{Name: "app", State: podContainerState{Waiting: &podContainerWaitingState{Reason: waitReason}}}} + return pod +} + +// closeoutDeletingRWOPod runs one orchestration or CLI step. +// Signature: closeoutDeletingRWOPod(ns, name, node string, deletedAt time.Time, runningWriter bool) podResource. +// Why: stale RWO tests need paired safe and unsafe terminating owner pods with +// the same PVC semantics but different live-writer evidence. +func closeoutDeletingRWOPod(ns, name, node string, deletedAt time.Time, runningWriter bool) podResource { + pod := closeoutPendingControllerPod(ns, name, node, "ReplicaSet", "", deletedAt.Add(-time.Hour)) + pod.Metadata.DeletionTimestamp = &deletedAt + pod.Status.Phase = "Running" + pod.Spec.Containers = []podContainer{ + {Name: "app", VolumeMounts: []podVolumeMount{{Name: "data", MountPath: "/data"}}}, + {Name: "sidecar"}, + } + statusName := "sidecar" + if runningWriter { + statusName = "app" + } + pod.Status.ContainerStatuses = []podContainerStatus{{Name: statusName, State: podContainerState{Running: &podContainerRunningState{StartedAt: deletedAt.Add(-time.Minute)}}}} + return pod +} diff --git a/internal/cluster/orchestrator_quality_gate_storage_closeout_test.go b/internal/cluster/orchestrator_quality_gate_storage_closeout_test.go new file mode 100644 index 0000000..95bb529 --- /dev/null +++ b/internal/cluster/orchestrator_quality_gate_storage_closeout_test.go @@ -0,0 +1,429 @@ +package cluster + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "scm.bstein.dev/bstein/ananke/internal/config" +) + +// TestLonghornRecoveryClassifierCloseoutBranches runs one orchestration or CLI step. +// Signature: TestLonghornRecoveryClassifierCloseoutBranches(t *testing.T). +// Why: Longhorn mount and attach recovery should skip noisy non-matches and +// only act on current, pod-scoped storage evidence. +func TestLonghornRecoveryClassifierCloseoutBranches(t *testing.T) { + ctx := context.Background() + old := time.Now().Add(-10 * time.Minute) + now := time.Now() + invalid := closeoutPendingControllerPod("", "invalid", "node-a", "ReplicaSet", "CreateContainerError", old) + running := closeoutPendingControllerPod("apps", "running", "node-a", "ReplicaSet", "CreateContainerError", old) + running.Status.Phase = "Running" + noOwner := closeoutPendingControllerPod("apps", "no-owner", "node-a", "", "CreateContainerError", old) + noOwner.Metadata.OwnerReferences = nil + newPod := closeoutPendingControllerPod("apps", "new", "node-a", "ReplicaSet", "CreateContainerError", time.Now()) + unmanaged := closeoutPendingControllerPod("apps", "unmanaged", "node-unmanaged", "ReplicaSet", "CreateContainerError", old) + reuseA := closeoutPendingControllerPod("apps", "reuse-a", "node-ok", "ReplicaSet", "CreateContainerError", old) + reuseB := closeoutPendingControllerPod("apps", "reuse-b", "node-ok", "ReplicaSet", "CreateContainerError", old) + fail := closeoutPendingControllerPod("apps", "fail", "node-fail", "ReplicaSet", "CreateContainerError", old) + events := fmt.Sprintf(`{"items":[ +{"type":"Normal","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"}, +{"type":"Warning","reason":"Other","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"}, +{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Service","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"}, +{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"missing"},"message":"cryptsetup: no such file or directory"}, +{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"}, +{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"different mount failure"}, +{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"unmanaged"},"message":"cryptsetup: no such file or directory"}, +{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"}, +{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-b"},"message":"cryptsetup: no such file or directory"}, +{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"fail"},"message":"cryptsetup: no such file or directory"} +]}`, + now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339), + old.Add(-time.Minute).Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339), + now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339)) + orch := buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"node-ok", "node-fail"}}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), out: events}, + {match: matchContains("ssh", "node-ok", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_PRESENT__"}, + {match: matchContains("ssh", "node-fail", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")}, + {match: matchContains("kubectl", "annotate", "node", "node-fail"), err: errors.New("annotate denied")}, + }) + reasons, err := orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{invalid, running, noOwner, newPod, unmanaged, reuseA, reuseB, fail}}, time.Second) + if err != nil { + t.Fatalf("repairEncryptedVolumeMountPrereqs: %v", err) + } + if reasons["apps/reuse-a"] != "EncryptedVolumeCryptsetupRepaired:node-ok" || reasons["apps/reuse-b"] != "EncryptedVolumeCryptsetupRepaired:node-ok" { + t.Fatalf("expected reused cryptsetup repair reasons, got %v", reasons) + } + if reasons["apps/fail"] != "" { + t.Fatalf("cordon failure should not mark failed pod repaired, reasons=%v", reasons) + } + + attachEvents := fmt.Sprintf(`{"items":[ +{"type":"Normal","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"}, +{"type":"Warning","reason":"Other","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"}, +{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Service","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"}, +{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"missing"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"}, +{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"}, +{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"some other attach message"}, +{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node other is not ready: longhorn-backend unavailable"} +]}`, + now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339), + old.Add(-time.Minute).Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339)) + attach := closeoutPendingControllerPod("apps", "attach", "node-bad", "ReplicaSet", "ContainerCreating", old) + notPending := closeoutPendingControllerPod("apps", "not-pending", "node-bad", "ReplicaSet", "ContainerCreating", old) + notPending.Status.Phase = "Running" + readyNode := closeoutPendingControllerPod("apps", "ready-node", "node-ready", "ReplicaSet", "ContainerCreating", old) + noAttachOwner := closeoutPendingControllerPod("apps", "no-owner", "node-bad", "", "ContainerCreating", old) + noAttachOwner.Metadata.OwnerReferences = nil + newAttach := closeoutPendingControllerPod("apps", "new", "node-bad", "ReplicaSet", "ContainerCreating", time.Now()) + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: "node-bad\tFalse\nnode-ready\tTrue\n"}, + {match: matchContains("kubectl", "get", "events", "-A"), out: attachEvents}, + }) + blocked, err := orch.longhornAttachBlockedPodReasons(ctx, podList{Items: []podResource{attach, notPending, readyNode, noAttachOwner, newAttach}}, time.Second) + if err != nil { + t.Fatalf("longhornAttachBlockedPodReasons: %v", err) + } + if blocked["apps/attach"] != "" { + t.Fatalf("all attach events should have been skipped, got %v", blocked) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), err: errors.New(`Error from server (NotFound): nodes.longhorn.io not found`)}, + }) + if unready, err := orch.longhornUnreadyNodes(ctx); err != nil || len(unready) != 0 { + t.Fatalf("expected missing longhorn nodes to be empty, unready=%v err=%v", unready, err) + } +} + +// TestStaleRWOClassifierCloseoutBranches runs one orchestration or CLI step. +// Signature: TestStaleRWOClassifierCloseoutBranches(t *testing.T). +// Why: stale RWO ownership must reject incomplete handoff evidence before it +// considers force-deleting a terminating pod. +func TestStaleRWOClassifierCloseoutBranches(t *testing.T) { + ctx := context.Background() + old := time.Now().Add(-10 * time.Minute) + pvcJSON := `{"items":[{"metadata":{"namespace":"apps","name":"data"},"spec":{"accessModes":["ReadWriteOnce"]}}]}` + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), out: "{"}, + }) + if _, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{}, time.Second); err == nil { + t.Fatalf("expected stale RWO event decode error") + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), out: ""}, + {match: matchContains("kubectl", "get", "pvc", "-A"), err: errors.New("pvc down")}, + }) + if _, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{}, time.Second); err == nil { + t.Fatalf("expected stale RWO pvc query error") + } + oldPod := closeoutDeletingRWOPod("apps", "old", "node-a", old, false) + sameNode := closeoutPendingControllerPod("apps", "same-node", "node-a", "ReplicaSet", "ContainerCreating", old) + noEvent := closeoutPendingControllerPod("apps", "no-event", "node-b", "ReplicaSet", "ContainerCreating", old) + noClaim := closeoutPendingControllerPod("apps", "no-claim", "node-b", "ReplicaSet", "ContainerCreating", old) + noClaim.Spec.Volumes = nil + events := `{"items":[{"type":"Warning","reason":"FailedAttachVolume","involvedObject":{"kind":"Pod","namespace":"apps","name":"no-claim"},"message":"Multi-Attach error"}]}` + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), out: events}, + {match: matchContains("kubectl", "get", "pvc", "-A"), out: pvcJSON}, + }) + decisions, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{Items: []podResource{oldPod, sameNode, noEvent, noClaim}}, time.Second) + if err != nil { + t.Fatalf("staleRWOPVCOwnerDecisions: %v", err) + } + if len(decisions) != 0 { + t.Fatalf("incomplete replacement evidence should not decide, got %v", decisions) + } + emptyPVCs, err := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "pvc", "-A"), out: ""}, + }).queryPVCs(ctx) + if err != nil || len(emptyPVCs) != 0 { + t.Fatalf("expected empty pvc output to return empty map, pvcs=%v err=%v", emptyPVCs, err) + } + if runningContainersMountAnyVolume(podResource{}, nil) { + t.Fatalf("empty volume set should be safe") + } + if podHasBlockedAttachEvent(podResource{}, eventList{Items: []eventResource{{Type: "Normal"}}}) { + t.Fatalf("normal events should not block attach") + } + if strings.TrimSpace(podControllerKey(podResource{})) != "" { + t.Fatalf("empty pod should not have controller key") + } +} + +// TestRecoveryCoordinatorMiscCloseoutBranches runs one orchestration or CLI step. +// Signature: TestRecoveryCoordinatorMiscCloseoutBranches(t *testing.T). +// Why: a handful of coordinator branches only fire on unusual repair failures; +// local stubs keep them covered without mutating the cluster. +func TestRecoveryCoordinatorMiscCloseoutBranches(t *testing.T) { + ctx := context.Background() + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "annotate", "node", "ok"), out: ""}, + {match: matchContains("kubectl", "cordon", "ok"), out: ""}, + }) + if err := orch.cordonNodeWithLease(ctx, "ok", "", strings.Repeat("detail ", 80)); err != nil { + t.Fatalf("expected cordon lease success: %v", err) + } + orch = buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"crypt"}}, []commandStub{ + {match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")}, + }) + if recovered, err := orch.recoverLeasedCordon(ctx, "crypt", map[string]string{anankeCordonReasonAnnotation: cordonReasonMissingCryptsetup}); err == nil || recovered { + t.Fatalf("expected cryptsetup recovery to remain failed, recovered=%v err=%v", recovered, err) + } + old := time.Now().Add(-10 * time.Minute) + wedgedPod := closeoutPendingControllerPod("apps", "wedged", "runtime", "ReplicaSet", "CreateContainerError", old) + wedgedPod.Spec.Volumes = nil + podsJSON := fmt.Sprintf(`{"items":[{"metadata":{"namespace":"apps","name":"wedged","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"runtime"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CreateContainerError"}}}]}}]}`, old.Format(time.RFC3339)) + eventsJSON := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"wedged"},"message":"context deadline exceeded while creating container"}]}`, time.Now().Format(time.RFC3339)) + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{ + {match: matchContains("kubectl", "get", "pods", "-A"), out: podsJSON}, + {match: matchContains("kubectl", "get", "events", "-A"), out: eventsJSON}, + }) + if recovered, err := orch.recoverLeasedCordon(ctx, "runtime", map[string]string{anankeCordonReasonAnnotation: cordonReasonRuntimeWedge}); err == nil || recovered { + t.Fatalf("expected runtime wedge to keep cordon, recovered=%v err=%v", recovered, err) + } + + workloadJSON := `{"spec":{"template":{"spec":{"containers":[{"name":"app","livenessProbe":{"httpGet":{"path":"/healthz"}}}]}}}}` + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{ + CriticalServiceEndpoints: []string{"apps/api"}, + CriticalServiceEndpointWaitSec: 1, + CriticalServiceEndpointPollSec: 1, + CriticalServiceStartupProbeRepair: true, + CriticalServiceStartupProbeThreshold: 2, + }}, []commandStub{ + {match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), out: `{"subsets":[]}`}, + {match: matchContains("kubectl", "-n", "apps", "scale", "deployment", "api"), err: errors.New("scale denied")}, + {match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: workloadJSON}, + {match: matchContains("kubectl", "-n", "apps", "patch", "deployment", "api"), out: ""}, + {match: matchContains("kubectl", "-n", "apps", "get", "statefulset", "api"), err: errors.New(`Error from server (NotFound): statefulsets.apps "api" not found`)}, + }) + canceled, cancel := context.WithCancel(ctx) + cancel() + if err := orch.waitForCriticalServiceEndpoints(canceled); !errors.Is(err, context.Canceled) { + t.Fatalf("expected canceled critical wait after repair attempt, got %v", err) + } + + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{ + CriticalServiceEndpoints: []string{"apps/api"}, + CriticalServiceStartupProbeRepair: true, + AutoRecycleStuckPods: true, + }}, []commandStub{ + {match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), out: `{"subsets":[]}`}, + {match: matchContains("kubectl", "-n", "apps", "scale", "deployment", "api"), err: errors.New("scale denied")}, + {match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: `{"spec":{"template":{"spec":{"containers":[{"name":"app"}]}}}}`}, + {match: matchContains("kubectl", "-n", "apps", "get", "statefulset", "api"), err: errors.New(`Error from server (NotFound): statefulsets.apps "api" not found`)}, + {match: matchContains("kubectl", "get", "pods", "-A"), out: `{"items":[]}`}, + {match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`}, + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: ""}, + {match: matchContains("kubectl", "get", "pvc", "-A"), out: ""}, + }) + if _, err := orch.healUnreadyConfiguredServiceBackends(ctx); err == nil { + t.Fatalf("expected heal error when probe repair finds no containers") + } +} + +// TestRecycleDecisionCloseoutBranches runs one orchestration or CLI step. +// Signature: TestRecycleDecisionCloseoutBranches(t *testing.T). +// Why: pod recycling must skip external blockers and force-delete only the +// stale-owner cases that have enough storage handoff evidence. +func TestRecycleDecisionCloseoutBranches(t *testing.T) { + ctx := context.Background() + old := time.Now().Add(-10 * time.Minute) + dnsPod := closeoutPendingControllerPod("apps", "dns", "node-a", "ReplicaSet", "ImagePullBackOff", old) + credPod := closeoutPendingControllerPod("apps", "cred", "node-a", "ReplicaSet", "ErrImagePull", old) + unsafeOld := closeoutDeletingRWOPod("apps", "unsafe-old", "node-a", old, true) + unsafeNew := closeoutPendingControllerPod("apps", "unsafe-new", "node-b", "ReplicaSet", "ContainerCreating", old) + forceOld := closeoutDeletingRWOPod("apps", "force-old", "node-a", old, false) + forceNew := closeoutPendingControllerPod("apps", "force-new", "node-b", "ReplicaSet", "ContainerCreating", old) + staleDeleting := closeoutDeletingRWOPod("apps", "stale-delete", "node-a", old, false) + staleDeleting.Spec.Volumes = nil + deleteFail := closeoutPendingControllerPod("apps", "delete-fail", "node-a", "ReplicaSet", "CrashLoopBackOff", old) + deleteFail.Spec.Volumes = nil + podsRaw, err := json.Marshal(podList{Items: []podResource{dnsPod, credPod, unsafeOld, unsafeNew, forceOld, forceNew, staleDeleting, deleteFail}}) + if err != nil { + t.Fatalf("marshal pods: %v", err) + } + events := eventList{Items: []eventResource{ + {Type: "Warning", Reason: "Failed", Message: "failed to pull image: lookup registry.local: no such host"}, + {Type: "Warning", Reason: "FailedPull", Message: "unauthorized: authentication required"}, + {Type: "Warning", Reason: "FailedAttachVolume", Message: "Multi-Attach error for unsafe"}, + {Type: "Warning", Reason: "FailedAttachVolume", Message: "Multi-Attach error for force"}, + }} + events.Items[0].InvolvedObject.Kind = "Pod" + events.Items[0].InvolvedObject.Namespace = "apps" + events.Items[0].InvolvedObject.Name = "dns" + events.Items[1].InvolvedObject.Kind = "Pod" + events.Items[1].InvolvedObject.Namespace = "apps" + events.Items[1].InvolvedObject.Name = "cred" + events.Items[2].InvolvedObject.Kind = "Pod" + events.Items[2].InvolvedObject.Namespace = "apps" + events.Items[2].InvolvedObject.Name = "unsafe-new" + events.Items[3].InvolvedObject.Kind = "Pod" + events.Items[3].InvolvedObject.Namespace = "apps" + events.Items[3].InvolvedObject.Name = "force-new" + eventsRaw, err := json.Marshal(events) + if err != nil { + t.Fatalf("marshal events: %v", err) + } + pvcJSON := `{"items":[{"metadata":{"namespace":"apps","name":"data"},"spec":{"accessModes":["ReadWriteOnce"]}}]}` + orch := buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{ + {match: matchContains("kubectl", "get", "pods", "-A"), out: string(podsRaw)}, + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: ""}, + {match: matchContains("kubectl", "get", "events", "-A"), out: string(eventsRaw)}, + {match: matchContains("kubectl", "get", "pvc", "-A"), out: pvcJSON}, + {match: matchContains("kubectl", "-n", "apps", "delete", "pod", "delete-fail"), err: errors.New("delete denied")}, + {match: matchContains("kubectl", "-n", "apps", "delete", "pod"), out: ""}, + }) + if err := orch.recycleStuckControllerPods(ctx); err != nil { + t.Fatalf("recycleStuckControllerPods: %v", err) + } +} + +// TestSmallCoordinatorErrorCloseoutBranches runs one orchestration or CLI step. +// Signature: TestSmallCoordinatorErrorCloseoutBranches(t *testing.T). +// Why: small coordinator branches should stay covered even when they are only +// reached by aggregated repair failures. +func TestSmallCoordinatorErrorCloseoutBranches(t *testing.T) { + ctx := context.Background() + cfg := config.Config{Startup: config.Startup{CriticalServiceEndpoints: []string{"apps/api"}}} + orch := buildOrchestratorWithStubs(t, cfg, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), err: errors.New("longhorn down")}, + {match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: "{"}, + {match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"}, + {match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), err: errors.New("endpoints down")}, + {match: matchContains("kubectl", "get", "events", "-A"), err: errors.New("events down")}, + }) + if err := orch.postStartAutoHeal(ctx); err == nil { + t.Fatalf("expected post-start auto-heal aggregation error") + } + if healthy, err := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/flaky/proxy/healthz"), err: errors.New("i/o timeout")}, + }).kubeletProxyHealthy(ctx, "flaky"); healthy || err == nil { + t.Fatalf("expected transient kubelet proxy retries to fail, healthy=%v err=%v", healthy, err) + } + if isTransientKubeletProxyCheckErr(nil) { + t.Fatalf("nil error is not transient") + } + + if repaired, err := buildOrchestratorWithStubs(t, config.Config{}, nil).maybeRepairCriticalBackendStartupProbes(ctx, "apps", "api"); err != nil || repaired != nil { + t.Fatalf("disabled startup probe repair should no-op, repaired=%v err=%v", repaired, err) + } + orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{ + {match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: "server said no", err: errors.New("get denied")}, + {match: matchContains("kubectl", "-n", "apps", "get", "statefulset", "api"), out: "server said no", err: errors.New("get denied")}, + }) + if _, err := orch.maybeRepairCriticalBackendStartupProbes(ctx, "apps", "api"); err == nil { + t.Fatalf("expected startup probe repair aggregation error") + } + + if err := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "annotate", "node", "manual"), err: errors.New("annotate denied")}, + }).clearCordonLease(ctx, "manual"); err == nil { + t.Fatalf("expected clear cordon lease error") + } + buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "annotate", "node", "manual"), err: errors.New("annotate denied")}, + }).markManualActionRequired(ctx, "manual", "operator please") +} + +// TestFinalSmallCoverageCloseoutBranches runs one orchestration or CLI step. +// Signature: TestFinalSmallCoverageCloseoutBranches(t *testing.T). +// Why: final near-threshold files need concise coverage for rare but important +// recovery error paths. +func TestFinalSmallCoverageCloseoutBranches(t *testing.T) { + ctx := context.Background() + oldDelay := etcdRestorePostStartDelay + etcdRestorePostStartDelay = 0 + t.Cleanup(func() { etcdRestorePostStartDelay = oldDelay }) + hash := strings.Repeat("a", 64) + orch := buildOrchestratorWithStubs(t, config.Config{ + ControlPlanes: []string{"cp1", "cp2"}, + SSHManagedNodes: []string{"cp1", "cp2"}, + }, []commandStub{ + {match: matchContains("ssh", "sudo systemctl cat k3s"), out: "ExecStart=/usr/local/bin/k3s server"}, + {match: matchContains("ssh", "test -s '/snap.db'"), out: "2097152"}, + {match: matchContains("ssh", "etcd-snapshot ls"), out: "/snap.db"}, + {match: matchContains("ssh", "sha256sum '/snap.db'"), out: hash}, + {match: matchContains("ssh", "sudo systemctl stop k3s"), out: ""}, + {match: matchContains("ssh", "server --cluster-reset --cluster-reset-restore-path /snap.db"), out: ""}, + {match: matchContains("ssh", "sudo systemctl start k3s"), out: ""}, + }) + if err := orch.EtcdRestore(ctx, EtcdRestoreOptions{ControlPlane: "cp1", SnapshotPath: "/snap.db"}); err != nil { + t.Fatalf("expected etcd restore success: %v", err) + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: ""}, + }) + if repaired, err := orch.reconcileLonghornKubernetesReadiness(ctx); err != nil || repaired != 0 { + t.Fatalf("empty longhorn nodes should no-op, repaired=%d err=%v", repaired, err) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[{"metadata":{"name":"n"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`}, + {match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{}}}}`}, + }) + if _, err := orch.reconcileLonghornKubernetesReadiness(ctx); err == nil { + t.Fatalf("expected empty longhorn selector error") + } + + orch = buildOrchestratorWithStubs(t, config.Config{ + SSHManagedNodes: []string{"managed"}, + Startup: config.Startup{ + IgnoreUnavailableNodes: []string{"ignored"}, + LonghornCryptsetupExemptNodes: []string{"exempt"}, + RequiredNodeLabels: map[string]map[string]string{"fallback": {"longhorn-host": "true"}}, + }, + }, []commandStub{ + {match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: "ignored\nexempt\nunmanaged\n"}, + {match: matchContains("kubectl", "annotate", "node", "unmanaged"), out: ""}, + {match: matchContains("kubectl", "cordon", "unmanaged"), out: ""}, + }) + guarded, err := orch.ensureLonghornEncryptedHostPrereqs(ctx, []string{"ignored", "exempt", "unmanaged", "managed"}) + if err != nil || strings.Join(guarded, ",") != "ignored,exempt,managed" { + t.Fatalf("unexpected longhorn prereq guard guarded=%v err=%v", guarded, err) + } + + runtimeCfg := config.Config{SSHManagedNodes: []string{"node"}, Startup: config.Startup{NodeRuntimeRestartWaitSeconds: 1}} + orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{ + {match: matchContains("ssh", "systemctl --no-block restart k3s-agent"), out: ""}, + {match: matchContains("kubectl", "wait", "node/node"), err: errors.New("not ready")}, + {match: matchContains("ssh", "systemctl show k3s-agent"), err: errors.New("show failed")}, + }) + if err := orch.recoverManagedNodeRuntime(ctx, "node", true, "test"); err == nil || !strings.Contains(err.Error(), "state check failed") { + t.Fatalf("expected runtime show error, got %v", err) + } + orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{ + {match: matchContains("ssh", "systemctl --no-block restart k3s-agent"), out: ""}, + {match: matchContains("kubectl", "wait", "node/node"), err: errors.New("not ready")}, + {match: matchContains("ssh", "systemctl show k3s-agent"), out: "ActiveState=deactivating"}, + }) + if err := orch.recoverManagedNodeRuntime(ctx, "node", true, "test"); err == nil || !strings.Contains(err.Error(), "host_repair_allow_reboot is false") { + t.Fatalf("expected reboot disallowed error, got %v", err) + } + + orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A"), out: "{"}, + }) + if _, err := orch.imagePullCredentialBlockerReasons(ctx); err == nil { + t.Fatalf("expected image-pull credential event decode error") + } + dry := buildOrchestratorWithStubs(t, config.Config{}, nil) + dry.runner.DryRun = true + if repaired, err := dry.healImagePullCredentialSync(ctx); err != nil || repaired != nil { + t.Fatalf("dry-run image credential sync should no-op, repaired=%v err=%v", repaired, err) + } + orch = buildOrchestratorWithStubs(t, config.Config{}, nil) + orch.runSensitiveOverride = nil + orch.runner.Kubeconfig = t.TempDir() + "/kubeconfig" + if out, err := orch.runSensitive(ctx, time.Second, "sh", "-c", "echo ok"); err != nil || out != "ok" { + t.Fatalf("expected runSensitive kubeconfig success, out=%q err=%v", out, err) + } + pod := closeoutPendingControllerPod("apps", "app", "node", "ReplicaSet", "ContainerCreating", time.Now()) + pod.Spec.Volumes = []podVolume{{Name: "empty", PersistentVolumeClaim: &podPersistentVolumeClaim{ClaimName: " "}}} + if got := podRWOPVCVolumeNames(pod, nil); len(got) != 0 { + t.Fatalf("blank claim should not map, got %v", got) + } +} diff --git a/internal/config/apply_defaults.go b/internal/config/apply_defaults.go index 55c5b9d..c5b32dc 100644 --- a/internal/config/apply_defaults.go +++ b/internal/config/apply_defaults.go @@ -43,6 +43,10 @@ func (c *Config) applyDefaults() { }, } } + c.Startup.RequiredNodeLabelsMode = strings.ToLower(strings.TrimSpace(c.Startup.RequiredNodeLabelsMode)) + if c.Startup.RequiredNodeLabelsMode == "" { + c.Startup.RequiredNodeLabelsMode = "enforce" + } if c.Startup.TimeSyncWaitSeconds <= 0 { c.Startup.TimeSyncWaitSeconds = 240 } diff --git a/internal/config/defaults.go b/internal/config/defaults.go index a4e75ca..3078567 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -71,6 +71,7 @@ func defaults() Config { "ananke.bstein.dev/harbor-bootstrap": "true", }, }, + RequiredNodeLabelsMode: "enforce", RequirePostStartProbes: true, PostStartProbeWaitSeconds: 240, PostStartProbePollSeconds: 5, diff --git a/internal/config/load.go b/internal/config/load.go index a960a85..ece8835 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -7,10 +7,35 @@ import ( "gopkg.in/yaml.v3" ) +type inventoryFragment struct { + SSHUser *string `yaml:"ssh_user"` + SSHPort *int `yaml:"ssh_port"` + SSHConfigFile *string `yaml:"ssh_config_file"` + SSHIdentityFile *string `yaml:"ssh_identity_file"` + SSHNodeHosts *map[string]string `yaml:"ssh_node_hosts"` + SSHNodeUsers *map[string]string `yaml:"ssh_node_users"` + SSHManagedNodes *[]string `yaml:"ssh_managed_nodes"` + ControlPlanes *[]string `yaml:"control_planes"` + Workers *[]string `yaml:"workers"` + Startup *struct { + RequiredNodeLabels *map[string]map[string]string `yaml:"required_node_labels"` + RequiredNodeLabelsMode *string `yaml:"required_node_labels_mode"` + IgnoreUnavailableNodes *[]string `yaml:"ignore_unavailable_nodes"` + } `yaml:"startup"` +} + // Load runs one orchestration or CLI step. // Signature: Load(path string) (Config, error). // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve. func Load(path string) (Config, error) { + return LoadWithFragments(path) +} + +// LoadWithFragments runs one orchestration or CLI step. +// Signature: LoadWithFragments(path string, fragments ...string) (Config, error). +// Why: Terraform can generate a narrow inventory fragment while Ananke keeps +// loading the ordinary host runtime config for imperative recovery behavior. +func LoadWithFragments(path string, fragments ...string) (Config, error) { cfg := defaults() b, err := os.ReadFile(path) @@ -24,6 +49,15 @@ func Load(path string) (Config, error) { if err := yaml.Unmarshal(b, &cfg); err != nil { return Config{}, fmt.Errorf("decode config %s: %w", path, err) } + for _, fragment := range fragments { + fb, err := os.ReadFile(fragment) + if err != nil { + return Config{}, fmt.Errorf("read config fragment %s: %w", fragment, err) + } + if err := applyInventoryFragment(fb, &cfg); err != nil { + return Config{}, fmt.Errorf("decode config fragment %s: %w", fragment, err) + } + } cfg.applyDefaults() if !startupProbeRepairConfigured { @@ -35,6 +69,57 @@ func Load(path string) (Config, error) { return cfg, nil } +// applyInventoryFragment runs one orchestration or CLI step. +// Signature: applyInventoryFragment(raw []byte, cfg *Config) error. +// Why: restricts generated Terraform fragments to declarative inventory fields +// so they cannot accidentally take over Ananke's recovery behavior. +func applyInventoryFragment(raw []byte, cfg *Config) error { + var fragment inventoryFragment + if err := yaml.Unmarshal(raw, &fragment); err != nil { + return err + } + if fragment.SSHUser != nil { + cfg.SSHUser = *fragment.SSHUser + } + if fragment.SSHPort != nil { + cfg.SSHPort = *fragment.SSHPort + } + if fragment.SSHConfigFile != nil { + cfg.SSHConfigFile = *fragment.SSHConfigFile + } + if fragment.SSHIdentityFile != nil { + cfg.SSHIdentityFile = *fragment.SSHIdentityFile + } + if fragment.SSHNodeHosts != nil { + cfg.SSHNodeHosts = *fragment.SSHNodeHosts + } + if fragment.SSHNodeUsers != nil { + cfg.SSHNodeUsers = *fragment.SSHNodeUsers + } + if fragment.SSHManagedNodes != nil { + cfg.SSHManagedNodes = *fragment.SSHManagedNodes + } + if fragment.ControlPlanes != nil { + cfg.ControlPlanes = *fragment.ControlPlanes + } + if fragment.Workers != nil { + cfg.Workers = *fragment.Workers + } + if fragment.Startup == nil { + return nil + } + if fragment.Startup.RequiredNodeLabels != nil { + cfg.Startup.RequiredNodeLabels = *fragment.Startup.RequiredNodeLabels + } + if fragment.Startup.RequiredNodeLabelsMode != nil { + cfg.Startup.RequiredNodeLabelsMode = *fragment.Startup.RequiredNodeLabelsMode + } + if fragment.Startup.IgnoreUnavailableNodes != nil { + cfg.Startup.IgnoreUnavailableNodes = *fragment.Startup.IgnoreUnavailableNodes + } + return nil +} + // yamlPathExists runs one orchestration or CLI step. // Signature: yamlPathExists(raw []byte, path ...string) (bool, error). // Why: selected boolean defaults need to distinguish an omitted YAML key from an diff --git a/internal/config/load_additional_test.go b/internal/config/load_additional_test.go index dd85311..162bb9a 100644 --- a/internal/config/load_additional_test.go +++ b/internal/config/load_additional_test.go @@ -89,3 +89,186 @@ ups: t.Fatalf("expected explicit gitea-api check, got %q", cfg.Startup.ServiceChecklist[0].Name) } } + +// TestLoadWithFragmentsReplacesGeneratedInventory runs one orchestration or CLI step. +// Signature: TestLoadWithFragmentsReplacesGeneratedInventory(t *testing.T). +// Why: Terraform-generated inventory fragments must be authoritative for the +// declarative bootstrap fields they own, without merging stale host config. +func TestLoadWithFragmentsReplacesGeneratedInventory(t *testing.T) { + tmp := t.TempDir() + basePath := filepath.Join(tmp, "ananke.yaml") + fragmentPath := filepath.Join(tmp, "ananke.inventory.yaml") + base := ` +control_planes: [old-cp] +workers: [old-worker] +ssh_user: atlas +ssh_port: 2277 +ssh_node_hosts: + old-cp: 192.0.2.10 +ssh_managed_nodes: [old-cp, old-worker] +expected_flux_branch: main +expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git +iac_repo_path: /opt/titan-iac +startup: + required_node_labels: + old-worker: + old-label: "true" + ignore_unavailable_nodes: [old-worker] +ups: + enabled: false +` + fragment := ` +ssh_user: atlas +ssh_port: 2277 +ssh_config_file: /home/atlas/.ssh/config +ssh_identity_file: /home/atlas/.ssh/id_ed25519 +ssh_node_hosts: + titan-0a: 192.168.22.11 + titan-04: 192.168.22.30 +ssh_node_users: + titan-04: workeradmin +ssh_managed_nodes: [titan-0a, titan-04] +control_planes: [titan-0a] +workers: [titan-04] +startup: + required_node_labels_mode: validate + required_node_labels: + titan-04: + node-role.kubernetes.io/worker: "true" + ignore_unavailable_nodes: [] +` + if err := os.WriteFile(basePath, []byte(base), 0o644); err != nil { + t.Fatalf("write base config: %v", err) + } + if err := os.WriteFile(fragmentPath, []byte(fragment), 0o644); err != nil { + t.Fatalf("write fragment config: %v", err) + } + + cfg, err := LoadWithFragments(basePath, fragmentPath) + if err != nil { + t.Fatalf("load config with fragment: %v", err) + } + if len(cfg.ControlPlanes) != 1 || cfg.ControlPlanes[0] != "titan-0a" { + t.Fatalf("expected fragment control planes, got %v", cfg.ControlPlanes) + } + if _, ok := cfg.SSHNodeHosts["old-cp"]; ok { + t.Fatalf("expected fragment ssh_node_hosts to replace base map, got %#v", cfg.SSHNodeHosts) + } + if cfg.SSHNodeUsers["titan-04"] != "workeradmin" { + t.Fatalf("expected fragment ssh_node_users override, got %#v", cfg.SSHNodeUsers) + } + if _, ok := cfg.Startup.RequiredNodeLabels["old-worker"]; ok { + t.Fatalf("expected fragment required_node_labels to replace base map, got %#v", cfg.Startup.RequiredNodeLabels) + } + if cfg.Startup.RequiredNodeLabelsMode != "validate" { + t.Fatalf("expected fragment label mode validate, got %q", cfg.Startup.RequiredNodeLabelsMode) + } + if len(cfg.Startup.IgnoreUnavailableNodes) != 0 { + t.Fatalf("expected fragment ignored unavailable nodes to replace base slice, got %v", cfg.Startup.IgnoreUnavailableNodes) + } +} + +// TestLoadWithFragmentsAllowsTopLevelInventoryOnly runs one orchestration or CLI step. +// Signature: TestLoadWithFragmentsAllowsTopLevelInventoryOnly(t *testing.T). +// Why: generated inventory fragments may omit startup labels during staged +// rollout while still replacing host/node inventory. +func TestLoadWithFragmentsAllowsTopLevelInventoryOnly(t *testing.T) { + tmp := t.TempDir() + basePath := filepath.Join(tmp, "ananke.yaml") + fragmentPath := filepath.Join(tmp, "ananke.inventory.yaml") + base := ` +control_planes: [old-cp] +workers: [old-worker] +ssh_user: atlas +ssh_port: 2277 +ssh_node_hosts: + old-cp: 192.0.2.10 +ssh_managed_nodes: [old-cp, old-worker] +expected_flux_branch: main +expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git +iac_repo_path: /opt/titan-iac +startup: + required_node_labels_mode: enforce + required_node_labels: + old-worker: + old-label: "true" +ups: + enabled: false +` + fragment := ` +ssh_node_hosts: + titan-0a: 192.168.22.11 +ssh_managed_nodes: [titan-0a] +control_planes: [titan-0a] +workers: [] +` + if err := os.WriteFile(basePath, []byte(base), 0o644); err != nil { + t.Fatalf("write base config: %v", err) + } + if err := os.WriteFile(fragmentPath, []byte(fragment), 0o644); err != nil { + t.Fatalf("write fragment config: %v", err) + } + + cfg, err := LoadWithFragments(basePath, fragmentPath) + if err != nil { + t.Fatalf("load config with top-level fragment: %v", err) + } + if len(cfg.ControlPlanes) != 1 || cfg.ControlPlanes[0] != "titan-0a" { + t.Fatalf("expected top-level fragment control plane, got %v", cfg.ControlPlanes) + } + if _, ok := cfg.Startup.RequiredNodeLabels["old-worker"]; !ok { + t.Fatalf("expected startup labels to remain when fragment omits startup, got %#v", cfg.Startup.RequiredNodeLabels) + } + if cfg.Startup.RequiredNodeLabelsMode != "enforce" { + t.Fatalf("expected base label mode to remain enforce, got %q", cfg.Startup.RequiredNodeLabelsMode) + } +} + +// TestLoadWithFragmentsFailsOnMissingFragment runs one orchestration or CLI step. +// Signature: TestLoadWithFragmentsFailsOnMissingFragment(t *testing.T). +// Why: installer and systemd mistakes should fail closed instead of silently +// running with stale embedded inventory. +func TestLoadWithFragmentsFailsOnMissingFragment(t *testing.T) { + cfgPath := filepath.Join(t.TempDir(), "ananke.yaml") + raw := ` +control_planes: [titan-0a] +expected_flux_branch: main +expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git +iac_repo_path: /opt/titan-iac +ups: + enabled: false +` + if err := os.WriteFile(cfgPath, []byte(raw), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + if _, err := LoadWithFragments(cfgPath, filepath.Join(t.TempDir(), "missing.yaml")); err == nil { + t.Fatalf("expected missing fragment error") + } +} + +// TestLoadWithFragmentsFailsOnBadFragmentYAML runs one orchestration or CLI step. +// Signature: TestLoadWithFragmentsFailsOnBadFragmentYAML(t *testing.T). +// Why: malformed generated inventory must fail closed instead of running with +// partially decoded declarative state. +func TestLoadWithFragmentsFailsOnBadFragmentYAML(t *testing.T) { + tmp := t.TempDir() + cfgPath := filepath.Join(tmp, "ananke.yaml") + fragmentPath := filepath.Join(tmp, "ananke.inventory.yaml") + raw := ` +control_planes: [titan-0a] +expected_flux_branch: main +expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git +iac_repo_path: /opt/titan-iac +ups: + enabled: false +` + if err := os.WriteFile(cfgPath, []byte(raw), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + if err := os.WriteFile(fragmentPath, []byte(":\n- invalid"), 0o644); err != nil { + t.Fatalf("write bad fragment: %v", err) + } + if _, err := LoadWithFragments(cfgPath, fragmentPath); err == nil { + t.Fatalf("expected bad fragment decode error") + } +} diff --git a/internal/config/startup_service_catalog_closeout_test.go b/internal/config/startup_service_catalog_closeout_test.go new file mode 100644 index 0000000..0d94037 --- /dev/null +++ b/internal/config/startup_service_catalog_closeout_test.go @@ -0,0 +1,25 @@ +package config + +import "testing" + +// TestMergeStartupCatalogDefaultsSkipsBlankNames runs one orchestration or CLI step. +// Signature: TestMergeStartupCatalogDefaultsSkipsBlankNames(t *testing.T). +// Why: blank generated or hand-written checklist names should not poison +// default-catalog merging or create duplicate unnamed runtime checks. +func TestMergeStartupCatalogDefaultsSkipsBlankNames(t *testing.T) { + httpMerged := mergeServiceChecklistDefaults( + []ServiceChecklistCheck{{Name: " "}, {Name: "custom"}}, + []ServiceChecklistCheck{{Name: " "}, {Name: "baseline"}}, + ) + if len(httpMerged) != 3 || httpMerged[0].Name != " " || httpMerged[1].Name != "baseline" || httpMerged[2].Name != "custom" { + t.Fatalf("unexpected HTTP checklist merge: %#v", httpMerged) + } + + tcpMerged := mergeTCPServiceChecklistDefaults( + []TCPServiceChecklistCheck{{Name: " "}, {Name: "smtp-custom"}}, + []TCPServiceChecklistCheck{{Name: " "}, {Name: "smtp-baseline"}}, + ) + if len(tcpMerged) != 3 || tcpMerged[0].Name != " " || tcpMerged[1].Name != "smtp-baseline" || tcpMerged[2].Name != "smtp-custom" { + t.Fatalf("unexpected TCP checklist merge: %#v", tcpMerged) + } +} diff --git a/internal/config/types.go b/internal/config/types.go index bd2b502..eb39467 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -38,6 +38,7 @@ type Startup struct { NodeInventoryReachRequiredNodes []string `yaml:"node_inventory_reachability_required_nodes"` LonghornCryptsetupExemptNodes []string `yaml:"longhorn_cryptsetup_exempt_nodes"` RequiredNodeLabels map[string]map[string]string `yaml:"required_node_labels"` + RequiredNodeLabelsMode string `yaml:"required_node_labels_mode"` RequireTimeSync bool `yaml:"require_time_sync"` TimeSyncWaitSeconds int `yaml:"time_sync_wait_seconds"` TimeSyncPollSeconds int `yaml:"time_sync_poll_seconds"` diff --git a/internal/config/validate.go b/internal/config/validate.go index 4711c70..cd452a4 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -87,6 +87,9 @@ func (c Config) Validate() error { } } } + if c.Startup.RequiredNodeLabelsMode != "enforce" && c.Startup.RequiredNodeLabelsMode != "validate" { + return fmt.Errorf("config.startup.required_node_labels_mode must be enforce or validate") + } if c.Startup.TimeSyncWaitSeconds <= 0 { return fmt.Errorf("config.startup.time_sync_wait_seconds must be > 0") } diff --git a/internal/config/validate_matrix_test.go b/internal/config/validate_matrix_test.go index beb5a64..073c5ab 100644 --- a/internal/config/validate_matrix_test.go +++ b/internal/config/validate_matrix_test.go @@ -32,6 +32,7 @@ func TestValidateRejectsInvalidFieldsMatrix(t *testing.T) { {"bad_node_inventory_poll", func(c *Config) { c.Startup.NodeInventoryReachPollSeconds = 0 }}, {"bad_empty_node_inventory_required_node", func(c *Config) { c.Startup.NodeInventoryReachRequiredNodes = []string{"titan-0a", ""} }}, {"bad_empty_longhorn_cryptsetup_exempt_node", func(c *Config) { c.Startup.LonghornCryptsetupExemptNodes = []string{"titan-23", ""} }}, + {"bad_required_node_labels_mode", func(c *Config) { c.Startup.RequiredNodeLabelsMode = "repair" }}, {"bad_time_sync_wait", func(c *Config) { c.Startup.TimeSyncWaitSeconds = 0 }}, {"bad_time_sync_poll", func(c *Config) { c.Startup.TimeSyncPollSeconds = 0 }}, {"bad_time_sync_quorum", func(c *Config) { c.Startup.TimeSyncMode = "quorum"; c.Startup.TimeSyncQuorum = 0 }}, @@ -161,6 +162,9 @@ func TestApplyDefaultsPopulatesZeroConfig(t *testing.T) { if cfg.Startup.TimeSyncMode == "" || cfg.Startup.EtcdRestoreControlPlane == "" || cfg.Startup.VaultUnsealKeyFile == "" { t.Fatalf("expected startup defaults to be set") } + if cfg.Startup.RequiredNodeLabelsMode != "enforce" { + t.Fatalf("expected required node labels mode default enforce, got %q", cfg.Startup.RequiredNodeLabelsMode) + } if cfg.Startup.PostStartAutoHealSeconds <= 0 || cfg.Startup.DeadNodeCleanupGraceSeconds <= 0 { t.Fatalf("expected post-start auto-heal defaults to be set") } diff --git a/scripts/install-config-migration.sh b/scripts/install-config-migration.sh index 29ce0af..ff4ee6b 100755 --- a/scripts/install-config-migration.sh +++ b/scripts/install-config-migration.sh @@ -338,3 +338,15 @@ sanitize_migrated_ananke_config() { chmod 0640 "${cfg}" || true fi } + +install_inventory_fragment() { + if [[ -z "${INVENTORY_FRAGMENT:-}" ]]; then + return 0 + fi + if [[ ! -f "${INVENTORY_FRAGMENT}" ]]; then + echo "[install] ANANKE_INVENTORY_FRAGMENT does not exist: ${INVENTORY_FRAGMENT}" >&2 + exit 1 + fi + install -m 0640 "${INVENTORY_FRAGMENT}" "${CONF_DIR}/ananke.inventory.yaml" + echo "[install] installed Terraform-generated inventory fragment to ${CONF_DIR}/ananke.inventory.yaml" +} diff --git a/scripts/install.sh b/scripts/install.sh index 5a9b96d..e97098e 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -22,6 +22,7 @@ NUT_PRODUCT_ID="${ANANKE_NUT_PRODUCT_ID:-0601}" NUT_MONITOR_USER="${ANANKE_NUT_MONITOR_USER:-monuser}" NUT_MONITOR_PASSWORD="${ANANKE_NUT_MONITOR_PASSWORD:-anankeupsmon}" FORCE_CONFIG_TEMPLATE="${ANANKE_FORCE_CONFIG_TEMPLATE:-}" +INVENTORY_FRAGMENT="${ANANKE_INVENTORY_FRAGMENT:-}" ENFORCE_QUALITY_GATE="${ANANKE_ENFORCE_QUALITY_GATE:-1}" while [[ $# -gt 0 ]]; do @@ -103,6 +104,7 @@ else fi migrate_ananke_config sanitize_migrated_ananke_config +install_inventory_fragment ensure_ananke_ssh_identity ensure_ananke_kubeconfig diff --git a/testing/hygiene/in_tree_test_allowlist.txt b/testing/hygiene/in_tree_test_allowlist.txt index 924b6c4..06aef93 100644 --- a/testing/hygiene/in_tree_test_allowlist.txt +++ b/testing/hygiene/in_tree_test_allowlist.txt @@ -20,6 +20,11 @@ internal/cluster/orchestrator_autorepair_proxy_test.go internal/cluster/orchestrator_autorepair_service_test.go internal/cluster/orchestrator_critical_endpoint_additional_test.go internal/cluster/orchestrator_cordon_lease_test.go +internal/cluster/orchestrator_quality_gate_more_closeout_test.go +internal/cluster/orchestrator_quality_gate_storage_closeout_test.go +internal/cluster/orchestrator_quality_gate_closeout_test.go +internal/cluster/orchestrator_quality_gate_final_closeout_test.go +internal/cluster/orchestrator_quality_gate_recovery_closeout_test.go internal/cluster/orchestrator_test.go internal/cluster/orchestrator_hardening_test.go internal/cluster/orchestrator_image_pull_credentials_test.go @@ -28,6 +33,7 @@ internal/cluster/orchestrator_workload_recovery_test.go internal/cluster/orchestrator_vault_test.go internal/config/config_test.go internal/config/load_additional_test.go +internal/config/startup_service_catalog_closeout_test.go internal/config/validate_matrix_test.go internal/service/daemon_additional_test.go internal/service/daemon_coverage_closeout_test.go diff --git a/testing/orchestrator/hooks_ingress_service_matrix_test.go b/testing/orchestrator/hooks_ingress_service_matrix_test.go index a05ae34..35bf094 100644 --- a/testing/orchestrator/hooks_ingress_service_matrix_test.go +++ b/testing/orchestrator/hooks_ingress_service_matrix_test.go @@ -95,6 +95,68 @@ func TestHookIngressServiceMatrix(t *testing.T) { } }) + t.Run("required-node-labels-validate-mode-reports-drift-without-applying", func(t *testing.T) { + cfg := lifecycleConfig(t) + cfg.Startup.RequiredNodeLabelsMode = "validate" + cfg.Startup.RequiredNodeLabels = map[string]map[string]string{ + "titan-23": { + "topology.kubernetes.io/zone": "lab-a", + }, + } + labelCalled := false + run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { + command := name + " " + strings.Join(args, " ") + switch { + case name == "kubectl" && strings.Contains(command, "label node titan-23 --overwrite"): + labelCalled = true + return "", nil + case name == "kubectl" && strings.Contains(command, "get node titan-23 -o json"): + return `{"metadata":{"labels":{"topology.kubernetes.io/zone":"lab-a"}}}`, nil + default: + return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) + } + } + orch, _ := newHookOrchestrator(t, cfg, run, run) + if err := orch.TestHookEnsureRequiredNodeLabels(context.Background()); err != nil { + t.Fatalf("expected validate mode success, got %v", err) + } + if labelCalled { + t.Fatalf("validate mode must not apply node labels") + } + + runDrift := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { + command := name + " " + strings.Join(args, " ") + if name == "kubectl" && strings.Contains(command, "label node titan-23 --overwrite") { + t.Fatalf("validate mode drift must not apply node labels, got %q", command) + } + if name == "kubectl" && strings.Contains(command, "get node titan-23 -o json") { + return `{"metadata":{"labels":{"topology.kubernetes.io/zone":"lab-b"}}}`, nil + } + return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) + } + orchDrift, _ := newHookOrchestrator(t, cfg, runDrift, runDrift) + err := orchDrift.TestHookEnsureRequiredNodeLabels(context.Background()) + if err == nil || !strings.Contains(err.Error(), "required node label drift") { + t.Fatalf("expected validate mode drift error, got %v", err) + } + + cfg.Startup.NodeInventoryReachRequiredNodes = []string{"titan-db"} + runAbsent := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { + command := name + " " + strings.Join(args, " ") + if name == "kubectl" && strings.Contains(command, "label node titan-23 --overwrite") { + t.Fatalf("validate mode absent node must not apply node labels, got %q", command) + } + if name == "kubectl" && strings.Contains(command, "get node titan-23 -o json") { + return "", errors.New("Error from server (NotFound): nodes \"titan-23\" not found") + } + return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) + } + orchAbsent, _ := newHookOrchestrator(t, cfg, runAbsent, runAbsent) + if err := orchAbsent.TestHookEnsureRequiredNodeLabels(context.Background()); err != nil { + t.Fatalf("expected validate mode absent non-core node to be skipped, got %v", err) + } + }) + t.Run("ingress-discovery-checklist-and-heal", func(t *testing.T) { tlsServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK)