package cluster import ( "bufio" "bytes" "context" "encoding/base64" "encoding/json" "errors" "log" "net" "strconv" "strings" "testing" "time" "scm.bstein.dev/bstein/ananke/internal/config" ) // TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret runs one orchestration or CLI step. // Signature: TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret(t *testing.T). // Why: host repair must recover from password-required sudo while keeping the // sudo password out of command arguments and logs. func TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret(t *testing.T) { secret := "correct horse battery staple" var logs bytes.Buffer orch := buildOrchestratorWithStubs(t, config.Config{ Startup: config.Startup{ HostSudoSecretNamespace: "ops", HostSudoSecretNameTemplate: "sudo-{node}", HostSudoSecretPasswordKey: "password", }, }, nil) orch.log = log.New(&logs, "", 0) orch.SetCommandOverrides(func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) { joined := strings.Join(args, " ") switch { case name == "ssh" && strings.Contains(joined, "sudo -n /usr/bin/systemctl --version"): return "sudo: a password is required", errors.New("exit status 1") case name == "kubectl" && strings.Contains(joined, "-n ops get secret sudo-titan-05 -o json"): return `{"data":{"password":"` + base64.StdEncoding.EncodeToString([]byte(secret)) + `"}}`, nil default: return "", nil } }, nil) inputSeen := "" commandSeen := "" orch.SetSSHInputOverride(func(_ context.Context, _ time.Duration, node string, command string, input string) (string, error) { if node != "titan-05" { t.Fatalf("unexpected node %q", node) } commandSeen = command inputSeen = input return "systemd 255", nil }) if err := orch.preflightHostPrivilege(context.Background(), "titan-05"); err != nil { t.Fatalf("preflightHostPrivilege failed: %v", err) } if !strings.Contains(commandSeen, "sudo -S -p '' /usr/bin/systemctl --version") { t.Fatalf("expected sudo stdin command, got %q", commandSeen) } if inputSeen != secret+"\n" { t.Fatalf("expected sudo password on stdin only") } if strings.Contains(logs.String(), secret) || strings.Contains(commandSeen, secret) { t.Fatalf("sudo password leaked through logs or command") } } // TestHostPrivilegeMissingSecretReportsUnavailable runs one orchestration or CLI step. // Signature: TestHostPrivilegeMissingSecretReportsUnavailable(t *testing.T). // Why: missing Vault/Kubernetes sudo material should be a classified blocker, // not an opaque loop of failed host commands. func TestHostPrivilegeMissingSecretReportsUnavailable(t *testing.T) { orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ { match: matchContains("ssh", "sudo -n /usr/bin/systemctl --version"), out: "sudo: a password is required", err: errors.New("exit status 1"), }, }) err := orch.preflightHostPrivilege(context.Background(), "titan-05") if err == nil || !strings.Contains(err.Error(), "host-privilege-unavailable") { t.Fatalf("expected host-privilege-unavailable, got %v", err) } } // TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel runs one orchestration or CLI step. // Signature: TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel(t *testing.T). // Why: a Kubernetes Ready node with a Longhorn Node CR should regain missing // manager selector labels instead of leaving PVC workloads blocked. func TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel(t *testing.T) { labeled := false orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ { match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`, }, { match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager", "-o", "json"), out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true"}}}}`, }, { match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-22","labels":{"kubernetes.io/hostname":"titan-22"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`, }, { match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json", "-l", "longhorn-host=true"), out: `{"items":[]}`, }, { match: func(name string, args []string) bool { if !matchContains("kubectl", "label", "node", "titan-22", "--overwrite", "longhorn-host=true")(name, args) { return false } labeled = true return true }, }, }) repaired, err := orch.reconcileLonghornKubernetesReadiness(context.Background()) if err != nil { t.Fatalf("reconcileLonghornKubernetesReadiness failed: %v", err) } if repaired != 1 || !labeled { t.Fatalf("expected one Longhorn label repair, repaired=%d labeled=%v", repaired, labeled) } } // TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter runs one orchestration or CLI step. // Signature: TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter(t *testing.T). // Why: stale RWO cleanup may force-delete sidecar-only owners but must not clear // an API object while a live application container still mounts the PVC. func TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter(t *testing.T) { old := time.Now().Add(-30 * time.Minute).UTC().Format(time.RFC3339) pvcJSON := `{"items":[{"metadata":{"namespace":"finance","name":"firefly-storage"},"spec":{"accessModes":["ReadWriteOnce"],"volumeName":"pvc-1"}}]}` eventsJSON := `{"items":[{"involvedObject":{"kind":"Pod","namespace":"finance","name":"firefly-new"},"type":"Warning","reason":"FailedAttachVolume","message":"Multi-Attach error for volume pvc-1: Volume is already used by pod(s) firefly-old"}]}` t.Run("sidecar-only force delete", func(t *testing.T) { podsJSON := `{"items":[` + staleRWOPodJSON("firefly-old", old, "titan-04", true, false) + `,` + replacementRWOPodJSON("firefly-new", "titan-06") + `]}` orch := staleRWOOrchestrator(t, podsJSON, pvcJSON, eventsJSON) var pods podList if err := jsonUnmarshal(podsJSON, &pods); err != nil { t.Fatalf("decode pods: %v", err) } decisions, err := orch.staleRWOPVCOwnerDecisions(context.Background(), pods, 5*time.Minute) if err != nil { t.Fatalf("staleRWOPVCOwnerDecisions failed: %v", err) } decision := decisions["finance/firefly-old"] if !decision.ForceDelete || decision.Unsafe { t.Fatalf("expected sidecar-only force delete decision, got %#v", decision) } }) t.Run("live writer unsafe", func(t *testing.T) { podsJSON := `{"items":[` + staleRWOPodJSON("firefly-old", old, "titan-04", true, true) + `,` + replacementRWOPodJSON("firefly-new", "titan-06") + `]}` orch := staleRWOOrchestrator(t, podsJSON, pvcJSON, eventsJSON) var pods podList if err := jsonUnmarshal(podsJSON, &pods); err != nil { t.Fatalf("decode pods: %v", err) } decisions, err := orch.staleRWOPVCOwnerDecisions(context.Background(), pods, 5*time.Minute) if err != nil { t.Fatalf("staleRWOPVCOwnerDecisions failed: %v", err) } decision := decisions["finance/firefly-old"] if !decision.Unsafe || decision.ForceDelete { t.Fatalf("expected live writer unsafe decision, got %#v", decision) } }) } // TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS runs one orchestration or CLI step. // Signature: TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS(t *testing.T). // Why: DNS-caused image pulls should be reported as registry/DNS blockers rather // than recycled as if deleting the pod could fix name resolution. func TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS(t *testing.T) { orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ { match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: `{"items":[{"involvedObject":{"kind":"Pod","namespace":"logging","name":"oauth2"},"type":"Warning","reason":"Failed","message":"Failed to pull image: lookup registry-1.docker.io: Try again"}]}`, }, }) reasons, err := orch.imagePullDNSBlockerReasons(context.Background()) if err != nil { t.Fatalf("imagePullDNSBlockerReasons failed: %v", err) } if got := reasons["logging/oauth2"]; got != "ImagePullDNSBlocker:registry-1.docker.io" { t.Fatalf("unexpected image pull DNS reason %q", got) } } // TestVaultSealedFallsBackToHTTPHealthWhenExecKilled runs one orchestration or CLI step. // Signature: TestVaultSealedFallsBackToHTTPHealthWhenExecKilled(t *testing.T). // Why: a transient killed kubectl exec should not block startup when Vault HTTP // health proves the pod is initialized and unsealed. func TestVaultSealedFallsBackToHTTPHealthWhenExecKilled(t *testing.T) { orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ { match: matchContains("kubectl", "-n", "vault", "exec", "vault-0"), err: errors.New("signal: killed"), }, { match: matchContains("kubectl", "-n", "vault", "get", "--raw", "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health"), out: `{"initialized":true,"sealed":false,"standby":false}`, }, }) sealed, err := orch.vaultSealed(context.Background()) if err != nil || sealed { t.Fatalf("expected HTTP fallback to report unsealed, sealed=%v err=%v", sealed, err) } } // TestTCPServiceChecklistReadyChecksMailProtocolBanner runs one orchestration or CLI step. // Signature: TestTCPServiceChecklistReadyChecksMailProtocolBanner(t *testing.T). // Why: Mailu is not covered by HTTP ingress checks, so Ananke needs direct // protocol probes that validate a TCP greeting and command response. func TestTCPServiceChecklistReadyChecksMailProtocolBanner(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen: %v", err) } defer listener.Close() done := make(chan struct{}) go func() { defer close(done) conn, acceptErr := listener.Accept() if acceptErr != nil { return } defer conn.Close() _, _ = conn.Write([]byte("220 mail.bstein.dev ESMTP ready\r\n")) _, _ = bufio.NewReader(conn).ReadString('\n') _, _ = conn.Write([]byte("221 2.0.0 Bye\r\n")) }() host, port, err := net.SplitHostPort(listener.Addr().String()) if err != nil { t.Fatalf("split listener address: %v", err) } check := config.TCPServiceChecklistCheck{ Name: "mail-smtp", Host: host, Port: atoiForTest(t, port), Send: "QUIT\r\n", ExpectContains: "ESMTP", TimeoutSeconds: 2, } orch := buildOrchestratorWithStubs(t, config.Config{}, nil) ok, detail := orch.tcpServiceCheckReady(context.Background(), check) if !ok { t.Fatalf("expected TCP mail check to pass, detail=%s", detail) } <-done } // TestServiceChecklistReadyIncludesTCPFailures runs one orchestration or CLI step. // Signature: TestServiceChecklistReadyIncludesTCPFailures(t *testing.T). // Why: startup should fail clearly when a configured mail protocol endpoint is // down even if there are no HTTP checks in that config. func TestServiceChecklistReadyIncludesTCPFailures(t *testing.T) { orch := buildOrchestratorWithStubs(t, config.Config{ Startup: config.Startup{ TCPServiceChecklist: []config.TCPServiceChecklistCheck{ { Name: "mail-smtp", Host: "127.0.0.1", Port: 1, ExpectContains: "ESMTP", TimeoutSeconds: 1, }, }, }, }, nil) ok, detail := orch.serviceChecklistReady(context.Background()) if ok { t.Fatalf("expected TCP-only checklist to fail") } if !strings.Contains(detail, "tcp mail-smtp") { t.Fatalf("expected TCP failure detail, got %q", detail) } } // TestEndpointAddressCountIgnoresKubectlWarnings runs one orchestration or CLI step. // Signature: TestEndpointAddressCountIgnoresKubectlWarnings(t *testing.T). // Why: Kubernetes endpoint deprecation warnings must not make an empty service // look ready by contributing a fake output line. func TestEndpointAddressCountIgnoresKubectlWarnings(t *testing.T) { orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ { match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "endpoints", "mailu-admin", "-o", "json"), out: "Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\n{\"subsets\":[]}", }, }) count, err := orch.endpointAddressCount(context.Background(), "mailu-mailserver", "mailu-admin") if err != nil { t.Fatalf("endpointAddressCount failed: %v", err) } if count != 0 { t.Fatalf("expected empty endpoint count after warning strip, got %d", count) } } // TestCriticalBackendStartupProbeRepairCopiesLivenessProbe runs one orchestration or CLI step. // Signature: TestCriticalBackendStartupProbeRepairCopiesLivenessProbe(t *testing.T). // Why: slow critical backends should get a bounded startup grace without // changing app-specific liveness semantics or requiring a Mailu special case. func TestCriticalBackendStartupProbeRepairCopiesLivenessProbe(t *testing.T) { var patchPayload string orch := buildOrchestratorWithStubs(t, config.Config{ Startup: config.Startup{ CriticalServiceStartupProbeRepair: true, CriticalServiceStartupProbeThreshold: 24, }, }, []commandStub{ { match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "deployment", "mailu-admin", "-o", "json"), out: `{"spec":{"template":{"spec":{"containers":[{"name":"admin","livenessProbe":{"httpGet":{"path":"/ping","port":"http"},"periodSeconds":10,"failureThreshold":3}},` + `{"name":"sidecar"}]}}}}`, }, { match: func(name string, args []string) bool { if !matchContains("kubectl", "-n", "mailu-mailserver", "patch", "deployment", "mailu-admin", "--type=strategic", "-p")(name, args) { return false } for i, arg := range args { if arg == "-p" && i+1 < len(args) { patchPayload = args[i+1] } } return true }, }, { match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "statefulset", "mailu-admin", "-o", "json"), err: errors.New("not found"), }, }) repaired, err := orch.maybeRepairCriticalBackendStartupProbes(context.Background(), "mailu-mailserver", "mailu-admin") if err != nil { t.Fatalf("maybeRepairCriticalBackendStartupProbes failed: %v", err) } if len(repaired) != 1 || repaired[0] != "mailu-mailserver/deployment/mailu-admin" { t.Fatalf("unexpected repaired list: %v", repaired) } if patchPayload == "" { t.Fatalf("expected strategic merge patch payload") } if !strings.Contains(patchPayload, `"startupProbe"`) || !strings.Contains(patchPayload, `"failureThreshold":24`) || !strings.Contains(patchPayload, `"path":"/ping"`) { t.Fatalf("unexpected startup-probe patch payload: %s", patchPayload) } } // TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet runs one orchestration or CLI step. // Signature: TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet(t *testing.T). // Why: Mailu backends are Deployments; a kubectl NotFound message on the // StatefulSet probe path must not turn a successful/no-op Deployment check into // an auto-heal failure. func TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet(t *testing.T) { orch := buildOrchestratorWithStubs(t, config.Config{ Startup: config.Startup{ CriticalServiceStartupProbeRepair: true, CriticalServiceStartupProbeThreshold: 24, }, }, []commandStub{ { match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "deployment", "mailu-rspamd", "-o", "json"), out: `{"spec":{"template":{"spec":{"containers":[{"name":"rspamd","livenessProbe":{"httpGet":{"path":"/","port":"http"}},"startupProbe":{"httpGet":{"path":"/","port":"http"}}}]}}}}`, }, { match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "statefulset", "mailu-rspamd", "-o", "json"), out: `Error from server (NotFound): statefulsets.apps "mailu-rspamd" not found`, err: errors.New("exit status 1"), }, }) repaired, err := orch.maybeRepairCriticalBackendStartupProbes(context.Background(), "mailu-mailserver", "mailu-rspamd") if err != nil { t.Fatalf("expected missing statefulset to be ignored, got %v", err) } if len(repaired) != 0 { t.Fatalf("expected no repair for existing startup probe, got %v", repaired) } } // atoiForTest runs one orchestration or CLI step. // Signature: atoiForTest(t *testing.T, raw string) int. // Why: TCP listener fixtures need the kernel-selected port as a typed config // value without obscuring test failures. func atoiForTest(t *testing.T, raw string) int { t.Helper() port, err := strconv.Atoi(raw) if err != nil { t.Fatalf("parse port %q: %v", raw, err) } return port } // staleRWOOrchestrator runs one orchestration or CLI step. // Signature: staleRWOOrchestrator(t *testing.T, podsJSON string, pvcJSON string, eventsJSON string) *Orchestrator. // Why: stale RWO tests need a tiny fake API surface for pods, PVCs, and events // without depending on a live Kubernetes cluster. func staleRWOOrchestrator(t *testing.T, podsJSON string, pvcJSON string, eventsJSON string) *Orchestrator { t.Helper() return buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ {match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: eventsJSON}, {match: matchContains("kubectl", "get", "pvc", "-A", "-o", "json"), out: pvcJSON}, {match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: podsJSON}, }) } // staleRWOPodJSON runs one orchestration or CLI step. // Signature: staleRWOPodJSON(name string, deletedAt string, node string, sidecarRunning bool, appRunning bool) string. // Why: the sidecar-only and live-writer fixtures differ only in container // status, so one builder keeps the safety distinction obvious. func staleRWOPodJSON(name string, deletedAt string, node string, sidecarRunning bool, appRunning bool) string { statuses := []string{} if sidecarRunning { statuses = append(statuses, `{"name":"vault-agent","state":{"running":{"startedAt":"`+deletedAt+`"}}}`) } if appRunning { statuses = append(statuses, `{"name":"firefly","state":{"running":{"startedAt":"`+deletedAt+`"}}}`) } return `{"metadata":{"namespace":"finance","name":"` + name + `","creationTimestamp":"` + deletedAt + `","deletionTimestamp":"` + deletedAt + `","ownerReferences":[{"kind":"ReplicaSet","name":"firefly"}]},"spec":{"nodeName":"` + node + `","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"firefly-storage"}},{"name":"vault-secret"}],"containers":[{"name":"firefly","volumeMounts":[{"name":"data","mountPath":"/var/www/html/storage"}]},{"name":"vault-agent","volumeMounts":[{"name":"vault-secret","mountPath":"/vault/secrets"}]}]},"status":{"phase":"Running","containerStatuses":[` + strings.Join(statuses, ",") + `]}}` } // replacementRWOPodJSON runs one orchestration or CLI step. // Signature: replacementRWOPodJSON(name string, node string) string. // Why: stale-owner tests need a controller sibling blocked in Pending on a // different node while sharing the same RWO claim. func replacementRWOPodJSON(name string, node string) string { return `{"metadata":{"namespace":"finance","name":"` + name + `","creationTimestamp":"` + time.Now().Add(-10*time.Minute).UTC().Format(time.RFC3339) + `","ownerReferences":[{"kind":"ReplicaSet","name":"firefly"}]},"spec":{"nodeName":"` + node + `","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"firefly-storage"}}],"containers":[{"name":"firefly","volumeMounts":[{"name":"data","mountPath":"/var/www/html/storage"}]}]},"status":{"phase":"Pending","containerStatuses":[{"name":"firefly","state":{"waiting":{"reason":"ContainerCreating"}}}]}}` } // jsonUnmarshal runs one orchestration or CLI step. // Signature: jsonUnmarshal(raw string, target any) error. // Why: local JSON fixtures should use the same decoder semantics as Kubernetes // command output parsing. func jsonUnmarshal(raw string, target any) error { return json.Unmarshal([]byte(raw), target) }