ananke/internal/cluster/orchestrator_hardening_test.go
2026-07-07 18:29:51 -03:00

456 lines
20 KiB
Go

package cluster
import (
"bufio"
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"log"
"net"
"strconv"
"strings"
"testing"
"time"
"scm.bstein.dev/bstein/ananke/internal/config"
)
// TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret runs one orchestration or CLI step.
// Signature: TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret(t *testing.T).
// Why: host repair must recover from password-required sudo while keeping the
// sudo password out of command arguments and logs.
func TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret(t *testing.T) {
secret := "correct horse battery staple"
var logs bytes.Buffer
orch := buildOrchestratorWithStubs(t, config.Config{
Startup: config.Startup{
HostSudoSecretNamespace: "ops",
HostSudoSecretNameTemplate: "sudo-{node}",
HostSudoSecretPasswordKey: "password",
},
}, nil)
orch.log = log.New(&logs, "", 0)
orch.SetCommandOverrides(func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
joined := strings.Join(args, " ")
switch {
case name == "ssh" && strings.Contains(joined, "sudo -n /usr/bin/systemctl --version"):
return "sudo: a password is required", errors.New("exit status 1")
case name == "kubectl" && strings.Contains(joined, "-n ops get secret sudo-titan-05 -o json"):
return `{"data":{"password":"` + base64.StdEncoding.EncodeToString([]byte(secret)) + `"}}`, nil
default:
return "", nil
}
}, nil)
inputSeen := ""
commandSeen := ""
orch.SetSSHInputOverride(func(_ context.Context, _ time.Duration, node string, command string, input string) (string, error) {
if node != "titan-05" {
t.Fatalf("unexpected node %q", node)
}
commandSeen = command
inputSeen = input
return "systemd 255", nil
})
if err := orch.preflightHostPrivilege(context.Background(), "titan-05"); err != nil {
t.Fatalf("preflightHostPrivilege failed: %v", err)
}
if !strings.Contains(commandSeen, "sudo -S -p '' /usr/bin/systemctl --version") {
t.Fatalf("expected sudo stdin command, got %q", commandSeen)
}
if inputSeen != secret+"\n" {
t.Fatalf("expected sudo password on stdin only")
}
if strings.Contains(logs.String(), secret) || strings.Contains(commandSeen, secret) {
t.Fatalf("sudo password leaked through logs or command")
}
}
// TestHostPrivilegeMissingSecretReportsUnavailable runs one orchestration or CLI step.
// Signature: TestHostPrivilegeMissingSecretReportsUnavailable(t *testing.T).
// Why: missing Vault/Kubernetes sudo material should be a classified blocker,
// not an opaque loop of failed host commands.
func TestHostPrivilegeMissingSecretReportsUnavailable(t *testing.T) {
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{
match: matchContains("ssh", "sudo -n /usr/bin/systemctl --version"),
out: "sudo: a password is required",
err: errors.New("exit status 1"),
},
})
err := orch.preflightHostPrivilege(context.Background(), "titan-05")
if err == nil || !strings.Contains(err.Error(), "host-privilege-unavailable") {
t.Fatalf("expected host-privilege-unavailable, got %v", err)
}
}
// TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel runs one orchestration or CLI step.
// Signature: TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel(t *testing.T).
// Why: a Kubernetes Ready node with a Longhorn Node CR should regain missing
// manager selector labels instead of leaving PVC workloads blocked.
func TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel(t *testing.T) {
labeled := false
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{
match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"),
out: `{"items":[{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`,
},
{
match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager", "-o", "json"),
out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true"}}}}`,
},
{
match: matchContains("kubectl", "get", "nodes", "-o", "json"),
out: `{"items":[{"metadata":{"name":"titan-22","labels":{"kubernetes.io/hostname":"titan-22"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`,
},
{
match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json", "-l", "longhorn-host=true"),
out: `{"items":[]}`,
},
{
match: func(name string, args []string) bool {
if !matchContains("kubectl", "label", "node", "titan-22", "--overwrite", "longhorn-host=true")(name, args) {
return false
}
labeled = true
return true
},
},
})
repaired, err := orch.reconcileLonghornKubernetesReadiness(context.Background())
if err != nil {
t.Fatalf("reconcileLonghornKubernetesReadiness failed: %v", err)
}
if repaired != 1 || !labeled {
t.Fatalf("expected one Longhorn label repair, repaired=%d labeled=%v", repaired, labeled)
}
}
// TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter runs one orchestration or CLI step.
// Signature: TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter(t *testing.T).
// Why: stale RWO cleanup may force-delete sidecar-only owners but must not clear
// an API object while a live application container still mounts the PVC.
func TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter(t *testing.T) {
old := time.Now().Add(-30 * time.Minute).UTC().Format(time.RFC3339)
pvcJSON := `{"items":[{"metadata":{"namespace":"finance","name":"firefly-storage"},"spec":{"accessModes":["ReadWriteOnce"],"volumeName":"pvc-1"}}]}`
eventsJSON := `{"items":[{"involvedObject":{"kind":"Pod","namespace":"finance","name":"firefly-new"},"type":"Warning","reason":"FailedAttachVolume","message":"Multi-Attach error for volume pvc-1: Volume is already used by pod(s) firefly-old"}]}`
t.Run("sidecar-only force delete", func(t *testing.T) {
podsJSON := `{"items":[` +
staleRWOPodJSON("firefly-old", old, "titan-04", true, false) + `,` +
replacementRWOPodJSON("firefly-new", "titan-06") +
`]}`
orch := staleRWOOrchestrator(t, podsJSON, pvcJSON, eventsJSON)
var pods podList
if err := jsonUnmarshal(podsJSON, &pods); err != nil {
t.Fatalf("decode pods: %v", err)
}
decisions, err := orch.staleRWOPVCOwnerDecisions(context.Background(), pods, 5*time.Minute)
if err != nil {
t.Fatalf("staleRWOPVCOwnerDecisions failed: %v", err)
}
decision := decisions["finance/firefly-old"]
if !decision.ForceDelete || decision.Unsafe {
t.Fatalf("expected sidecar-only force delete decision, got %#v", decision)
}
})
t.Run("live writer unsafe", func(t *testing.T) {
podsJSON := `{"items":[` +
staleRWOPodJSON("firefly-old", old, "titan-04", true, true) + `,` +
replacementRWOPodJSON("firefly-new", "titan-06") +
`]}`
orch := staleRWOOrchestrator(t, podsJSON, pvcJSON, eventsJSON)
var pods podList
if err := jsonUnmarshal(podsJSON, &pods); err != nil {
t.Fatalf("decode pods: %v", err)
}
decisions, err := orch.staleRWOPVCOwnerDecisions(context.Background(), pods, 5*time.Minute)
if err != nil {
t.Fatalf("staleRWOPVCOwnerDecisions failed: %v", err)
}
decision := decisions["finance/firefly-old"]
if !decision.Unsafe || decision.ForceDelete {
t.Fatalf("expected live writer unsafe decision, got %#v", decision)
}
})
}
// TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS runs one orchestration or CLI step.
// Signature: TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS(t *testing.T).
// Why: DNS-caused image pulls should be reported as registry/DNS blockers rather
// than recycled as if deleting the pod could fix name resolution.
func TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS(t *testing.T) {
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{
match: matchContains("kubectl", "get", "events", "-A", "-o", "json"),
out: `{"items":[{"involvedObject":{"kind":"Pod","namespace":"logging","name":"oauth2"},"type":"Warning","reason":"Failed","message":"Failed to pull image: lookup registry-1.docker.io: Try again"}]}`,
},
})
reasons, err := orch.imagePullDNSBlockerReasons(context.Background())
if err != nil {
t.Fatalf("imagePullDNSBlockerReasons failed: %v", err)
}
if got := reasons["logging/oauth2"]; got != "ImagePullDNSBlocker:registry-1.docker.io" {
t.Fatalf("unexpected image pull DNS reason %q", got)
}
}
// TestVaultSealedFallsBackToHTTPHealthWhenExecKilled runs one orchestration or CLI step.
// Signature: TestVaultSealedFallsBackToHTTPHealthWhenExecKilled(t *testing.T).
// Why: a transient killed kubectl exec should not block startup when Vault HTTP
// health proves the pod is initialized and unsealed.
func TestVaultSealedFallsBackToHTTPHealthWhenExecKilled(t *testing.T) {
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{
match: matchContains("kubectl", "-n", "vault", "exec", "vault-0"),
err: errors.New("signal: killed"),
},
{
match: matchContains("kubectl", "-n", "vault", "get", "--raw", "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health"),
out: `{"initialized":true,"sealed":false,"standby":false}`,
},
})
sealed, err := orch.vaultSealed(context.Background())
if err != nil || sealed {
t.Fatalf("expected HTTP fallback to report unsealed, sealed=%v err=%v", sealed, err)
}
}
// TestTCPServiceChecklistReadyChecksMailProtocolBanner runs one orchestration or CLI step.
// Signature: TestTCPServiceChecklistReadyChecksMailProtocolBanner(t *testing.T).
// Why: Mailu is not covered by HTTP ingress checks, so Ananke needs direct
// protocol probes that validate a TCP greeting and command response.
func TestTCPServiceChecklistReadyChecksMailProtocolBanner(t *testing.T) {
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
defer listener.Close()
done := make(chan struct{})
go func() {
defer close(done)
conn, acceptErr := listener.Accept()
if acceptErr != nil {
return
}
defer conn.Close()
_, _ = conn.Write([]byte("220 mail.bstein.dev ESMTP ready\r\n"))
_, _ = bufio.NewReader(conn).ReadString('\n')
_, _ = conn.Write([]byte("221 2.0.0 Bye\r\n"))
}()
host, port, err := net.SplitHostPort(listener.Addr().String())
if err != nil {
t.Fatalf("split listener address: %v", err)
}
check := config.TCPServiceChecklistCheck{
Name: "mail-smtp",
Host: host,
Port: atoiForTest(t, port),
Send: "QUIT\r\n",
ExpectContains: "ESMTP",
TimeoutSeconds: 2,
}
orch := buildOrchestratorWithStubs(t, config.Config{}, nil)
ok, detail := orch.tcpServiceCheckReady(context.Background(), check)
if !ok {
t.Fatalf("expected TCP mail check to pass, detail=%s", detail)
}
<-done
}
// TestServiceChecklistReadyIncludesTCPFailures runs one orchestration or CLI step.
// Signature: TestServiceChecklistReadyIncludesTCPFailures(t *testing.T).
// Why: startup should fail clearly when a configured mail protocol endpoint is
// down even if there are no HTTP checks in that config.
func TestServiceChecklistReadyIncludesTCPFailures(t *testing.T) {
orch := buildOrchestratorWithStubs(t, config.Config{
Startup: config.Startup{
TCPServiceChecklist: []config.TCPServiceChecklistCheck{
{
Name: "mail-smtp",
Host: "127.0.0.1",
Port: 1,
ExpectContains: "ESMTP",
TimeoutSeconds: 1,
},
},
},
}, nil)
ok, detail := orch.serviceChecklistReady(context.Background())
if ok {
t.Fatalf("expected TCP-only checklist to fail")
}
if !strings.Contains(detail, "tcp mail-smtp") {
t.Fatalf("expected TCP failure detail, got %q", detail)
}
}
// TestEndpointAddressCountIgnoresKubectlWarnings runs one orchestration or CLI step.
// Signature: TestEndpointAddressCountIgnoresKubectlWarnings(t *testing.T).
// Why: Kubernetes endpoint deprecation warnings must not make an empty service
// look ready by contributing a fake output line.
func TestEndpointAddressCountIgnoresKubectlWarnings(t *testing.T) {
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{
match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "endpoints", "mailu-admin", "-o", "json"),
out: "Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\n{\"subsets\":[]}",
},
})
count, err := orch.endpointAddressCount(context.Background(), "mailu-mailserver", "mailu-admin")
if err != nil {
t.Fatalf("endpointAddressCount failed: %v", err)
}
if count != 0 {
t.Fatalf("expected empty endpoint count after warning strip, got %d", count)
}
}
// TestCriticalBackendStartupProbeRepairCopiesLivenessProbe runs one orchestration or CLI step.
// Signature: TestCriticalBackendStartupProbeRepairCopiesLivenessProbe(t *testing.T).
// Why: slow critical backends should get a bounded startup grace without
// changing app-specific liveness semantics or requiring a Mailu special case.
func TestCriticalBackendStartupProbeRepairCopiesLivenessProbe(t *testing.T) {
var patchPayload string
orch := buildOrchestratorWithStubs(t, config.Config{
Startup: config.Startup{
CriticalServiceStartupProbeRepair: true,
CriticalServiceStartupProbeThreshold: 24,
},
}, []commandStub{
{
match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "deployment", "mailu-admin", "-o", "json"),
out: `{"spec":{"template":{"spec":{"containers":[{"name":"admin","livenessProbe":{"httpGet":{"path":"/ping","port":"http"},"periodSeconds":10,"failureThreshold":3}},` +
`{"name":"sidecar"}]}}}}`,
},
{
match: func(name string, args []string) bool {
if !matchContains("kubectl", "-n", "mailu-mailserver", "patch", "deployment", "mailu-admin", "--type=strategic", "-p")(name, args) {
return false
}
for i, arg := range args {
if arg == "-p" && i+1 < len(args) {
patchPayload = args[i+1]
}
}
return true
},
},
{
match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "statefulset", "mailu-admin", "-o", "json"),
err: errors.New("not found"),
},
})
repaired, err := orch.maybeRepairCriticalBackendStartupProbes(context.Background(), "mailu-mailserver", "mailu-admin")
if err != nil {
t.Fatalf("maybeRepairCriticalBackendStartupProbes failed: %v", err)
}
if len(repaired) != 1 || repaired[0] != "mailu-mailserver/deployment/mailu-admin" {
t.Fatalf("unexpected repaired list: %v", repaired)
}
if patchPayload == "" {
t.Fatalf("expected strategic merge patch payload")
}
if !strings.Contains(patchPayload, `"startupProbe"`) || !strings.Contains(patchPayload, `"failureThreshold":24`) || !strings.Contains(patchPayload, `"path":"/ping"`) {
t.Fatalf("unexpected startup-probe patch payload: %s", patchPayload)
}
}
// TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet runs one orchestration or CLI step.
// Signature: TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet(t *testing.T).
// Why: Mailu backends are Deployments; a kubectl NotFound message on the
// StatefulSet probe path must not turn a successful/no-op Deployment check into
// an auto-heal failure.
func TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet(t *testing.T) {
orch := buildOrchestratorWithStubs(t, config.Config{
Startup: config.Startup{
CriticalServiceStartupProbeRepair: true,
CriticalServiceStartupProbeThreshold: 24,
},
}, []commandStub{
{
match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "deployment", "mailu-rspamd", "-o", "json"),
out: `{"spec":{"template":{"spec":{"containers":[{"name":"rspamd","livenessProbe":{"httpGet":{"path":"/","port":"http"}},"startupProbe":{"httpGet":{"path":"/","port":"http"}}}]}}}}`,
},
{
match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "statefulset", "mailu-rspamd", "-o", "json"),
out: `Error from server (NotFound): statefulsets.apps "mailu-rspamd" not found`,
err: errors.New("exit status 1"),
},
})
repaired, err := orch.maybeRepairCriticalBackendStartupProbes(context.Background(), "mailu-mailserver", "mailu-rspamd")
if err != nil {
t.Fatalf("expected missing statefulset to be ignored, got %v", err)
}
if len(repaired) != 0 {
t.Fatalf("expected no repair for existing startup probe, got %v", repaired)
}
}
// atoiForTest runs one orchestration or CLI step.
// Signature: atoiForTest(t *testing.T, raw string) int.
// Why: TCP listener fixtures need the kernel-selected port as a typed config
// value without obscuring test failures.
func atoiForTest(t *testing.T, raw string) int {
t.Helper()
port, err := strconv.Atoi(raw)
if err != nil {
t.Fatalf("parse port %q: %v", raw, err)
}
return port
}
// staleRWOOrchestrator runs one orchestration or CLI step.
// Signature: staleRWOOrchestrator(t *testing.T, podsJSON string, pvcJSON string, eventsJSON string) *Orchestrator.
// Why: stale RWO tests need a tiny fake API surface for pods, PVCs, and events
// without depending on a live Kubernetes cluster.
func staleRWOOrchestrator(t *testing.T, podsJSON string, pvcJSON string, eventsJSON string) *Orchestrator {
t.Helper()
return buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: eventsJSON},
{match: matchContains("kubectl", "get", "pvc", "-A", "-o", "json"), out: pvcJSON},
{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: podsJSON},
})
}
// staleRWOPodJSON runs one orchestration or CLI step.
// Signature: staleRWOPodJSON(name string, deletedAt string, node string, sidecarRunning bool, appRunning bool) string.
// Why: the sidecar-only and live-writer fixtures differ only in container
// status, so one builder keeps the safety distinction obvious.
func staleRWOPodJSON(name string, deletedAt string, node string, sidecarRunning bool, appRunning bool) string {
statuses := []string{}
if sidecarRunning {
statuses = append(statuses, `{"name":"vault-agent","state":{"running":{"startedAt":"`+deletedAt+`"}}}`)
}
if appRunning {
statuses = append(statuses, `{"name":"firefly","state":{"running":{"startedAt":"`+deletedAt+`"}}}`)
}
return `{"metadata":{"namespace":"finance","name":"` + name + `","creationTimestamp":"` + deletedAt + `","deletionTimestamp":"` + deletedAt + `","ownerReferences":[{"kind":"ReplicaSet","name":"firefly"}]},"spec":{"nodeName":"` + node + `","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"firefly-storage"}},{"name":"vault-secret"}],"containers":[{"name":"firefly","volumeMounts":[{"name":"data","mountPath":"/var/www/html/storage"}]},{"name":"vault-agent","volumeMounts":[{"name":"vault-secret","mountPath":"/vault/secrets"}]}]},"status":{"phase":"Running","containerStatuses":[` + strings.Join(statuses, ",") + `]}}`
}
// replacementRWOPodJSON runs one orchestration or CLI step.
// Signature: replacementRWOPodJSON(name string, node string) string.
// Why: stale-owner tests need a controller sibling blocked in Pending on a
// different node while sharing the same RWO claim.
func replacementRWOPodJSON(name string, node string) string {
return `{"metadata":{"namespace":"finance","name":"` + name + `","creationTimestamp":"` + time.Now().Add(-10*time.Minute).UTC().Format(time.RFC3339) + `","ownerReferences":[{"kind":"ReplicaSet","name":"firefly"}]},"spec":{"nodeName":"` + node + `","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"firefly-storage"}}],"containers":[{"name":"firefly","volumeMounts":[{"name":"data","mountPath":"/var/www/html/storage"}]}]},"status":{"phase":"Pending","containerStatuses":[{"name":"firefly","state":{"waiting":{"reason":"ContainerCreating"}}}]}}`
}
// jsonUnmarshal runs one orchestration or CLI step.
// Signature: jsonUnmarshal(raw string, target any) error.
// Why: local JSON fixtures should use the same decoder semantics as Kubernetes
// command output parsing.
func jsonUnmarshal(raw string, target any) error {
return json.Unmarshal([]byte(raw), target)
}