ananke/internal/cluster/orchestrator_quality_gate_closeout_test.go

package cluster

import (
	"bufio"
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"log"
	"net"
	"strings"
	"testing"
	"time"

	"scm.bstein.dev/bstein/ananke/internal/config"
	"scm.bstein.dev/bstein/ananke/internal/execx"
	"scm.bstein.dev/bstein/ananke/internal/state"
)

// TestTCPServiceChecklistAndBackendHealBranches runs one orchestration or CLI step.
// Signature: TestTCPServiceChecklistAndBackendHealBranches(t *testing.T).
// Why: TCP health checks drive Mailu recovery paths and need branch coverage for
// banner matching, unnamed checks, duplicate backend suppression, and heal notes.
func TestTCPServiceChecklistAndBackendHealBranches(t *testing.T) {
	addr := startTCPFixture(t, "220 ready\r\n", "250 pong\r\n")
	host, portText, err := net.SplitHostPort(addr)
	if err != nil {
		t.Fatalf("split tcp fixture address: %v", err)
	}
	var port int
	if _, err := fmt.Sscanf(portText, "%d", &port); err != nil {
		t.Fatalf("parse tcp fixture port: %v", err)
	}

	cfg := config.Config{
		Startup: config.Startup{
			TCPServiceChecklist: []config.TCPServiceChecklistCheck{
				{Name: "smtp", Host: host, Port: port, Send: "PING\r\n", ExpectContains: "pong", Namespace: "mail", Service: "mailu"},
			},
		},
	}
	orch := buildOrchestratorWithStubs(t, cfg, nil)
	ok, detail := orch.tcpServiceChecklistReady(context.Background())
	if !ok || detail != "tcp-checks=1" {
		t.Fatalf("expected tcp checklist success, ok=%v detail=%q", ok, detail)
	}

	unnamed := config.TCPServiceChecklistCheck{Host: host, Port: port, ExpectContains: "missing"}
	ok, detail = orch.tcpServiceCheckReady(context.Background(), unnamed)
	if ok || !strings.Contains(detail, `banner missing expected marker "missing"`) {
		t.Fatalf("expected missing marker detail, ok=%v detail=%q", ok, detail)
	}
	orch.cfg.Startup.TCPServiceChecklist = []config.TCPServiceChecklistCheck{unnamed}
	ok, detail = orch.tcpServiceChecklistReady(context.Background())
	if ok || !strings.Contains(detail, host+":"+portText) {
		t.Fatalf("expected unnamed tcp check to use host:port detail, ok=%v detail=%q", ok, detail)
	}

	healAddr := startTCPFixture(t, "220 ready\r\n", "250 pong\r\n")
	healHost, healPortText, _ := net.SplitHostPort(healAddr)
	var healPort int
	_, _ = fmt.Sscanf(healPortText, "%d", &healPort)
	cfg.Startup.TCPServiceChecklist = []config.TCPServiceChecklistCheck{
		{Host: healHost, Port: healPort, Namespace: "", Service: "skip"},
		{Name: "ok", Host: healHost, Port: healPort, ExpectContains: "ready", Namespace: "mail", Service: "mailu"},
		{Name: "bad", Host: healHost, Port: healPort, ExpectContains: "absent", Namespace: "mail", Service: "mailu"},
		{Name: "dup", Host: healHost, Port: healPort, ExpectContains: "absent", Namespace: "mail", Service: "mailu"},
	}
	var scaled, rolled bool
	orch = buildOrchestratorWithStubs(t, cfg, []commandStub{
		{match: func(name string, args []string) bool {
			joined := strings.Join(args, " ")
			if name == "kubectl" && strings.Contains(joined, "-n mail scale deployment mailu --replicas=1") {
				scaled = true
				return true
			}
			if name == "kubectl" && strings.Contains(joined, "-n mail rollout status deployment/mailu") {
				rolled = true
				return true
			}
			return false
		}},
		{match: matchContains("kubectl", "-n", "mail", "scale", "statefulset", "mailu"), err: errors.New(`Error from server (NotFound): statefulsets.apps "mailu" not found`)},
	})
	healed, err := orch.healFailedTCPServiceBackends(context.Background())
	if err != nil {
		t.Fatalf("healFailedTCPServiceBackends: %v", err)
	}
	if len(healed) != 1 || healed[0] != "mail/deployment/mailu" || !scaled || !rolled {
		t.Fatalf("unexpected tcp backend heal result healed=%v scaled=%v rolled=%v", healed, scaled, rolled)
	}
	lastAttempt := time.Now().Add(-time.Minute)
	orch.maybeAutoHealTCPServiceBackends(context.Background(), &lastAttempt)
}

// TestHostPrivilegeFailureAndSecretBranches runs one orchestration or CLI step.
// Signature: TestHostPrivilegeFailureAndSecretBranches(t *testing.T).
// Why: host repair depends on stable allowlist, secret, sudo, and failure
// classes before Ananke can safely restart or reboot managed nodes.
func TestHostPrivilegeFailureAndSecretBranches(t *testing.T) {
	cfg := config.Config{
		SSHManagedNodes: []string{"titan-23"},
		SSHUser:         "atlas",
		Startup: config.Startup{
			HostSudoSecretNamespace:    "host-secrets",
			HostSudoSecretNameTemplate: "sudo-{node}",
			HostSudoSecretPasswordKey:  "password",
		},
	}
	orch := buildOrchestratorWithStubs(t, cfg, nil)
	if _, err := orch.runHostPrivilegedAction(context.Background(), " ", hostActionSudoPreflight, 0); err == nil {
		t.Fatalf("expected empty node privilege error")
	}
	if _, err := orch.runHostPrivilegedAction(context.Background(), "unmanaged", hostActionSudoPreflight, 0); err == nil {
		t.Fatalf("expected unmanaged node privilege error")
	}
	if _, err := orch.runHostPrivilegedAction(context.Background(), "titan-23", hostPrivilegedAction("bad-action"), 0); err == nil {
		t.Fatalf("expected bad action privilege error")
	}
	if got := orch.hostPrivilegedCommandTimeout(); got != 90*time.Second {
		t.Fatalf("expected default privileged timeout, got %s", got)
	}
	orch.cfg.Startup.HostPrivilegedCommandTimeoutSec = 7
	if got := orch.hostPrivilegedCommandTimeout(); got != 7*time.Second {
		t.Fatalf("expected configured privileged timeout, got %s", got)
	}

	actions := []struct {
		action hostPrivilegedAction
		args   []string
		want   string
	}{
		{hostActionHostReboot, nil, "systemctl reboot"},
		{hostActionModprobeDMCrypt, nil, "modprobe dm_crypt"},
		{hostActionCrictlPods, nil, "crictl pods -o json"},
		{hostActionCrictlPs, nil, "crictl ps -a -o json"},
		{hostActionCrictlInspect, []string{"abc123"}, "crictl inspect abc123"},
		{hostActionCrictlStop, []string{"abc123"}, "crictl stop abc123"},
	}
	for _, tc := range actions {
		got, err := hostPrivilegedCommandArgs(tc.action, tc.args...)
		if err != nil {
			t.Fatalf("hostPrivilegedCommandArgs(%s): %v", tc.action, err)
		}
		if strings.Join(got, " ") != tc.want {
			t.Fatalf("hostPrivilegedCommandArgs(%s)=%q want %q", tc.action, strings.Join(got, " "), tc.want)
		}
	}
	for _, tc := range []struct {
		action hostPrivilegedAction
		arg    string
	}{
		{hostActionCrictlInspect, "bad;id"},
		{hostActionCrictlStop, ""},
	} {
		if _, err := hostPrivilegedCommandArgs(tc.action, tc.arg); err == nil {
			t.Fatalf("expected unsafe arg rejection for %s", tc.action)
		}
	}

	errs := []error{
		classifyHostPrivilegeFailure("n", hostActionSudoPreflight, context.DeadlineExceeded, ""),
		classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("try again"), ""),
		classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("permission denied publickey"), ""),
		classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("exit 1"), "line one\nline two"),
	}
	wantClasses := []string{"host-command-timeout", "host-privilege-auth-failed", "host-ssh-unavailable", "host-command-failed"}
	for i, err := range errs {
		if !strings.Contains(err.Error(), wantClasses[i]) {
			t.Fatalf("expected %s in %q", wantClasses[i], err)
		}
	}

	secretJSON := `{"data":{"password":"` + base64.StdEncoding.EncodeToString([]byte("pw\n")) + `"}}`
	orch = buildOrchestratorWithStubs(t, cfg, []commandStub{
		{match: matchContains("ssh", "sudo -n /usr/bin/systemctl --version"), out: "sudo: a password is required", err: errors.New("sudo failed")},
		{match: matchContains("kubectl", "-n", "host-secrets", "get", "secret", "sudo-titan-23"), out: secretJSON},
	})
	orch.SetSSHInputOverride(func(_ context.Context, _ time.Duration, node, command, input string) (string, error) {
		if node != "titan-23" || !strings.Contains(command, "sudo -S") || input != "pw\n" {
			t.Fatalf("unexpected password-backed ssh node=%q command=%q input=%q", node, command, input)
		}
		return "incorrect password", errors.New("try again")
	})
	if _, err := orch.runHostPrivilegedAction(context.Background(), "titan-23", hostActionSudoPreflight, 0); err == nil || !strings.Contains(err.Error(), "host-privilege-auth-failed") {
		t.Fatalf("expected password-backed auth failure, got %v", err)
	}

	secretCases := []struct {
		name  string
		cfg   config.Config
		node  string
		out   string
		err   error
		class string
	}{
		{name: "unconfigured", cfg: config.Config{}, class: "secret-lookup-unconfigured"},
		{name: "empty-name", cfg: config.Config{Startup: config.Startup{HostSudoSecretNamespace: "ns", HostSudoSecretNameTemplate: "{node}"}}, node: " ", class: "secret-lookup-invalid"},
		{name: "query-error", cfg: cfg, err: errors.New("denied"), class: "secret-lookup-failed"},
		{name: "bad-json", cfg: cfg, out: "{", class: "secret-decode-failed"},
		{name: "missing-key", cfg: cfg, out: `{"data":{"other":"cHc="}}`, class: "secret-key-missing"},
		{name: "bad-base64", cfg: cfg, out: `{"data":{"password":"%"}}`, class: "secret-decode-failed"},
		{name: "empty-password", cfg: cfg, out: `{"data":{"password":"Cg=="}}`, class: "secret-empty"},
	}
	for _, tc := range secretCases {
		orch := buildOrchestratorWithStubs(t, tc.cfg, []commandStub{
			{match: matchContains("kubectl", "get", "secret"), out: tc.out, err: tc.err},
		})
		node := tc.node
		if node == "" {
			node = "titan-23"
		}
		_, class, err := orch.hostSudoPassword(context.Background(), node)
		if err == nil || class != tc.class {
			t.Fatalf("%s: expected class %q error, got class=%q err=%v", tc.name, tc.class, class, err)
		}
	}
}

// TestImagePullDNSAndCredentialBranches runs one orchestration or CLI step.
// Signature: TestImagePullDNSAndCredentialBranches(t *testing.T).
// Why: image-pull blockers should be classified instead of blindly recycling
// pods when DNS or registry credentials are the real outage cause.
func TestImagePullDNSAndCredentialBranches(t *testing.T) {
	eventJSON := `{"items":[
{"type":"Normal","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"skip","name":"normal"},"message":"lookup registry.bstein.dev no such host"},
{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Service","namespace":"skip","name":"svc"},"message":"lookup registry.bstein.dev no such host"},
{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"","name":""},"message":"lookup registry.bstein.dev no such host"},
{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"media","name":"app"},"message":"failed to pull image: lookup registry.bstein.dev: no such host"},
{"type":"Warning","reason":"FailedPull","metadata":{"namespace":"finance"},"involvedObject":{"kind":"Pod","name":"budget"},"message":"unauthorized: authentication required"},
{"type":"Warning","reason":"FailedToRetrieveImagePullSecret","involvedObject":{"kind":"Pod","namespace":"sso","name":"keycloak"},"message":"unable to retrieve some image pull secrets"}
]}`
	eventJSON, podsJSON := currentCredentialFixture(t, eventJSON)
	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
		{match: matchContains("kubectl", "get", "pods", "-A"), out: podsJSON},
		{match: matchContains("kubectl", "get", "events", "-A"), out: eventJSON},
	})
	dnsReasons, err := orch.imagePullDNSBlockerReasons(context.Background())
	if err != nil {
		t.Fatalf("imagePullDNSBlockerReasons: %v", err)
	}
	if dnsReasons["media/app"] != "ImagePullDNSBlocker:registry.bstein.dev" {
		t.Fatalf("unexpected DNS reasons: %#v", dnsReasons)
	}
	credentialReasons, err := orch.imagePullCredentialBlockerReasons(context.Background())
	if err != nil {
		t.Fatalf("imagePullCredentialBlockerReasons: %v", err)
	}
	if credentialReasons["finance/budget"] != "ImagePullCredentialBlocker:unauthorized" || credentialReasons["sso/keycloak"] != "ImagePullCredentialBlocker:missing-pull-secret" {
		t.Fatalf("unexpected credential reasons: %#v", credentialReasons)
	}
	for _, message := range []string{
		"invalid username/password",
		"403 forbidden",
		"pull access denied",
		"registry returned something strange",
	} {
		if got := imagePullCredentialFailureClass("Failed", message); got == "" {
			t.Fatalf("expected credential class for %q", message)
		}
	}
	if !vaultSyncDeployment("worker", map[string]string{"app": "secret-vault-sync"}) {
		t.Fatalf("expected vault-sync label value to identify sync deployment")
	}
	for _, message := range []string{"no resolver marker here", "lookup ", "lookup []"} {
		if got := imagePullRegistryHost(message); got != "unknown-registry" {
			t.Fatalf("expected unknown registry for %q, got %q", message, got)
		}
	}

	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
		{match: matchContains("kubectl", "-n", "finance", "get", "deployment"), out: `{"items":[{"metadata":{"name":"finance-vault-sync","labels":{}}},{"metadata":{"name":"app","labels":{}}}]}`},
		{match: matchContains("kubectl", "-n", "finance", "rollout", "restart", "deployment", "finance-vault-sync"), out: ""},
		{match: matchContains("kubectl", "-n", "finance", "rollout", "status", "deployment/finance-vault-sync"), out: ""},
	})
	repaired, err := orch.restartVaultSyncDeployments(context.Background(), "finance")
	if err != nil || len(repaired) != 1 || repaired[0] != "finance/deployment/finance-vault-sync" {
		t.Fatalf("unexpected vault-sync repair repaired=%v err=%v", repaired, err)
	}
	for _, tc := range []struct {
		name string
		out  string
		err  error
	}{
		{name: "query", err: errors.New("api down")},
		{name: "decode", out: "{"},
		{name: "missing", out: `{"items":[{"metadata":{"name":"app","labels":{}}}]}`},
		{name: "restart", out: `{"items":[{"metadata":{"name":"vault-sync","labels":{}}}]}`, err: errors.New("restart denied")},
	} {
		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
			{match: matchContains("kubectl", "-n", "ns", "get", "deployment"), out: tc.out, err: mapRestartErr(tc.name, "query", tc.err)},
			{match: matchContains("kubectl", "-n", "ns", "rollout", "restart"), err: mapRestartErr(tc.name, "restart", tc.err)},
		})
		if _, err := orch.restartVaultSyncDeployments(context.Background(), "ns"); err == nil {
			t.Fatalf("%s: expected restartVaultSyncDeployments error", tc.name)
		}
	}
}

// TestLonghornSelectorAndRuntimeRecoveryBranches runs one orchestration or CLI step.
// Signature: TestLonghornSelectorAndRuntimeRecoveryBranches(t *testing.T).
// Why: Longhorn manager drift and kubelet proxy recovery are safety-critical
// repair paths that should stay covered without a live cluster.
func TestLonghornSelectorAndRuntimeRecoveryBranches(t *testing.T) {
	if selectorLabelString(map[string]string{"b": "2", "a": "1"}) != "a=1,b=2" {
		t.Fatalf("selector labels should be sorted")
	}
	if podMatchesLabels(nil, map[string]string{"app": "longhorn"}) {
		t.Fatalf("nil pod labels must not match selector")
	}
	if len(missingSelectorLabels(map[string]string{"app": "old"}, map[string]string{"app": "longhorn"})) != 1 {
		t.Fatalf("expected missing selector label")
	}

	cfg := config.Config{}
	var labeled bool
	orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[
{"metadata":{"name":""}},
{"metadata":{"name":"missing-kube"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
{"metadata":{"name":"has-manager"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
{"metadata":{"name":"ready"},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ManagerPodMissing"}]}},
{"metadata":{"name":"other-reason"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"DiskPressure"}]}},
{"metadata":{"name":"already-labelled"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
{"metadata":{"name":"repair"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}
]}`},
		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true","skip":""}}}}`},
		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[
{"metadata":{"name":"has-manager","labels":{}}},
{"metadata":{"name":"ready","labels":{}}},
{"metadata":{"name":"other-reason","labels":{}}},
{"metadata":{"name":"already-labelled","labels":{"longhorn-host":"true"}}},
{"metadata":{"name":"repair","labels":{}}}
]}`},
		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json", "-l", "longhorn-host=true"), out: `{"items":[{"metadata":{"labels":{"longhorn-host":"true"}},"spec":{"nodeName":"has-manager"}},{"metadata":{"labels":{"longhorn-host":"false"}},"spec":{"nodeName":"repair"}},{"metadata":{"labels":{"longhorn-host":"true"}},"spec":{"nodeName":""}}]}`},
		{match: func(name string, args []string) bool {
			if name == "kubectl" && strings.Contains(strings.Join(args, " "), "label node repair --overwrite longhorn-host=true") {
				labeled = true
				return true
			}
			return false
		}, out: ""},
	})
	repaired, err := orch.reconcileLonghornKubernetesReadiness(context.Background())
	if err != nil || repaired != 1 || !labeled {
		t.Fatalf("unexpected longhorn reconcile repaired=%d labeled=%v err=%v", repaired, labeled, err)
	}

	errOrch := buildOrchestratorWithStubs(t, cfg, []commandStub{
		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[{"metadata":{"name":"repair"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`},
		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true"}}}}`},
		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"repair","labels":{}}}]}`},
		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json"), out: `{"items":[]}`},
		{match: matchContains("kubectl", "label", "node", "repair"), err: errors.New("label denied")},
	})
	if repaired, err := errOrch.reconcileLonghornKubernetesReadiness(context.Background()); err == nil || repaired != 0 {
		t.Fatalf("expected longhorn label error, repaired=%d err=%v", repaired, err)
	}

	runtimeCfg := config.Config{
		SSHManagedNodes: []string{"titan-23"},
		SSHUser:         "atlas",
		Startup: config.Startup{
			NodeRuntimeRestartWaitSeconds: 1,
			NodeRuntimeRebootWaitSeconds:  1,
			HostRepairAllowReboot:         true,
		},
	}
	orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
		{match: matchContains("ssh", "sudo -n systemctl --no-block restart k3s-agent"), out: ""},
		{match: matchContains("kubectl", "wait", "node/titan-23"), err: errors.New("not ready")},
		{match: matchContains("ssh", "sudo -n systemctl show k3s-agent"), out: "ActiveState=deactivating\nSubState=stop-sigterm"},
		{match: matchContains("ssh", "sudo -n systemctl reboot"), out: ""},
	})
	if err := orch.recoverManagedNodeRuntime(context.Background(), "titan-23", true, "proxy broken"); err == nil || !strings.Contains(err.Error(), "node did not recover after controlled reboot") {
		t.Fatalf("expected reboot wait failure, got %v", err)
	}
	if err := orch.recoverManagedNodeRuntime(context.Background(), " ", true, ""); err == nil {
		t.Fatalf("expected empty runtime node error")
	}
	if !k3sAgentStateRequiresReboot("Result=timeout") || k3sAgentStateRequiresReboot("ActiveState=active") {
		t.Fatalf("unexpected k3s-agent reboot predicate")
	}
	if summarizeSystemctlShow("") != "unknown" || !strings.Contains(summarizeSystemctlShow("ActiveState=activating\nSubState=start"), "ActiveState") {
		t.Fatalf("unexpected systemctl summary")
	}
}

// startTCPFixture runs one orchestration or CLI step.
// Signature: startTCPFixture(t *testing.T, banner string, reply string) string.
// Why: TCP checklist tests need a local loopback endpoint that emits the same
// banner/response shape as SMTP-like protocol probes.
func startTCPFixture(t *testing.T, banner string, reply string) string {
	t.Helper()
	listener, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatalf("listen tcp fixture: %v", err)
	}
	t.Cleanup(func() { _ = listener.Close() })
	go func() {
		for {
			conn, err := listener.Accept()
			if err != nil {
				return
			}
			go func() {
				defer conn.Close()
				_, _ = conn.Write([]byte(banner))
				_ = conn.SetDeadline(time.Now().Add(2 * time.Second))
				_, _ = bufio.NewReader(conn).ReadString('\n')
				_, _ = conn.Write([]byte(reply))
			}()
		}
	}()
	return listener.Addr().String()
}

// mapRestartErr runs one orchestration or CLI step.
// Signature: mapRestartErr(caseName, target string, err error) error.
// Why: table-driven deployment restart tests need concise error injection
// without repeating one-off closures for each command phase.
func mapRestartErr(caseName, target string, err error) error {
	if caseName == target {
		return err
	}
	return nil
}

// TestLocalOrchestratorConstructorUsesQuietLogger runs one orchestration or CLI step.
// Signature: TestLocalOrchestratorConstructorUsesQuietLogger(t *testing.T).
// Why: keep imports in this closeout file anchored to real construction helpers
// while preserving package-local test readability.
func TestLocalOrchestratorConstructorUsesQuietLogger(t *testing.T) {
	orch := &Orchestrator{
		cfg:    config.Config{},
		runner: &execx.Runner{},
		store:  state.New(t.TempDir() + "/runs.json"),
		log:    log.New(io.Discard, "", 0),
	}
	if orch.log == nil || orch.runner == nil || orch.store == nil {
		t.Fatalf("expected constructed orchestrator test fixture")
	}
}