443 lines
21 KiB
Go
443 lines
21 KiB
Go
package cluster
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/base64"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"scm.bstein.dev/bstein/ananke/internal/config"
|
|
"scm.bstein.dev/bstein/ananke/internal/execx"
|
|
"scm.bstein.dev/bstein/ananke/internal/state"
|
|
)
|
|
|
|
// TestTCPServiceChecklistAndBackendHealBranches runs one orchestration or CLI step.
|
|
// Signature: TestTCPServiceChecklistAndBackendHealBranches(t *testing.T).
|
|
// Why: TCP health checks drive Mailu recovery paths and need branch coverage for
|
|
// banner matching, unnamed checks, duplicate backend suppression, and heal notes.
|
|
func TestTCPServiceChecklistAndBackendHealBranches(t *testing.T) {
|
|
addr := startTCPFixture(t, "220 ready\r\n", "250 pong\r\n")
|
|
host, portText, err := net.SplitHostPort(addr)
|
|
if err != nil {
|
|
t.Fatalf("split tcp fixture address: %v", err)
|
|
}
|
|
var port int
|
|
if _, err := fmt.Sscanf(portText, "%d", &port); err != nil {
|
|
t.Fatalf("parse tcp fixture port: %v", err)
|
|
}
|
|
|
|
cfg := config.Config{
|
|
Startup: config.Startup{
|
|
TCPServiceChecklist: []config.TCPServiceChecklistCheck{
|
|
{Name: "smtp", Host: host, Port: port, Send: "PING\r\n", ExpectContains: "pong", Namespace: "mail", Service: "mailu"},
|
|
},
|
|
},
|
|
}
|
|
orch := buildOrchestratorWithStubs(t, cfg, nil)
|
|
ok, detail := orch.tcpServiceChecklistReady(context.Background())
|
|
if !ok || detail != "tcp-checks=1" {
|
|
t.Fatalf("expected tcp checklist success, ok=%v detail=%q", ok, detail)
|
|
}
|
|
|
|
unnamed := config.TCPServiceChecklistCheck{Host: host, Port: port, ExpectContains: "missing"}
|
|
ok, detail = orch.tcpServiceCheckReady(context.Background(), unnamed)
|
|
if ok || !strings.Contains(detail, `banner missing expected marker "missing"`) {
|
|
t.Fatalf("expected missing marker detail, ok=%v detail=%q", ok, detail)
|
|
}
|
|
orch.cfg.Startup.TCPServiceChecklist = []config.TCPServiceChecklistCheck{unnamed}
|
|
ok, detail = orch.tcpServiceChecklistReady(context.Background())
|
|
if ok || !strings.Contains(detail, host+":"+portText) {
|
|
t.Fatalf("expected unnamed tcp check to use host:port detail, ok=%v detail=%q", ok, detail)
|
|
}
|
|
|
|
healAddr := startTCPFixture(t, "220 ready\r\n", "250 pong\r\n")
|
|
healHost, healPortText, _ := net.SplitHostPort(healAddr)
|
|
var healPort int
|
|
_, _ = fmt.Sscanf(healPortText, "%d", &healPort)
|
|
cfg.Startup.TCPServiceChecklist = []config.TCPServiceChecklistCheck{
|
|
{Host: healHost, Port: healPort, Namespace: "", Service: "skip"},
|
|
{Name: "ok", Host: healHost, Port: healPort, ExpectContains: "ready", Namespace: "mail", Service: "mailu"},
|
|
{Name: "bad", Host: healHost, Port: healPort, ExpectContains: "absent", Namespace: "mail", Service: "mailu"},
|
|
{Name: "dup", Host: healHost, Port: healPort, ExpectContains: "absent", Namespace: "mail", Service: "mailu"},
|
|
}
|
|
var scaled, rolled bool
|
|
orch = buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{match: func(name string, args []string) bool {
|
|
joined := strings.Join(args, " ")
|
|
if name == "kubectl" && strings.Contains(joined, "-n mail scale deployment mailu --replicas=1") {
|
|
scaled = true
|
|
return true
|
|
}
|
|
if name == "kubectl" && strings.Contains(joined, "-n mail rollout status deployment/mailu") {
|
|
rolled = true
|
|
return true
|
|
}
|
|
return false
|
|
}},
|
|
{match: matchContains("kubectl", "-n", "mail", "scale", "statefulset", "mailu"), err: errors.New(`Error from server (NotFound): statefulsets.apps "mailu" not found`)},
|
|
})
|
|
healed, err := orch.healFailedTCPServiceBackends(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("healFailedTCPServiceBackends: %v", err)
|
|
}
|
|
if len(healed) != 1 || healed[0] != "mail/deployment/mailu" || !scaled || !rolled {
|
|
t.Fatalf("unexpected tcp backend heal result healed=%v scaled=%v rolled=%v", healed, scaled, rolled)
|
|
}
|
|
lastAttempt := time.Now().Add(-time.Minute)
|
|
orch.maybeAutoHealTCPServiceBackends(context.Background(), &lastAttempt)
|
|
}
|
|
|
|
// TestHostPrivilegeFailureAndSecretBranches runs one orchestration or CLI step.
|
|
// Signature: TestHostPrivilegeFailureAndSecretBranches(t *testing.T).
|
|
// Why: host repair depends on stable allowlist, secret, sudo, and failure
|
|
// classes before Ananke can safely restart or reboot managed nodes.
|
|
func TestHostPrivilegeFailureAndSecretBranches(t *testing.T) {
|
|
cfg := config.Config{
|
|
SSHManagedNodes: []string{"titan-23"},
|
|
SSHUser: "atlas",
|
|
Startup: config.Startup{
|
|
HostSudoSecretNamespace: "host-secrets",
|
|
HostSudoSecretNameTemplate: "sudo-{node}",
|
|
HostSudoSecretPasswordKey: "password",
|
|
},
|
|
}
|
|
orch := buildOrchestratorWithStubs(t, cfg, nil)
|
|
if _, err := orch.runHostPrivilegedAction(context.Background(), " ", hostActionSudoPreflight, 0); err == nil {
|
|
t.Fatalf("expected empty node privilege error")
|
|
}
|
|
if _, err := orch.runHostPrivilegedAction(context.Background(), "unmanaged", hostActionSudoPreflight, 0); err == nil {
|
|
t.Fatalf("expected unmanaged node privilege error")
|
|
}
|
|
if _, err := orch.runHostPrivilegedAction(context.Background(), "titan-23", hostPrivilegedAction("bad-action"), 0); err == nil {
|
|
t.Fatalf("expected bad action privilege error")
|
|
}
|
|
if got := orch.hostPrivilegedCommandTimeout(); got != 90*time.Second {
|
|
t.Fatalf("expected default privileged timeout, got %s", got)
|
|
}
|
|
orch.cfg.Startup.HostPrivilegedCommandTimeoutSec = 7
|
|
if got := orch.hostPrivilegedCommandTimeout(); got != 7*time.Second {
|
|
t.Fatalf("expected configured privileged timeout, got %s", got)
|
|
}
|
|
|
|
actions := []struct {
|
|
action hostPrivilegedAction
|
|
args []string
|
|
want string
|
|
}{
|
|
{hostActionHostReboot, nil, "systemctl reboot"},
|
|
{hostActionModprobeDMCrypt, nil, "modprobe dm_crypt"},
|
|
{hostActionCrictlPods, nil, "crictl pods -o json"},
|
|
{hostActionCrictlPs, nil, "crictl ps -a -o json"},
|
|
{hostActionCrictlInspect, []string{"abc123"}, "crictl inspect abc123"},
|
|
{hostActionCrictlStop, []string{"abc123"}, "crictl stop abc123"},
|
|
}
|
|
for _, tc := range actions {
|
|
got, err := hostPrivilegedCommandArgs(tc.action, tc.args...)
|
|
if err != nil {
|
|
t.Fatalf("hostPrivilegedCommandArgs(%s): %v", tc.action, err)
|
|
}
|
|
if strings.Join(got, " ") != tc.want {
|
|
t.Fatalf("hostPrivilegedCommandArgs(%s)=%q want %q", tc.action, strings.Join(got, " "), tc.want)
|
|
}
|
|
}
|
|
for _, tc := range []struct {
|
|
action hostPrivilegedAction
|
|
arg string
|
|
}{
|
|
{hostActionCrictlInspect, "bad;id"},
|
|
{hostActionCrictlStop, ""},
|
|
} {
|
|
if _, err := hostPrivilegedCommandArgs(tc.action, tc.arg); err == nil {
|
|
t.Fatalf("expected unsafe arg rejection for %s", tc.action)
|
|
}
|
|
}
|
|
|
|
errs := []error{
|
|
classifyHostPrivilegeFailure("n", hostActionSudoPreflight, context.DeadlineExceeded, ""),
|
|
classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("try again"), ""),
|
|
classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("permission denied publickey"), ""),
|
|
classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("exit 1"), "line one\nline two"),
|
|
}
|
|
wantClasses := []string{"host-command-timeout", "host-privilege-auth-failed", "host-ssh-unavailable", "host-command-failed"}
|
|
for i, err := range errs {
|
|
if !strings.Contains(err.Error(), wantClasses[i]) {
|
|
t.Fatalf("expected %s in %q", wantClasses[i], err)
|
|
}
|
|
}
|
|
|
|
secretJSON := `{"data":{"password":"` + base64.StdEncoding.EncodeToString([]byte("pw\n")) + `"}}`
|
|
orch = buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{match: matchContains("ssh", "sudo -n /usr/bin/systemctl --version"), out: "sudo: a password is required", err: errors.New("sudo failed")},
|
|
{match: matchContains("kubectl", "-n", "host-secrets", "get", "secret", "sudo-titan-23"), out: secretJSON},
|
|
})
|
|
orch.SetSSHInputOverride(func(_ context.Context, _ time.Duration, node, command, input string) (string, error) {
|
|
if node != "titan-23" || !strings.Contains(command, "sudo -S") || input != "pw\n" {
|
|
t.Fatalf("unexpected password-backed ssh node=%q command=%q input=%q", node, command, input)
|
|
}
|
|
return "incorrect password", errors.New("try again")
|
|
})
|
|
if _, err := orch.runHostPrivilegedAction(context.Background(), "titan-23", hostActionSudoPreflight, 0); err == nil || !strings.Contains(err.Error(), "host-privilege-auth-failed") {
|
|
t.Fatalf("expected password-backed auth failure, got %v", err)
|
|
}
|
|
|
|
secretCases := []struct {
|
|
name string
|
|
cfg config.Config
|
|
node string
|
|
out string
|
|
err error
|
|
class string
|
|
}{
|
|
{name: "unconfigured", cfg: config.Config{}, class: "secret-lookup-unconfigured"},
|
|
{name: "empty-name", cfg: config.Config{Startup: config.Startup{HostSudoSecretNamespace: "ns", HostSudoSecretNameTemplate: "{node}"}}, node: " ", class: "secret-lookup-invalid"},
|
|
{name: "query-error", cfg: cfg, err: errors.New("denied"), class: "secret-lookup-failed"},
|
|
{name: "bad-json", cfg: cfg, out: "{", class: "secret-decode-failed"},
|
|
{name: "missing-key", cfg: cfg, out: `{"data":{"other":"cHc="}}`, class: "secret-key-missing"},
|
|
{name: "bad-base64", cfg: cfg, out: `{"data":{"password":"%"}}`, class: "secret-decode-failed"},
|
|
{name: "empty-password", cfg: cfg, out: `{"data":{"password":"Cg=="}}`, class: "secret-empty"},
|
|
}
|
|
for _, tc := range secretCases {
|
|
orch := buildOrchestratorWithStubs(t, tc.cfg, []commandStub{
|
|
{match: matchContains("kubectl", "get", "secret"), out: tc.out, err: tc.err},
|
|
})
|
|
node := tc.node
|
|
if node == "" {
|
|
node = "titan-23"
|
|
}
|
|
_, class, err := orch.hostSudoPassword(context.Background(), node)
|
|
if err == nil || class != tc.class {
|
|
t.Fatalf("%s: expected class %q error, got class=%q err=%v", tc.name, tc.class, class, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestImagePullDNSAndCredentialBranches runs one orchestration or CLI step.
|
|
// Signature: TestImagePullDNSAndCredentialBranches(t *testing.T).
|
|
// Why: image-pull blockers should be classified instead of blindly recycling
|
|
// pods when DNS or registry credentials are the real outage cause.
|
|
func TestImagePullDNSAndCredentialBranches(t *testing.T) {
|
|
eventJSON := `{"items":[
|
|
{"type":"Normal","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"skip","name":"normal"},"message":"lookup registry.bstein.dev no such host"},
|
|
{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Service","namespace":"skip","name":"svc"},"message":"lookup registry.bstein.dev no such host"},
|
|
{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"","name":""},"message":"lookup registry.bstein.dev no such host"},
|
|
{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"media","name":"app"},"message":"failed to pull image: lookup registry.bstein.dev: no such host"},
|
|
{"type":"Warning","reason":"FailedPull","metadata":{"namespace":"finance"},"involvedObject":{"kind":"Pod","name":"budget"},"message":"unauthorized: authentication required"},
|
|
{"type":"Warning","reason":"FailedToRetrieveImagePullSecret","involvedObject":{"kind":"Pod","namespace":"sso","name":"keycloak"},"message":"unable to retrieve some image pull secrets"}
|
|
]}`
|
|
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
|
|
{match: matchContains("kubectl", "get", "events", "-A"), out: eventJSON},
|
|
})
|
|
dnsReasons, err := orch.imagePullDNSBlockerReasons(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("imagePullDNSBlockerReasons: %v", err)
|
|
}
|
|
if dnsReasons["media/app"] != "ImagePullDNSBlocker:registry.bstein.dev" {
|
|
t.Fatalf("unexpected DNS reasons: %#v", dnsReasons)
|
|
}
|
|
credentialReasons, err := orch.imagePullCredentialBlockerReasons(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("imagePullCredentialBlockerReasons: %v", err)
|
|
}
|
|
if credentialReasons["finance/budget"] != "ImagePullCredentialBlocker:unauthorized" || credentialReasons["sso/keycloak"] != "ImagePullCredentialBlocker:missing-pull-secret" {
|
|
t.Fatalf("unexpected credential reasons: %#v", credentialReasons)
|
|
}
|
|
for _, message := range []string{
|
|
"invalid username/password",
|
|
"403 forbidden",
|
|
"pull access denied",
|
|
"registry returned something strange",
|
|
} {
|
|
if got := imagePullCredentialFailureClass("Failed", message); got == "" {
|
|
t.Fatalf("expected credential class for %q", message)
|
|
}
|
|
}
|
|
if !vaultSyncDeployment("worker", map[string]string{"app": "secret-vault-sync"}) {
|
|
t.Fatalf("expected vault-sync label value to identify sync deployment")
|
|
}
|
|
for _, message := range []string{"no resolver marker here", "lookup ", "lookup []"} {
|
|
if got := imagePullRegistryHost(message); got != "unknown-registry" {
|
|
t.Fatalf("expected unknown registry for %q, got %q", message, got)
|
|
}
|
|
}
|
|
|
|
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
|
|
{match: matchContains("kubectl", "-n", "finance", "get", "deployment"), out: `{"items":[{"metadata":{"name":"finance-vault-sync","labels":{}}},{"metadata":{"name":"app","labels":{}}}]}`},
|
|
{match: matchContains("kubectl", "-n", "finance", "rollout", "restart", "deployment", "finance-vault-sync"), out: ""},
|
|
{match: matchContains("kubectl", "-n", "finance", "rollout", "status", "deployment/finance-vault-sync"), out: ""},
|
|
})
|
|
repaired, err := orch.restartVaultSyncDeployments(context.Background(), "finance")
|
|
if err != nil || len(repaired) != 1 || repaired[0] != "finance/deployment/finance-vault-sync" {
|
|
t.Fatalf("unexpected vault-sync repair repaired=%v err=%v", repaired, err)
|
|
}
|
|
for _, tc := range []struct {
|
|
name string
|
|
out string
|
|
err error
|
|
}{
|
|
{name: "query", err: errors.New("api down")},
|
|
{name: "decode", out: "{"},
|
|
{name: "missing", out: `{"items":[{"metadata":{"name":"app","labels":{}}}]}`},
|
|
{name: "restart", out: `{"items":[{"metadata":{"name":"vault-sync","labels":{}}}]}`, err: errors.New("restart denied")},
|
|
} {
|
|
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
|
|
{match: matchContains("kubectl", "-n", "ns", "get", "deployment"), out: tc.out, err: mapRestartErr(tc.name, "query", tc.err)},
|
|
{match: matchContains("kubectl", "-n", "ns", "rollout", "restart"), err: mapRestartErr(tc.name, "restart", tc.err)},
|
|
})
|
|
if _, err := orch.restartVaultSyncDeployments(context.Background(), "ns"); err == nil {
|
|
t.Fatalf("%s: expected restartVaultSyncDeployments error", tc.name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestLonghornSelectorAndRuntimeRecoveryBranches runs one orchestration or CLI step.
|
|
// Signature: TestLonghornSelectorAndRuntimeRecoveryBranches(t *testing.T).
|
|
// Why: Longhorn manager drift and kubelet proxy recovery are safety-critical
|
|
// repair paths that should stay covered without a live cluster.
|
|
func TestLonghornSelectorAndRuntimeRecoveryBranches(t *testing.T) {
|
|
if selectorLabelString(map[string]string{"b": "2", "a": "1"}) != "a=1,b=2" {
|
|
t.Fatalf("selector labels should be sorted")
|
|
}
|
|
if podMatchesLabels(nil, map[string]string{"app": "longhorn"}) {
|
|
t.Fatalf("nil pod labels must not match selector")
|
|
}
|
|
if len(missingSelectorLabels(map[string]string{"app": "old"}, map[string]string{"app": "longhorn"})) != 1 {
|
|
t.Fatalf("expected missing selector label")
|
|
}
|
|
|
|
cfg := config.Config{}
|
|
var labeled bool
|
|
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[
|
|
{"metadata":{"name":""}},
|
|
{"metadata":{"name":"missing-kube"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
|
|
{"metadata":{"name":"has-manager"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
|
|
{"metadata":{"name":"ready"},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ManagerPodMissing"}]}},
|
|
{"metadata":{"name":"other-reason"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"DiskPressure"}]}},
|
|
{"metadata":{"name":"already-labelled"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
|
|
{"metadata":{"name":"repair"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}
|
|
]}`},
|
|
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true","skip":""}}}}`},
|
|
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[
|
|
{"metadata":{"name":"has-manager","labels":{}}},
|
|
{"metadata":{"name":"ready","labels":{}}},
|
|
{"metadata":{"name":"other-reason","labels":{}}},
|
|
{"metadata":{"name":"already-labelled","labels":{"longhorn-host":"true"}}},
|
|
{"metadata":{"name":"repair","labels":{}}}
|
|
]}`},
|
|
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json", "-l", "longhorn-host=true"), out: `{"items":[{"metadata":{"labels":{"longhorn-host":"true"}},"spec":{"nodeName":"has-manager"}},{"metadata":{"labels":{"longhorn-host":"false"}},"spec":{"nodeName":"repair"}},{"metadata":{"labels":{"longhorn-host":"true"}},"spec":{"nodeName":""}}]}`},
|
|
{match: func(name string, args []string) bool {
|
|
if name == "kubectl" && strings.Contains(strings.Join(args, " "), "label node repair --overwrite longhorn-host=true") {
|
|
labeled = true
|
|
return true
|
|
}
|
|
return false
|
|
}, out: ""},
|
|
})
|
|
repaired, err := orch.reconcileLonghornKubernetesReadiness(context.Background())
|
|
if err != nil || repaired != 1 || !labeled {
|
|
t.Fatalf("unexpected longhorn reconcile repaired=%d labeled=%v err=%v", repaired, labeled, err)
|
|
}
|
|
|
|
errOrch := buildOrchestratorWithStubs(t, cfg, []commandStub{
|
|
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[{"metadata":{"name":"repair"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`},
|
|
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true"}}}}`},
|
|
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"repair","labels":{}}}]}`},
|
|
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json"), out: `{"items":[]}`},
|
|
{match: matchContains("kubectl", "label", "node", "repair"), err: errors.New("label denied")},
|
|
})
|
|
if repaired, err := errOrch.reconcileLonghornKubernetesReadiness(context.Background()); err == nil || repaired != 0 {
|
|
t.Fatalf("expected longhorn label error, repaired=%d err=%v", repaired, err)
|
|
}
|
|
|
|
runtimeCfg := config.Config{
|
|
SSHManagedNodes: []string{"titan-23"},
|
|
SSHUser: "atlas",
|
|
Startup: config.Startup{
|
|
NodeRuntimeRestartWaitSeconds: 1,
|
|
NodeRuntimeRebootWaitSeconds: 1,
|
|
HostRepairAllowReboot: true,
|
|
},
|
|
}
|
|
orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
|
|
{match: matchContains("ssh", "sudo -n systemctl --no-block restart k3s-agent"), out: ""},
|
|
{match: matchContains("kubectl", "wait", "node/titan-23"), err: errors.New("not ready")},
|
|
{match: matchContains("ssh", "sudo -n systemctl show k3s-agent"), out: "ActiveState=deactivating\nSubState=stop-sigterm"},
|
|
{match: matchContains("ssh", "sudo -n systemctl reboot"), out: ""},
|
|
})
|
|
if err := orch.recoverManagedNodeRuntime(context.Background(), "titan-23", true, "proxy broken"); err == nil || !strings.Contains(err.Error(), "node did not recover after controlled reboot") {
|
|
t.Fatalf("expected reboot wait failure, got %v", err)
|
|
}
|
|
if err := orch.recoverManagedNodeRuntime(context.Background(), " ", true, ""); err == nil {
|
|
t.Fatalf("expected empty runtime node error")
|
|
}
|
|
if !k3sAgentStateRequiresReboot("Result=timeout") || k3sAgentStateRequiresReboot("ActiveState=active") {
|
|
t.Fatalf("unexpected k3s-agent reboot predicate")
|
|
}
|
|
if summarizeSystemctlShow("") != "unknown" || !strings.Contains(summarizeSystemctlShow("ActiveState=activating\nSubState=start"), "ActiveState") {
|
|
t.Fatalf("unexpected systemctl summary")
|
|
}
|
|
}
|
|
|
|
// startTCPFixture runs one orchestration or CLI step.
|
|
// Signature: startTCPFixture(t *testing.T, banner string, reply string) string.
|
|
// Why: TCP checklist tests need a local loopback endpoint that emits the same
|
|
// banner/response shape as SMTP-like protocol probes.
|
|
func startTCPFixture(t *testing.T, banner string, reply string) string {
|
|
t.Helper()
|
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
|
if err != nil {
|
|
t.Fatalf("listen tcp fixture: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = listener.Close() })
|
|
go func() {
|
|
for {
|
|
conn, err := listener.Accept()
|
|
if err != nil {
|
|
return
|
|
}
|
|
go func() {
|
|
defer conn.Close()
|
|
_, _ = conn.Write([]byte(banner))
|
|
_ = conn.SetDeadline(time.Now().Add(2 * time.Second))
|
|
_, _ = bufio.NewReader(conn).ReadString('\n')
|
|
_, _ = conn.Write([]byte(reply))
|
|
}()
|
|
}
|
|
}()
|
|
return listener.Addr().String()
|
|
}
|
|
|
|
// mapRestartErr runs one orchestration or CLI step.
|
|
// Signature: mapRestartErr(caseName, target string, err error) error.
|
|
// Why: table-driven deployment restart tests need concise error injection
|
|
// without repeating one-off closures for each command phase.
|
|
func mapRestartErr(caseName, target string, err error) error {
|
|
if caseName == target {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// TestLocalOrchestratorConstructorUsesQuietLogger runs one orchestration or CLI step.
|
|
// Signature: TestLocalOrchestratorConstructorUsesQuietLogger(t *testing.T).
|
|
// Why: keep imports in this closeout file anchored to real construction helpers
|
|
// while preserving package-local test readability.
|
|
func TestLocalOrchestratorConstructorUsesQuietLogger(t *testing.T) {
|
|
orch := &Orchestrator{
|
|
cfg: config.Config{},
|
|
runner: &execx.Runner{},
|
|
store: state.New(t.TempDir() + "/runs.json"),
|
|
log: log.New(io.Discard, "", 0),
|
|
}
|
|
if orch.log == nil || orch.runner == nil || orch.store == nil {
|
|
t.Fatalf("expected constructed orchestrator test fixture")
|
|
}
|
|
}
|