195 lines
8.6 KiB
Go
195 lines
8.6 KiB
Go
|
package orchestrator
|
||
|
|
|
||
|
|
import (
|
||
|
|
"context"
|
||
|
|
"errors"
|
||
|
|
"io"
|
||
|
|
"log"
|
||
|
|
"os"
|
||
|
|
"path/filepath"
|
||
|
|
"strings"
|
||
|
|
"testing"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"scm.bstein.dev/bstein/ananke/internal/cluster"
|
||
|
|
"scm.bstein.dev/bstein/ananke/internal/execx"
|
||
|
|
"scm.bstein.dev/bstein/ananke/internal/state"
|
||
|
|
)
|
||
|
|
|
||
|
|
// TestHookAccessDeepMatrixReconcileAndSSHAuth runs one orchestration or CLI step.
|
||
|
|
// Signature: TestHookAccessDeepMatrixReconcileAndSSHAuth(t *testing.T).
|
||
|
|
// Why: closes remaining access-gate branch gaps for access validation and SSH
|
||
|
|
// auth wait behavior in the top-level testing module.
|
||
|
|
func TestHookAccessDeepMatrixReconcileAndSSHAuth(t *testing.T) {
|
||
|
|
t.Run("reconcile-access-aggregates-errors", func(t *testing.T) {
|
||
|
|
cfg := lifecycleConfig(t)
|
||
|
|
run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "ssh" && strings.Contains(command, "/usr/bin/systemctl --version") {
|
||
|
|
return "", errors.New("sudo denied")
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orch, _ := newHookOrchestrator(t, cfg, run, run)
|
||
|
|
err := orch.TestHookReconcileNodeAccess(context.Background(), []string{"titan-db", "titan-23"})
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "access validation had") || !strings.Contains(err.Error(), "missing sudo access") {
|
||
|
|
t.Fatalf("expected aggregated reconcile error, got %v", err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
t.Run("ssh-auth-denied-fails-fast", func(t *testing.T) {
|
||
|
|
cfg := lifecycleConfig(t)
|
||
|
|
cfg.Startup.RequireNodeSSHAuth = true
|
||
|
|
cfg.Startup.NodeSSHAuthWaitSeconds = 1
|
||
|
|
cfg.Startup.NodeSSHAuthPollSeconds = 1
|
||
|
|
run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "ssh" && strings.Contains(command, "__ANANKE_SSH_AUTH_OK__") {
|
||
|
|
return "Permission denied (publickey)", errors.New("permission denied")
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orch, _ := newHookOrchestrator(t, cfg, run, run)
|
||
|
|
err := orch.TestHookWaitForNodeSSHAuth(context.Background(), []string{"titan-db", "titan-23"})
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "ssh auth gate failed") {
|
||
|
|
t.Fatalf("expected ssh auth denied error, got %v", err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
t.Run("ssh-auth-timeout-and-context-cancel", func(t *testing.T) {
|
||
|
|
cfg := lifecycleConfig(t)
|
||
|
|
cfg.Startup.RequireNodeSSHAuth = true
|
||
|
|
cfg.Startup.NodeSSHAuthWaitSeconds = 1
|
||
|
|
cfg.Startup.NodeSSHAuthPollSeconds = 1
|
||
|
|
run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "ssh" && strings.Contains(command, "__ANANKE_SSH_AUTH_OK__") {
|
||
|
|
return "no route to host", errors.New("network unreachable")
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orch, _ := newHookOrchestrator(t, cfg, run, run)
|
||
|
|
err := orch.TestHookWaitForNodeSSHAuth(context.Background(), []string{"titan-db"})
|
||
|
|
if err == nil || !strings.Contains(err.Error(), "node ssh auth gate did not pass within") {
|
||
|
|
t.Fatalf("expected timeout branch, got %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
cancelCtx, cancel := context.WithCancel(context.Background())
|
||
|
|
cancel()
|
||
|
|
err = orch.TestHookWaitForNodeSSHAuth(cancelCtx, []string{"titan-db"})
|
||
|
|
if !errors.Is(err, context.Canceled) {
|
||
|
|
t.Fatalf("expected canceled context, got %v", err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
// TestHookAccessDeepMatrixLocalRepoAndCache runs one orchestration or CLI step.
|
||
|
|
// Signature: TestHookAccessDeepMatrixLocalRepoAndCache(t *testing.T).
|
||
|
|
// Why: covers local-repo sync and bootstrap-cache error branches that are not
|
||
|
|
// always exercised by lifecycle startup tests.
|
||
|
|
func TestHookAccessDeepMatrixLocalRepoAndCache(t *testing.T) {
|
||
|
|
t.Run("sync-local-repo-input-and-command-errors", func(t *testing.T) {
|
||
|
|
cfg := lifecycleConfig(t)
|
||
|
|
cfg.IACRepoPath = ""
|
||
|
|
orch, _ := newHookOrchestrator(t, cfg, nil, nil)
|
||
|
|
if err := orch.TestHookSyncLocalIACRepo(context.Background()); err == nil {
|
||
|
|
t.Fatalf("expected empty repo path error")
|
||
|
|
}
|
||
|
|
|
||
|
|
repo := t.TempDir()
|
||
|
|
cfg = lifecycleConfig(t)
|
||
|
|
cfg.IACRepoPath = repo
|
||
|
|
orch, _ = newHookOrchestrator(t, cfg, nil, nil)
|
||
|
|
if err := orch.TestHookSyncLocalIACRepo(context.Background()); err == nil || !strings.Contains(err.Error(), "is not a git checkout") {
|
||
|
|
t.Fatalf("expected missing .git error, got %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil {
|
||
|
|
t.Fatalf("mkdir .git: %v", err)
|
||
|
|
}
|
||
|
|
runStatusErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "git" && strings.Contains(command, "status --porcelain") {
|
||
|
|
return "", errors.New("status failed")
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orch, _ = newHookOrchestrator(t, cfg, runStatusErr, runStatusErr)
|
||
|
|
if err := orch.TestHookSyncLocalIACRepo(context.Background()); err == nil || !strings.Contains(err.Error(), "inspect iac repo working tree") {
|
||
|
|
t.Fatalf("expected status error branch, got %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
runDirty := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "git" && strings.Contains(command, "status --porcelain") {
|
||
|
|
return " M README.md", nil
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orch, _ = newHookOrchestrator(t, cfg, runDirty, runDirty)
|
||
|
|
if err := orch.TestHookSyncLocalIACRepo(context.Background()); err != nil {
|
||
|
|
t.Fatalf("expected dirty-tree skip branch success, got %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
runFetchErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
switch {
|
||
|
|
case name == "git" && strings.Contains(command, "status --porcelain"):
|
||
|
|
return "", nil
|
||
|
|
case name == "git" && strings.Contains(command, "fetch origin --prune"):
|
||
|
|
return "", errors.New("fetch failed")
|
||
|
|
default:
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
orch, _ = newHookOrchestrator(t, cfg, runFetchErr, runFetchErr)
|
||
|
|
if err := orch.TestHookSyncLocalIACRepo(context.Background()); err == nil || !strings.Contains(err.Error(), "git fetch origin") {
|
||
|
|
t.Fatalf("expected fetch error branch, got %v", err)
|
||
|
|
}
|
||
|
|
})
|
||
|
|
|
||
|
|
t.Run("bootstrap-cache-apply-and-refresh-errors", func(t *testing.T) {
|
||
|
|
cfg := lifecycleConfig(t)
|
||
|
|
cfg.LocalBootstrapPaths = []string{"services/bootstrap"}
|
||
|
|
cfg.IACRepoPath = t.TempDir()
|
||
|
|
orch, _ := newHookOrchestrator(t, cfg, nil, nil)
|
||
|
|
|
||
|
|
if err := orch.TestHookApplyBootstrapCache(context.Background(), "services/bootstrap"); err == nil || !strings.Contains(err.Error(), "bootstrap cache missing") {
|
||
|
|
t.Fatalf("expected missing cache error, got %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
stateFile := filepath.Join(t.TempDir(), "state-file")
|
||
|
|
if err := os.WriteFile(stateFile, []byte("x"), 0o644); err != nil {
|
||
|
|
t.Fatalf("write state-file: %v", err)
|
||
|
|
}
|
||
|
|
cfgFailMkdir := lifecycleConfig(t)
|
||
|
|
cfgFailMkdir.State.Dir = stateFile
|
||
|
|
cfgFailMkdir.LocalBootstrapPaths = []string{"services/bootstrap"}
|
||
|
|
cfgFailMkdir.IACRepoPath = t.TempDir()
|
||
|
|
orchFailMkdir := cluster.New(cfgFailMkdir, &execx.Runner{DryRun: false}, state.New(cfgFailMkdir.State.RunHistoryPath), log.New(io.Discard, "", 0))
|
||
|
|
orchFailMkdir.SetCommandOverrides(lifecycleDispatcher(&commandRecorder{}), lifecycleDispatcher(&commandRecorder{}))
|
||
|
|
if err := orchFailMkdir.TestHookRefreshBootstrapCache(context.Background()); err == nil || !strings.Contains(err.Error(), "ensure bootstrap cache dir") {
|
||
|
|
t.Fatalf("expected cache-dir creation failure, got %v", err)
|
||
|
|
}
|
||
|
|
|
||
|
|
runApplyErr := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
|
||
|
|
command := name + " " + strings.Join(args, " ")
|
||
|
|
if name == "kubectl" && strings.Contains(command, " apply -f ") {
|
||
|
|
return "", errors.New("apply failed")
|
||
|
|
}
|
||
|
|
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
|
||
|
|
}
|
||
|
|
orchApply, _ := newHookOrchestrator(t, cfg, runApplyErr, runApplyErr)
|
||
|
|
cachePath := orchApply.TestHookBootstrapCachePath("services/bootstrap")
|
||
|
|
if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil {
|
||
|
|
t.Fatalf("mkdir cache dir: %v", err)
|
||
|
|
}
|
||
|
|
if err := os.WriteFile(cachePath, []byte("apiVersion: v1\nkind: Namespace\nmetadata:\n name: cached\n"), 0o644); err != nil {
|
||
|
|
t.Fatalf("write cache file: %v", err)
|
||
|
|
}
|
||
|
|
if err := orchApply.TestHookApplyBootstrapCache(context.Background(), "services/bootstrap"); err == nil {
|
||
|
|
t.Fatalf("expected apply cache command failure")
|
||
|
|
}
|
||
|
|
})
|
||
|
|
}
|