bstein/ananke
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support terraform-owned bootstrap inputs

Browse Source
This commit is contained in:
codex 2026-07-16 02:27:44 -03:00
parent cecd82cff3
commit 76876ef895
29 changed files with 2699 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@ -50,6 +50,14 @@ Host files:
- `/var/lib/ananke/last-shutdown-report.json`
- `/var/log/ananke/update.log`
## Terraform-Generated Inventory
Ananke can load a generated inventory fragment from `/etc/ananke/ananke.inventory.yaml` alongside `/etc/ananke/ananke.yaml`. The installer also accepts `ANANKE_INVENTORY_FRAGMENT=/path/to/ananke.inventory.yaml` and copies it into place.
That fragment is for non-secret declarative inputs only: node hosts, managed nodes, control planes, workers, required node labels, and ignored unavailable nodes. When Terraform owns node labels, the fragment should set `startup.required_node_labels_mode: validate`; Ananke will report label drift but will not apply `kubectl label` during normal startup or post-start auto-heal.
Keep recovery behavior in Ananke: UPS shutdown, SSH repair, k3s restart/reboot, Longhorn/runtime recovery, cordon/uncordon, pod recycling, and Flux suspend/resume remain imperative Ananke actions.
## Development
Local testing check before installing:

46
cmd/ananke/builder.go
View File

@ -2,6 +2,9 @@ package main
import (
"log"
"os"
"path/filepath"
"strings"
"scm.bstein.dev/bstein/ananke/internal/cluster"
"scm.bstein.dev/bstein/ananke/internal/config"
@ -13,7 +16,7 @@ import (
// Signature: buildOrchestrator(logger *log.Logger, cfgPath string, dryRun bool) (config.Config, *cluster.Orchestrator, error).
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
func buildOrchestrator(logger *log.Logger, cfgPath string, dryRun bool) (config.Config, *cluster.Orchestrator, error) {
cfg, err := config.Load(cfgPath)
cfg, err := config.LoadWithFragments(cfgPath, configFragmentPaths(cfgPath)...)
if err != nil {
return config.Config{}, nil, err
}
@ -32,3 +35,44 @@ func buildOrchestrator(logger *log.Logger, cfgPath string, dryRun bool) (config.
orch := cluster.New(cfg, runner, store, logger)
return cfg, orch, nil
}
// configFragmentPaths runs one orchestration or CLI step.
// Signature: configFragmentPaths(cfgPath string) []string.
// Why: keeps generated inventory overlays discoverable without changing every
// systemd command line or recovery script invocation.
func configFragmentPaths(cfgPath string) []string {
paths := []string{}
if env := strings.TrimSpace(os.Getenv("ANANKE_CONFIG_FRAGMENTS")); env != "" {
for _, path := range filepath.SplitList(env) {
path = strings.TrimSpace(path)
if path != "" {
paths = append(paths, path)
}
}
}
defaultPath := defaultInventoryFragmentPath(cfgPath)
if defaultPath != "" {
if _, err := os.Stat(defaultPath); err == nil {
paths = append(paths, defaultPath)
}
}
return paths
}
// defaultInventoryFragmentPath runs one orchestration or CLI step.
// Signature: defaultInventoryFragmentPath(cfgPath string) string.
// Why: pairs ananke.yaml with an adjacent Terraform-generated inventory
// fragment using a deterministic host-local convention.
func defaultInventoryFragmentPath(cfgPath string) string {
dir := filepath.Dir(cfgPath)
base := filepath.Base(cfgPath)
ext := filepath.Ext(base)
name := strings.TrimSuffix(base, ext)
if name == "" {
return ""
}
if ext == "" {
return filepath.Join(dir, name+".inventory.yaml")
}
return filepath.Join(dir, name+".inventory"+ext)
}

107
cmd/ananke/builder_test.go
View File

@ -33,6 +33,57 @@ func TestBuildOrchestratorCreatesStateDirs(t *testing.T) {
}
}
// TestBuildOrchestratorLoadsAdjacentInventoryFragment runs one orchestration or CLI step.
// Signature: TestBuildOrchestratorLoadsAdjacentInventoryFragment(t *testing.T).
// Why: the systemd default path must consume Terraform-generated inventory
// without requiring extra command-line flags.
func TestBuildOrchestratorLoadsAdjacentInventoryFragment(t *testing.T) {
cfgPath := writeTestConfig(t)
fragmentPath := filepath.Join(filepath.Dir(cfgPath), "ananke.inventory.yaml")
raw := `
ssh_node_hosts:
titan-0a: 192.168.22.11
titan-05: 192.168.22.31
ssh_managed_nodes: [titan-0a, titan-05]
control_planes: [titan-0a]
workers: [titan-05]
startup:
required_node_labels_mode: validate
required_node_labels: {}
ignore_unavailable_nodes: []
`
if err := os.WriteFile(fragmentPath, []byte(raw), 0o644); err != nil {
t.Fatalf("write inventory fragment: %v", err)
}
cfg, orch, err := buildOrchestrator(log.New(io.Discard, "", 0), cfgPath, true)
if err != nil {
t.Fatalf("buildOrchestrator failed: %v", err)
}
if orch == nil {
t.Fatalf("expected orchestrator")
}
if len(cfg.ControlPlanes) != 1 || cfg.ControlPlanes[0] != "titan-0a" {
t.Fatalf("expected adjacent fragment control plane, got %v", cfg.ControlPlanes)
}
if len(cfg.Workers) != 1 || cfg.Workers[0] != "titan-05" {
t.Fatalf("expected adjacent fragment workers, got %v", cfg.Workers)
}
if cfg.Startup.RequiredNodeLabelsMode != "validate" {
t.Fatalf("expected adjacent fragment label mode validate, got %q", cfg.Startup.RequiredNodeLabelsMode)
}
}
// TestBuildOrchestratorPropagatesLoadError runs one orchestration or CLI step.
// Signature: TestBuildOrchestratorPropagatesLoadError(t *testing.T).
// Why: missing host configs must fail before any recovery runner is built.
func TestBuildOrchestratorPropagatesLoadError(t *testing.T) {
missing := filepath.Join(t.TempDir(), "missing.yaml")
if _, orch, err := buildOrchestrator(log.New(io.Discard, "", 0), missing, true); err == nil || orch != nil {
t.Fatalf("expected missing config to fail without orchestrator, orch=%v err=%v", orch, err)
}
}
// TestBuildOrchestratorFailsWhenStateDirIsFile runs one orchestration or CLI step.
// Signature: TestBuildOrchestratorFailsWhenStateDirIsFile(t *testing.T).
// Why: covers EnsureDir error path in orchestrator builder.
@ -116,3 +167,59 @@ state:
t.Fatalf("expected buildOrchestrator failure when reports dir is a file")
}
}
// TestConfigFragmentPathsUsesEnvAndAdjacentInventory runs one orchestration or CLI step.
// Signature: TestConfigFragmentPathsUsesEnvAndAdjacentInventory(t *testing.T).
// Why: host installs should be able to consume Terraform-generated inventory
// without changing the systemd command line.
func TestConfigFragmentPathsUsesEnvAndAdjacentInventory(t *testing.T) {
tmp := t.TempDir()
cfgPath := filepath.Join(tmp, "ananke.yaml")
defaultFragment := filepath.Join(tmp, "ananke.inventory.yaml")
envFragment := filepath.Join(tmp, "env-fragment.yaml")
if err := os.WriteFile(defaultFragment, []byte("ssh_managed_nodes: []\n"), 0o644); err != nil {
t.Fatalf("write default fragment: %v", err)
}
t.Setenv("ANANKE_CONFIG_FRAGMENTS", envFragment)
paths := configFragmentPaths(cfgPath)
if len(paths) != 2 {
t.Fatalf("expected env and adjacent fragments, got %v", paths)
}
if paths[0] != envFragment || paths[1] != defaultFragment {
t.Fatalf("unexpected fragment order: %v", paths)
}
}
// TestConfigFragmentPathsSkipsMissingAndBlankEntries runs one orchestration or CLI step.
// Signature: TestConfigFragmentPathsSkipsMissingAndBlankEntries(t *testing.T).
// Why: optional generated fragments should be discovered deterministically without
// turning an absent adjacent file into a startup blocker.
func TestConfigFragmentPathsSkipsMissingAndBlankEntries(t *testing.T) {
tmp := t.TempDir()
envFragment := filepath.Join(tmp, "env.yaml")
t.Setenv("ANANKE_CONFIG_FRAGMENTS", string(os.PathListSeparator)+envFragment+string(os.PathListSeparator)+" ")
paths := configFragmentPaths(filepath.Join(tmp, "ananke.yaml"))
if len(paths) != 1 || paths[0] != envFragment {
t.Fatalf("expected only env fragment, got %v", paths)
}
}
// TestDefaultInventoryFragmentPathVariants runs one orchestration or CLI step.
// Signature: TestDefaultInventoryFragmentPathVariants(t *testing.T).
// Why: generated inventory naming should stay stable for config paths with and
// without a YAML extension.
func TestDefaultInventoryFragmentPathVariants(t *testing.T) {
tmp := t.TempDir()
cases := map[string]string{
filepath.Join(tmp, "ananke.yaml"): filepath.Join(tmp, "ananke.inventory.yaml"),
filepath.Join(tmp, "ananke"): filepath.Join(tmp, "ananke.inventory.yaml"),
".": "",
}
for in, want := range cases {
if got := defaultInventoryFragmentPath(in); got != want {
t.Fatalf("defaultInventoryFragmentPath(%q)=%q want %q", in, got, want)
}
}
}

30
cmd/ananke/command_handlers_autoheal_test.go
View File

@ -2,8 +2,10 @@ package main
import (
"context"
"errors"
"io"
"log"
"strings"
"testing"
"scm.bstein.dev/bstein/ananke/internal/cluster"
@ -38,3 +40,31 @@ func TestRunAutoHealInvokesPostStartRepair(t *testing.T) {
t.Fatalf("expected auto-heal orchestrator hook to be called")
}
}
// TestDefaultAutoHealWrapperHonorsDryRunOrchestrator runs one orchestration or CLI step.
// Signature: TestDefaultAutoHealWrapperHonorsDryRunOrchestrator(t *testing.T).
// Why: the default command hook should keep pointing at the orchestrator method,
// not only at test-injected replacements.
func TestDefaultAutoHealWrapperHonorsDryRunOrchestrator(t *testing.T) {
cfg := minimalHandlerConfig(t)
orch := newTestOrchestrator(cfg, true)
if err := autoHealOrchestratorCommand(context.Background(), orch); err != nil {
t.Fatalf("default auto-heal wrapper failed: %v", err)
}
}
// TestRunAutoHealBuildError runs one orchestration or CLI step.
// Signature: TestRunAutoHealBuildError(t *testing.T).
// Why: command handlers should return config/build failures before invoking
// repair actions.
func TestRunAutoHealBuildError(t *testing.T) {
restore := stubCommandHandlerHooks()
defer restore()
buildOrchestratorCommand = func(_ *log.Logger, _ string, _ bool) (config.Config, *cluster.Orchestrator, error) {
return config.Config{}, nil, errors.New("build failed")
}
err := runAutoHeal(log.New(io.Discard, "", 0), []string{"--config", "/bad.yaml"})
if err == nil || !strings.Contains(err.Error(), "build failed") {
t.Fatalf("expected build failure, got %v", err)
}
}

1
configs/ananke.example.yaml
View File

@ -56,6 +56,7 @@ startup:
required_node_labels:
titan-09:
ananke.bstein.dev/harbor-bootstrap: "true"
required_node_labels_mode: enforce
require_time_sync: true
time_sync_wait_seconds: 240
time_sync_poll_seconds: 5

1
configs/ananke.tethys.yaml
View File

@ -166,6 +166,7 @@ startup:
titan-09:
node-role.kubernetes.io/worker: "true"
ananke.bstein.dev/harbor-bootstrap: "true"
required_node_labels_mode: enforce
require_time_sync: true
time_sync_wait_seconds: 240
time_sync_poll_seconds: 5

1
configs/ananke.titan-db.yaml
View File

@ -166,6 +166,7 @@ startup:
titan-09:
node-role.kubernetes.io/worker: "true"
ananke.bstein.dev/harbor-bootstrap: "true"
required_node_labels_mode: enforce
require_time_sync: true
time_sync_wait_seconds: 240
time_sync_poll_seconds: 5

88
internal/cluster/orchestrator_autorepair.go
View File

@ -205,65 +205,6 @@ func (o *Orchestrator) rerunVaultK8sAuthConfigJob(ctx context.Context) error {
return nil
}
// cleanupTerminatingPodsOnUnavailableNodes runs one orchestration or CLI step.
// Signature: (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error).
// Why: dead nodes can strand terminating pods indefinitely, so the daemon should
// clear only that narrow failure class instead of leaving garbage behind forever.
func (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error) {
if o.runner.DryRun {
return 0, nil
}
unavailable, err := o.unavailableNodeSet(ctx)
if err != nil {
return 0, err
}
if len(unavailable) == 0 {
return 0, nil
}
out, err := o.kubectl(ctx, 30*time.Second, "get", "pods", "-A", "-o", "json")
if err != nil {
return 0, fmt.Errorf("query pods: %w", err)
}
var pods podDeleteList
if err := json.Unmarshal([]byte(out), &pods); err != nil {
return 0, fmt.Errorf("decode pods: %w", err)
}
grace := time.Duration(o.cfg.Startup.DeadNodeCleanupGraceSeconds) * time.Second
now := time.Now()
count := 0
for _, item := range pods.Items {
if item.Metadata.DeletionTimestamp == nil || item.Spec.NodeName == "" {
continue
}
if _, badNode := unavailable[item.Spec.NodeName]; !badNode {
continue
}
if now.Sub(*item.Metadata.DeletionTimestamp) < grace {
continue
}
o.log.Printf("warning: force deleting terminating pod %s/%s on unavailable node %s", item.Metadata.Namespace, item.Metadata.Name, item.Spec.NodeName)
if _, err := o.kubectl(
ctx,
20*time.Second,
"-n", item.Metadata.Namespace,
"delete", "pod", item.Metadata.Name,
"--grace-period=0",
"--force",
"--wait=false",
); err != nil && !isNotFoundErr(err) {
return count, fmt.Errorf("delete pod %s/%s: %w", item.Metadata.Namespace, item.Metadata.Name, err)
}
count++
}
if count > 0 {
o.log.Printf("post-start auto-heal cleaned %d terminating pod(s) from unavailable nodes", count)
}
return count, nil
}
// repairBrokenKubeletProxies runs one orchestration or CLI step.
// Signature: (o *Orchestrator) repairBrokenKubeletProxies(ctx context.Context) (int, error).
// Why: a Ready node can still have a dead kubelet tunnel, which breaks Jenkins
@ -422,35 +363,6 @@ func isRepairableKubeletProxyErr(err error) bool {
return false
}
// unavailableNodeSet runs one orchestration or CLI step.
// Signature: (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error).
// Why: isolates Ready-condition parsing so dead-node cleanup stays targeted.
func (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error) {
out, err := o.kubectl(ctx, 20*time.Second, "get", "nodes", "-o", "json")
if err != nil {
return nil, fmt.Errorf("query nodes: %w", err)
}
var nodes nodeReadyList
if err := json.Unmarshal([]byte(out), &nodes); err != nil {
return nil, fmt.Errorf("decode nodes: %w", err)
}
unavailable := map[string]struct{}{}
for _, item := range nodes.Items {
ready := ""
for _, cond := range item.Status.Conditions {
if strings.EqualFold(strings.TrimSpace(cond.Type), "Ready") {
ready = strings.TrimSpace(cond.Status)
break
}
}
if ready != "True" {
unavailable[item.Metadata.Name] = struct{}{}
}
}
return unavailable, nil
}
// requestFluxReconcile runs one orchestration or CLI step.
// Signature: (o *Orchestrator) requestFluxReconcile(ctx context.Context) error.
// Why: post-start repairs need a lightweight way to refresh GitOps health

97
internal/cluster/orchestrator_autorepair_cleanup.go Normal file
View File

@ -0,0 +1,97 @@
package cluster
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
)
// cleanupTerminatingPodsOnUnavailableNodes runs one orchestration or CLI step.
// Signature: (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error).
// Why: dead nodes can strand terminating pods indefinitely, so the daemon should
// clear only that narrow failure class instead of leaving garbage behind forever.
func (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error) {
if o.runner.DryRun {
return 0, nil
}
unavailable, err := o.unavailableNodeSet(ctx)
if err != nil {
return 0, err
}
if len(unavailable) == 0 {
return 0, nil
}
out, err := o.kubectl(ctx, 30*time.Second, "get", "pods", "-A", "-o", "json")
if err != nil {
return 0, fmt.Errorf("query pods: %w", err)
}
var pods podDeleteList
if err := json.Unmarshal([]byte(out), &pods); err != nil {
return 0, fmt.Errorf("decode pods: %w", err)
}
grace := time.Duration(o.cfg.Startup.DeadNodeCleanupGraceSeconds) * time.Second
now := time.Now()
count := 0
for _, item := range pods.Items {
if item.Metadata.DeletionTimestamp == nil || item.Spec.NodeName == "" {
continue
}
if _, badNode := unavailable[item.Spec.NodeName]; !badNode {
continue
}
if now.Sub(*item.Metadata.DeletionTimestamp) < grace {
continue
}
o.log.Printf("warning: force deleting terminating pod %s/%s on unavailable node %s", item.Metadata.Namespace, item.Metadata.Name, item.Spec.NodeName)
if _, err := o.kubectl(
ctx,
20*time.Second,
"-n", item.Metadata.Namespace,
"delete", "pod", item.Metadata.Name,
"--grace-period=0",
"--force",
"--wait=false",
); err != nil && !isNotFoundErr(err) {
return count, fmt.Errorf("delete pod %s/%s: %w", item.Metadata.Namespace, item.Metadata.Name, err)
}
count++
}
if count > 0 {
o.log.Printf("post-start auto-heal cleaned %d terminating pod(s) from unavailable nodes", count)
}
return count, nil
}
// unavailableNodeSet runs one orchestration or CLI step.
// Signature: (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error).
// Why: isolates Ready-condition parsing so dead-node cleanup stays targeted.
func (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error) {
out, err := o.kubectl(ctx, 20*time.Second, "get", "nodes", "-o", "json")
if err != nil {
return nil, fmt.Errorf("query nodes: %w", err)
}
var nodes nodeReadyList
if err := json.Unmarshal([]byte(out), &nodes); err != nil {
return nil, fmt.Errorf("decode nodes: %w", err)
}
unavailable := map[string]struct{}{}
for _, item := range nodes.Items {
ready := ""
for _, cond := range item.Status.Conditions {
if strings.EqualFold(strings.TrimSpace(cond.Type), "Ready") {
ready = strings.TrimSpace(cond.Status)
break
}
}
if ready != "True" {
unavailable[item.Metadata.Name] = struct{}{}
}
}
return unavailable, nil
}

5
internal/cluster/orchestrator_drain.go
View File

@ -164,7 +164,12 @@ func (o *Orchestrator) ensureLonghornEncryptedHostPrereqs(ctx context.Context, w
exempt := makeStringSet(o.cfg.Startup.LonghornCryptsetupExemptNodes)
unsafe := map[string]struct{}{}
var errs []string
nodes := make([]string, 0, len(longhornHosts))
for node := range longhornHosts {
nodes = append(nodes, node)
}
sort.Strings(nodes)
for _, node := range nodes {
if _, skip := ignored[node]; skip {
continue
}

80
internal/cluster/orchestrator_ingress.go
View File

@ -19,6 +19,9 @@ func (o *Orchestrator) ensureRequiredNodeLabels(ctx context.Context) error {
if o.runner.DryRun || len(o.cfg.Startup.RequiredNodeLabels) == 0 {
return nil
}
if o.cfg.Startup.RequiredNodeLabelsMode == "validate" {
return o.validateRequiredNodeLabels(ctx)
}
ignored := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
nodes := make([]string, 0, len(o.cfg.Startup.RequiredNodeLabels))
for node := range o.cfg.Startup.RequiredNodeLabels {
@ -72,6 +75,83 @@ func (o *Orchestrator) ensureRequiredNodeLabels(ctx context.Context) error {
return nil
}
type nodeLabelItem struct {
Metadata struct {
Labels map[string]string `json:"labels"`
} `json:"metadata"`
}
// validateRequiredNodeLabels runs one orchestration or CLI step.
// Signature: (o *Orchestrator) validateRequiredNodeLabels(ctx context.Context) error.
// Why: lets Terraform own steady-state labels while Ananke still reports drift
// during startup and post-start health checks.
func (o *Orchestrator) validateRequiredNodeLabels(ctx context.Context) error {
ignored := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
nodes := make([]string, 0, len(o.cfg.Startup.RequiredNodeLabels))
for node := range o.cfg.Startup.RequiredNodeLabels {
node = strings.TrimSpace(node)
if node != "" {
nodes = append(nodes, node)
}
}
sort.Strings(nodes)
drift := []string{}
for _, node := range nodes {
if _, skip := ignored[node]; skip {
o.log.Printf("skipping required node label validation for ignored unavailable node %s", node)
continue
}
required := o.cfg.Startup.RequiredNodeLabels[node]
if len(required) == 0 {
continue
}
keys := make([]string, 0, len(required))
for key := range required {
key = strings.TrimSpace(key)
if key != "" {
keys = append(keys, key)
}
}
sort.Strings(keys)
if len(keys) == 0 {
continue
}
out, err := o.kubectl(ctx, 25*time.Second, "get", "node", node, "-o", "json")
if err != nil {
if isNotFoundErr(err) && !o.startupNodeStrictlyRequired(node) {
o.log.Printf("warning: skipping required label validation for absent non-core node %s: %v", node, err)
o.noteStartupAutoHeal(fmt.Sprintf("skipped required node label validation for absent non-core node %s", node))
continue
}
return fmt.Errorf("validate required node labels on %s: %w", node, err)
}
var item nodeLabelItem
if err := json.Unmarshal([]byte(out), &item); err != nil {
return fmt.Errorf("decode node labels for %s: %w", node, err)
}
for _, key := range keys {
want := strings.TrimSpace(required[key])
if want == "" {
continue
}
got := strings.TrimSpace(item.Metadata.Labels[key])
if got != want {
if got == "" {
got = "<missing>"
}
drift = append(drift, fmt.Sprintf("%s %s want=%s got=%s", node, key, want, got))
}
}
}
if len(drift) > 0 {
return fmt.Errorf("required node label drift: %s", strings.Join(drift, "; "))
}
o.log.Printf("validated required node labels on %d node(s)", len(nodes))
return nil
}
// waitForStartupConvergence runs one orchestration or CLI step.
// Signature: (o *Orchestrator) waitForStartupConvergence(ctx context.Context) error.
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

4
internal/cluster/orchestrator_lifecycle.go
View File

@ -11,6 +11,8 @@ import (
"scm.bstein.dev/bstein/ananke/internal/state"
)
var etcdRestorePostStartDelay = 10 * time.Second
// Startup runs one orchestration or CLI step.
// Signature: (o *Orchestrator) Startup(ctx context.Context, opts StartupOptions) (err error).
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
@ -433,7 +435,7 @@ func (o *Orchestrator) EtcdRestore(ctx context.Context, opts EtcdRestoreOptions)
if _, err := o.ssh(ctx, controlPlane, "sudo systemctl start k3s || true"); err != nil {
return fmt.Errorf("failed to start k3s on restore control plane %s: %w", controlPlane, err)
}
time.Sleep(10 * time.Second)
time.Sleep(etcdRestorePostStartDelay)
for _, cp := range o.cfg.ControlPlanes {
cp := cp
if cp == controlPlane {

442
internal/cluster/orchestrator_quality_gate_closeout_test.go Normal file
View File

@ -0,0 +1,442 @@
package cluster
import (
"bufio"
"context"
"encoding/base64"
"errors"
"fmt"
"io"
"log"
"net"
"strings"
"testing"
"time"
"scm.bstein.dev/bstein/ananke/internal/config"
"scm.bstein.dev/bstein/ananke/internal/execx"
"scm.bstein.dev/bstein/ananke/internal/state"
)
// TestTCPServiceChecklistAndBackendHealBranches runs one orchestration or CLI step.
// Signature: TestTCPServiceChecklistAndBackendHealBranches(t *testing.T).
// Why: TCP health checks drive Mailu recovery paths and need branch coverage for
// banner matching, unnamed checks, duplicate backend suppression, and heal notes.
func TestTCPServiceChecklistAndBackendHealBranches(t *testing.T) {
addr := startTCPFixture(t, "220 ready\r\n", "250 pong\r\n")
host, portText, err := net.SplitHostPort(addr)
if err != nil {
t.Fatalf("split tcp fixture address: %v", err)
}
var port int
if _, err := fmt.Sscanf(portText, "%d", &port); err != nil {
t.Fatalf("parse tcp fixture port: %v", err)
}
cfg := config.Config{
Startup: config.Startup{
TCPServiceChecklist: []config.TCPServiceChecklistCheck{
{Name: "smtp", Host: host, Port: port, Send: "PING\r\n", ExpectContains: "pong", Namespace: "mail", Service: "mailu"},
},
},
}
orch := buildOrchestratorWithStubs(t, cfg, nil)
ok, detail := orch.tcpServiceChecklistReady(context.Background())
if !ok || detail != "tcp-checks=1" {
t.Fatalf("expected tcp checklist success, ok=%v detail=%q", ok, detail)
}
unnamed := config.TCPServiceChecklistCheck{Host: host, Port: port, ExpectContains: "missing"}
ok, detail = orch.tcpServiceCheckReady(context.Background(), unnamed)
if ok || !strings.Contains(detail, `banner missing expected marker "missing"`) {
t.Fatalf("expected missing marker detail, ok=%v detail=%q", ok, detail)
}
orch.cfg.Startup.TCPServiceChecklist = []config.TCPServiceChecklistCheck{unnamed}
ok, detail = orch.tcpServiceChecklistReady(context.Background())
if ok || !strings.Contains(detail, host+":"+portText) {
t.Fatalf("expected unnamed tcp check to use host:port detail, ok=%v detail=%q", ok, detail)
}
healAddr := startTCPFixture(t, "220 ready\r\n", "250 pong\r\n")
healHost, healPortText, _ := net.SplitHostPort(healAddr)
var healPort int
_, _ = fmt.Sscanf(healPortText, "%d", &healPort)
cfg.Startup.TCPServiceChecklist = []config.TCPServiceChecklistCheck{
{Host: healHost, Port: healPort, Namespace: "", Service: "skip"},
{Name: "ok", Host: healHost, Port: healPort, ExpectContains: "ready", Namespace: "mail", Service: "mailu"},
{Name: "bad", Host: healHost, Port: healPort, ExpectContains: "absent", Namespace: "mail", Service: "mailu"},
{Name: "dup", Host: healHost, Port: healPort, ExpectContains: "absent", Namespace: "mail", Service: "mailu"},
}
var scaled, rolled bool
orch = buildOrchestratorWithStubs(t, cfg, []commandStub{
{match: func(name string, args []string) bool {
joined := strings.Join(args, " ")
if name == "kubectl" && strings.Contains(joined, "-n mail scale deployment mailu --replicas=1") {
scaled = true
return true
}
if name == "kubectl" && strings.Contains(joined, "-n mail rollout status deployment/mailu") {
rolled = true
return true
}
return false
}},
{match: matchContains("kubectl", "-n", "mail", "scale", "statefulset", "mailu"), err: errors.New(`Error from server (NotFound): statefulsets.apps "mailu" not found`)},
})
healed, err := orch.healFailedTCPServiceBackends(context.Background())
if err != nil {
t.Fatalf("healFailedTCPServiceBackends: %v", err)
}
if len(healed) != 1 || healed[0] != "mail/deployment/mailu" || !scaled || !rolled {
t.Fatalf("unexpected tcp backend heal result healed=%v scaled=%v rolled=%v", healed, scaled, rolled)
}
lastAttempt := time.Now().Add(-time.Minute)
orch.maybeAutoHealTCPServiceBackends(context.Background(), &lastAttempt)
}
// TestHostPrivilegeFailureAndSecretBranches runs one orchestration or CLI step.
// Signature: TestHostPrivilegeFailureAndSecretBranches(t *testing.T).
// Why: host repair depends on stable allowlist, secret, sudo, and failure
// classes before Ananke can safely restart or reboot managed nodes.
func TestHostPrivilegeFailureAndSecretBranches(t *testing.T) {
cfg := config.Config{
SSHManagedNodes: []string{"titan-23"},
SSHUser: "atlas",
Startup: config.Startup{
HostSudoSecretNamespace: "host-secrets",
HostSudoSecretNameTemplate: "sudo-{node}",
HostSudoSecretPasswordKey: "password",
},
}
orch := buildOrchestratorWithStubs(t, cfg, nil)
if _, err := orch.runHostPrivilegedAction(context.Background(), " ", hostActionSudoPreflight, 0); err == nil {
t.Fatalf("expected empty node privilege error")
}
if _, err := orch.runHostPrivilegedAction(context.Background(), "unmanaged", hostActionSudoPreflight, 0); err == nil {
t.Fatalf("expected unmanaged node privilege error")
}
if _, err := orch.runHostPrivilegedAction(context.Background(), "titan-23", hostPrivilegedAction("bad-action"), 0); err == nil {
t.Fatalf("expected bad action privilege error")
}
if got := orch.hostPrivilegedCommandTimeout(); got != 90*time.Second {
t.Fatalf("expected default privileged timeout, got %s", got)
}
orch.cfg.Startup.HostPrivilegedCommandTimeoutSec = 7
if got := orch.hostPrivilegedCommandTimeout(); got != 7*time.Second {
t.Fatalf("expected configured privileged timeout, got %s", got)
}
actions := []struct {
action hostPrivilegedAction
args []string
want string
}{
{hostActionHostReboot, nil, "systemctl reboot"},
{hostActionModprobeDMCrypt, nil, "modprobe dm_crypt"},
{hostActionCrictlPods, nil, "crictl pods -o json"},
{hostActionCrictlPs, nil, "crictl ps -a -o json"},
{hostActionCrictlInspect, []string{"abc123"}, "crictl inspect abc123"},
{hostActionCrictlStop, []string{"abc123"}, "crictl stop abc123"},
}
for _, tc := range actions {
got, err := hostPrivilegedCommandArgs(tc.action, tc.args...)
if err != nil {
t.Fatalf("hostPrivilegedCommandArgs(%s): %v", tc.action, err)
}
if strings.Join(got, " ") != tc.want {
t.Fatalf("hostPrivilegedCommandArgs(%s)=%q want %q", tc.action, strings.Join(got, " "), tc.want)
}
}
for _, tc := range []struct {
action hostPrivilegedAction
arg string
}{
{hostActionCrictlInspect, "bad;id"},
{hostActionCrictlStop, ""},
} {
if _, err := hostPrivilegedCommandArgs(tc.action, tc.arg); err == nil {
t.Fatalf("expected unsafe arg rejection for %s", tc.action)
}
}
errs := []error{
classifyHostPrivilegeFailure("n", hostActionSudoPreflight, context.DeadlineExceeded, ""),
classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("try again"), ""),
classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("permission denied publickey"), ""),
classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("exit 1"), "line one\nline two"),
}
wantClasses := []string{"host-command-timeout", "host-privilege-auth-failed", "host-ssh-unavailable", "host-command-failed"}
for i, err := range errs {
if !strings.Contains(err.Error(), wantClasses[i]) {
t.Fatalf("expected %s in %q", wantClasses[i], err)
}
}
secretJSON := `{"data":{"password":"` + base64.StdEncoding.EncodeToString([]byte("pw\n")) + `"}}`
orch = buildOrchestratorWithStubs(t, cfg, []commandStub{
{match: matchContains("ssh", "sudo -n /usr/bin/systemctl --version"), out: "sudo: a password is required", err: errors.New("sudo failed")},
{match: matchContains("kubectl", "-n", "host-secrets", "get", "secret", "sudo-titan-23"), out: secretJSON},
})
orch.SetSSHInputOverride(func(_ context.Context, _ time.Duration, node, command, input string) (string, error) {
if node != "titan-23" || !strings.Contains(command, "sudo -S") || input != "pw\n" {
t.Fatalf("unexpected password-backed ssh node=%q command=%q input=%q", node, command, input)
}
return "incorrect password", errors.New("try again")
})
if _, err := orch.runHostPrivilegedAction(context.Background(), "titan-23", hostActionSudoPreflight, 0); err == nil || !strings.Contains(err.Error(), "host-privilege-auth-failed") {
t.Fatalf("expected password-backed auth failure, got %v", err)
}
secretCases := []struct {
name string
cfg config.Config
node string
out string
err error
class string
}{
{name: "unconfigured", cfg: config.Config{}, class: "secret-lookup-unconfigured"},
{name: "empty-name", cfg: config.Config{Startup: config.Startup{HostSudoSecretNamespace: "ns", HostSudoSecretNameTemplate: "{node}"}}, node: " ", class: "secret-lookup-invalid"},
{name: "query-error", cfg: cfg, err: errors.New("denied"), class: "secret-lookup-failed"},
{name: "bad-json", cfg: cfg, out: "{", class: "secret-decode-failed"},
{name: "missing-key", cfg: cfg, out: `{"data":{"other":"cHc="}}`, class: "secret-key-missing"},
{name: "bad-base64", cfg: cfg, out: `{"data":{"password":"%"}}`, class: "secret-decode-failed"},
{name: "empty-password", cfg: cfg, out: `{"data":{"password":"Cg=="}}`, class: "secret-empty"},
}
for _, tc := range secretCases {
orch := buildOrchestratorWithStubs(t, tc.cfg, []commandStub{
{match: matchContains("kubectl", "get", "secret"), out: tc.out, err: tc.err},
})
node := tc.node
if node == "" {
node = "titan-23"
}
_, class, err := orch.hostSudoPassword(context.Background(), node)
if err == nil || class != tc.class {
t.Fatalf("%s: expected class %q error, got class=%q err=%v", tc.name, tc.class, class, err)
}
}
}
// TestImagePullDNSAndCredentialBranches runs one orchestration or CLI step.
// Signature: TestImagePullDNSAndCredentialBranches(t *testing.T).
// Why: image-pull blockers should be classified instead of blindly recycling
// pods when DNS or registry credentials are the real outage cause.
func TestImagePullDNSAndCredentialBranches(t *testing.T) {
eventJSON := `{"items":[
{"type":"Normal","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"skip","name":"normal"},"message":"lookup registry.bstein.dev no such host"},
{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Service","namespace":"skip","name":"svc"},"message":"lookup registry.bstein.dev no such host"},
{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"","name":""},"message":"lookup registry.bstein.dev no such host"},
{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"media","name":"app"},"message":"failed to pull image: lookup registry.bstein.dev: no such host"},
{"type":"Warning","reason":"FailedPull","metadata":{"namespace":"finance"},"involvedObject":{"kind":"Pod","name":"budget"},"message":"unauthorized: authentication required"},
{"type":"Warning","reason":"FailedToRetrieveImagePullSecret","involvedObject":{"kind":"Pod","namespace":"sso","name":"keycloak"},"message":"unable to retrieve some image pull secrets"}
]}`
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), out: eventJSON},
})
dnsReasons, err := orch.imagePullDNSBlockerReasons(context.Background())
if err != nil {
t.Fatalf("imagePullDNSBlockerReasons: %v", err)
}
if dnsReasons["media/app"] != "ImagePullDNSBlocker:registry.bstein.dev" {
t.Fatalf("unexpected DNS reasons: %#v", dnsReasons)
}
credentialReasons, err := orch.imagePullCredentialBlockerReasons(context.Background())
if err != nil {
t.Fatalf("imagePullCredentialBlockerReasons: %v", err)
}
if credentialReasons["finance/budget"] != "ImagePullCredentialBlocker:unauthorized" || credentialReasons["sso/keycloak"] != "ImagePullCredentialBlocker:missing-pull-secret" {
t.Fatalf("unexpected credential reasons: %#v", credentialReasons)
}
for _, message := range []string{
"invalid username/password",
"403 forbidden",
"pull access denied",
"registry returned something strange",
} {
if got := imagePullCredentialFailureClass("Failed", message); got == "" {
t.Fatalf("expected credential class for %q", message)
}
}
if !vaultSyncDeployment("worker", map[string]string{"app": "secret-vault-sync"}) {
t.Fatalf("expected vault-sync label value to identify sync deployment")
}
for _, message := range []string{"no resolver marker here", "lookup ", "lookup []"} {
if got := imagePullRegistryHost(message); got != "unknown-registry" {
t.Fatalf("expected unknown registry for %q, got %q", message, got)
}
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "finance", "get", "deployment"), out: `{"items":[{"metadata":{"name":"finance-vault-sync","labels":{}}},{"metadata":{"name":"app","labels":{}}}]}`},
{match: matchContains("kubectl", "-n", "finance", "rollout", "restart", "deployment", "finance-vault-sync"), out: ""},
{match: matchContains("kubectl", "-n", "finance", "rollout", "status", "deployment/finance-vault-sync"), out: ""},
})
repaired, err := orch.restartVaultSyncDeployments(context.Background(), "finance")
if err != nil || len(repaired) != 1 || repaired[0] != "finance/deployment/finance-vault-sync" {
t.Fatalf("unexpected vault-sync repair repaired=%v err=%v", repaired, err)
}
for _, tc := range []struct {
name string
out string
err error
}{
{name: "query", err: errors.New("api down")},
{name: "decode", out: "{"},
{name: "missing", out: `{"items":[{"metadata":{"name":"app","labels":{}}}]}`},
{name: "restart", out: `{"items":[{"metadata":{"name":"vault-sync","labels":{}}}]}`, err: errors.New("restart denied")},
} {
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "ns", "get", "deployment"), out: tc.out, err: mapRestartErr(tc.name, "query", tc.err)},
{match: matchContains("kubectl", "-n", "ns", "rollout", "restart"), err: mapRestartErr(tc.name, "restart", tc.err)},
})
if _, err := orch.restartVaultSyncDeployments(context.Background(), "ns"); err == nil {
t.Fatalf("%s: expected restartVaultSyncDeployments error", tc.name)
}
}
}
// TestLonghornSelectorAndRuntimeRecoveryBranches runs one orchestration or CLI step.
// Signature: TestLonghornSelectorAndRuntimeRecoveryBranches(t *testing.T).
// Why: Longhorn manager drift and kubelet proxy recovery are safety-critical
// repair paths that should stay covered without a live cluster.
func TestLonghornSelectorAndRuntimeRecoveryBranches(t *testing.T) {
if selectorLabelString(map[string]string{"b": "2", "a": "1"}) != "a=1,b=2" {
t.Fatalf("selector labels should be sorted")
}
if podMatchesLabels(nil, map[string]string{"app": "longhorn"}) {
t.Fatalf("nil pod labels must not match selector")
}
if len(missingSelectorLabels(map[string]string{"app": "old"}, map[string]string{"app": "longhorn"})) != 1 {
t.Fatalf("expected missing selector label")
}
cfg := config.Config{}
var labeled bool
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[
{"metadata":{"name":""}},
{"metadata":{"name":"missing-kube"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
{"metadata":{"name":"has-manager"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
{"metadata":{"name":"ready"},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ManagerPodMissing"}]}},
{"metadata":{"name":"other-reason"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"DiskPressure"}]}},
{"metadata":{"name":"already-labelled"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
{"metadata":{"name":"repair"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}
]}`},
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true","skip":""}}}}`},
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[
{"metadata":{"name":"has-manager","labels":{}}},
{"metadata":{"name":"ready","labels":{}}},
{"metadata":{"name":"other-reason","labels":{}}},
{"metadata":{"name":"already-labelled","labels":{"longhorn-host":"true"}}},
{"metadata":{"name":"repair","labels":{}}}
]}`},
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json", "-l", "longhorn-host=true"), out: `{"items":[{"metadata":{"labels":{"longhorn-host":"true"}},"spec":{"nodeName":"has-manager"}},{"metadata":{"labels":{"longhorn-host":"false"}},"spec":{"nodeName":"repair"}},{"metadata":{"labels":{"longhorn-host":"true"}},"spec":{"nodeName":""}}]}`},
{match: func(name string, args []string) bool {
if name == "kubectl" && strings.Contains(strings.Join(args, " "), "label node repair --overwrite longhorn-host=true") {
labeled = true
return true
}
return false
}, out: ""},
})
repaired, err := orch.reconcileLonghornKubernetesReadiness(context.Background())
if err != nil || repaired != 1 || !labeled {
t.Fatalf("unexpected longhorn reconcile repaired=%d labeled=%v err=%v", repaired, labeled, err)
}
errOrch := buildOrchestratorWithStubs(t, cfg, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[{"metadata":{"name":"repair"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`},
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true"}}}}`},
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"repair","labels":{}}}]}`},
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json"), out: `{"items":[]}`},
{match: matchContains("kubectl", "label", "node", "repair"), err: errors.New("label denied")},
})
if repaired, err := errOrch.reconcileLonghornKubernetesReadiness(context.Background()); err == nil || repaired != 0 {
t.Fatalf("expected longhorn label error, repaired=%d err=%v", repaired, err)
}
runtimeCfg := config.Config{
SSHManagedNodes: []string{"titan-23"},
SSHUser: "atlas",
Startup: config.Startup{
NodeRuntimeRestartWaitSeconds: 1,
NodeRuntimeRebootWaitSeconds: 1,
HostRepairAllowReboot: true,
},
}
orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
{match: matchContains("ssh", "sudo -n systemctl --no-block restart k3s-agent"), out: ""},
{match: matchContains("kubectl", "wait", "node/titan-23"), err: errors.New("not ready")},
{match: matchContains("ssh", "sudo -n systemctl show k3s-agent"), out: "ActiveState=deactivating\nSubState=stop-sigterm"},
{match: matchContains("ssh", "sudo -n systemctl reboot"), out: ""},
})
if err := orch.recoverManagedNodeRuntime(context.Background(), "titan-23", true, "proxy broken"); err == nil || !strings.Contains(err.Error(), "node did not recover after controlled reboot") {
t.Fatalf("expected reboot wait failure, got %v", err)
}
if err := orch.recoverManagedNodeRuntime(context.Background(), " ", true, ""); err == nil {
t.Fatalf("expected empty runtime node error")
}
if !k3sAgentStateRequiresReboot("Result=timeout") || k3sAgentStateRequiresReboot("ActiveState=active") {
t.Fatalf("unexpected k3s-agent reboot predicate")
}
if summarizeSystemctlShow("") != "unknown" || !strings.Contains(summarizeSystemctlShow("ActiveState=activating\nSubState=start"), "ActiveState") {
t.Fatalf("unexpected systemctl summary")
}
}
// startTCPFixture runs one orchestration or CLI step.
// Signature: startTCPFixture(t *testing.T, banner string, reply string) string.
// Why: TCP checklist tests need a local loopback endpoint that emits the same
// banner/response shape as SMTP-like protocol probes.
func startTCPFixture(t *testing.T, banner string, reply string) string {
t.Helper()
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen tcp fixture: %v", err)
}
t.Cleanup(func() { _ = listener.Close() })
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
go func() {
defer conn.Close()
_, _ = conn.Write([]byte(banner))
_ = conn.SetDeadline(time.Now().Add(2 * time.Second))
_, _ = bufio.NewReader(conn).ReadString('\n')
_, _ = conn.Write([]byte(reply))
}()
}
}()
return listener.Addr().String()
}
// mapRestartErr runs one orchestration or CLI step.
// Signature: mapRestartErr(caseName, target string, err error) error.
// Why: table-driven deployment restart tests need concise error injection
// without repeating one-off closures for each command phase.
func mapRestartErr(caseName, target string, err error) error {
if caseName == target {
return err
}
return nil
}
// TestLocalOrchestratorConstructorUsesQuietLogger runs one orchestration or CLI step.
// Signature: TestLocalOrchestratorConstructorUsesQuietLogger(t *testing.T).
// Why: keep imports in this closeout file anchored to real construction helpers
// while preserving package-local test readability.
func TestLocalOrchestratorConstructorUsesQuietLogger(t *testing.T) {
orch := &Orchestrator{
cfg: config.Config{},
runner: &execx.Runner{},
store: state.New(t.TempDir() + "/runs.json"),
log: log.New(io.Discard, "", 0),
}
if orch.log == nil || orch.runner == nil || orch.store == nil {
t.Fatalf("expected constructed orchestrator test fixture")
}
}

212
internal/cluster/orchestrator_quality_gate_final_closeout_test.go Normal file
View File

@ -0,0 +1,212 @@
package cluster
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"scm.bstein.dev/bstein/ananke/internal/config"
)
// TestFinalCoverageLeaseDrainAndSSHBranches runs one orchestration or CLI step.
// Signature: TestFinalCoverageLeaseDrainAndSSHBranches(t *testing.T).
// Why: close the last local-only safety branches for cordon release, Longhorn
// host preflight errors, and stdin-backed SSH known-host retry.
func TestFinalCoverageLeaseDrainAndSSHBranches(t *testing.T) {
ctx := context.Background()
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), err: errors.New("api down")},
})
if _, err := orch.ensureLonghornEncryptedHostPrereqs(ctx, []string{"worker-a"}); err == nil || !strings.Contains(err.Error(), "api down") {
t.Fatalf("expected longhorn host query error, got %v", err)
}
future := time.Now().UTC().Add(time.Hour).Format(time.RFC3339)
nodesJSON := fmt.Sprintf(`{"items":[
{"metadata":{"name":"clear-fails","annotations":{%q:"ananke"}},"spec":{"unschedulable":false}},
{"metadata":{"name":"crypt-pending","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}}
]}`,
anankeCordonOwnerAnnotation,
anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonMissingCryptsetup, anankeCordonDeadlineAnnotation, future)
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: nodesJSON},
{match: matchContains("kubectl", "annotate", "node", "clear-fails"), err: errors.New("clear denied")},
{match: matchContains("ssh", "crypt-pending", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")},
})
if released, err := orch.enforceRecoveryCordonLeases(ctx); released != 0 || err == nil || !strings.Contains(err.Error(), "clear denied") {
t.Fatalf("expected clear-cordon error and pending repair log, released=%d err=%v", released, err)
}
releaseErrOrch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("ssh", "crypt-release", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_PRESENT__"},
{match: matchContains("kubectl", "get", "pods", "-A"), out: `{"items":[]}`},
{match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`},
{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/proxy-release/proxy/healthz"), out: "ok"},
{match: matchContains("kubectl", "uncordon"), err: errors.New("uncordon denied")},
})
for _, tc := range []struct {
node string
reason string
}{
{"crypt-release", cordonReasonMissingCryptsetup},
{"runtime-release", cordonReasonRuntimeWedge},
{"proxy-release", cordonReasonKubeletProxy},
} {
if recovered, err := releaseErrOrch.recoverLeasedCordon(ctx, tc.node, map[string]string{anankeCordonReasonAnnotation: tc.reason}); recovered || err == nil || !strings.Contains(err.Error(), "uncordon denied") {
t.Fatalf("expected %s uncordon denial, recovered=%v err=%v", tc.reason, recovered, err)
}
}
binDir := t.TempDir()
countPath := filepath.Join(t.TempDir(), "ssh-count")
writeTestExecutable(t, filepath.Join(binDir, "ssh"), `#!/bin/sh
count="${ANANKE_FAKE_SSH_COUNT:?}"
n=0
if [ -f "$count" ]; then n="$(cat "$count")"; fi
n=$((n + 1))
printf '%s\n' "$n" > "$count"
if [ "$n" -eq 1 ]; then
printf '%s\n' 'Host key verification failed.' >&2
exit 255
fi
cat >/dev/null
printf '%s\n' 'stdin-ok'
`)
writeTestExecutable(t, filepath.Join(binDir, "ssh-keygen"), "#!/bin/sh\nexit 0\n")
t.Setenv("ANANKE_FAKE_SSH_COUNT", countPath)
t.Setenv("HOME", t.TempDir())
t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
sshDir := t.TempDir()
orch = buildOrchestratorWithStubs(t, config.Config{
SSHPort: 2222,
SSHUser: "atlas",
SSHConfigFile: filepath.Join(sshDir, "config"),
SSHIdentityFile: filepath.Join(sshDir, "id_ed25519"),
SSHNodeHosts: map[string]string{"node-a": "node-a.local"},
}, nil)
out, err := orch.sshWithInput(ctx, "node-a", "sudo -S true", "pw\n", time.Second)
if err != nil || out != "stdin-ok" {
t.Fatalf("expected known-host retry success, out=%q err=%v", out, err)
}
if raw, err := os.ReadFile(countPath); err != nil || strings.TrimSpace(string(raw)) != "2" {
t.Fatalf("expected exactly two fake ssh attempts, count=%q err=%v", strings.TrimSpace(string(raw)), err)
}
}
// TestPostStartAutoHealRequestsReconcileAfterImagePullRepair runs one orchestration or CLI step.
// Signature: TestPostStartAutoHealRequestsReconcileAfterImagePullRepair(t *testing.T).
// Why: a successful image-pull credential repair should request a Flux reconcile
// without requiring any real Kubernetes or Flux command.
func TestPostStartAutoHealRequestsReconcileAfterImagePullRepair(t *testing.T) {
ctx := context.Background()
eventsJSON := `{"items":[{"type":"Warning","reason":"FailedPull","message":"unauthorized: authentication required","involvedObject":{"kind":"Pod","namespace":"apps","name":"web"}}]}`
deploymentsJSON := `{"items":[{"metadata":{"name":"vault-sync","labels":{"app":"vault-sync"}}}]}`
sawFluxSourceReconcile := false
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[]}`},
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[]}`},
{match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"},
{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: eventsJSON},
{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "-o", "json"), out: deploymentsJSON},
{match: matchContains("kubectl", "-n", "apps", "rollout", "restart", "deployment", "vault-sync"), out: ""},
{match: matchContains("kubectl", "-n", "apps", "rollout", "status", "deployment/vault-sync"), out: ""},
{
match: func(name string, args []string) bool {
if !matchContains("kubectl", "-n", "flux-system", "annotate", "gitrepository", "flux-system")(name, args) {
return false
}
sawFluxSourceReconcile = true
return true
},
},
{match: matchContains("kubectl", "annotate"), out: ""},
})
if err := orch.postStartAutoHeal(ctx); err != nil {
t.Fatalf("expected post-start image-pull repair success, got %v", err)
}
if !sawFluxSourceReconcile {
t.Fatalf("expected post-start auto-heal to request flux source reconcile")
}
}
// TestStartupQuarantinesRuntimeWedgeBeforeSSHAuthGate runs one orchestration or CLI step.
// Signature: TestStartupQuarantinesRuntimeWedgeBeforeSSHAuthGate(t *testing.T).
// Why: startup should quarantine proven runtime-wedged workers before later SSH
// gates so bad schedulers are removed from the active worker set.
func TestStartupQuarantinesRuntimeWedgeBeforeSSHAuthGate(t *testing.T) {
ctx := context.Background()
tmpDir := t.TempDir()
old := time.Now().Add(-10 * time.Minute).Format(time.RFC3339)
now := time.Now().Format(time.RFC3339)
podsJSON := fmt.Sprintf(`{"items":[
{"metadata":{"namespace":"apps","name":"wedged-a","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"worker-bad"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CreateContainerError"}}}]}},
{"metadata":{"namespace":"apps","name":"wedged-b","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"worker-bad"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"RunContainerError"}}}]}}
]}`, old, old)
eventsJSON := fmt.Sprintf(`{"items":[
{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"wedged-a"},"message":"context deadline exceeded while creating container"},
{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"wedged-b"},"message":"failed to reserve container name"}
]}`, now, now)
cfg := config.Config{
Workers: []string{"worker-good", "worker-bad"},
SSHManagedNodes: []string{"worker-good", "worker-bad"},
SSHNodeHosts: map[string]string{
"worker-good": "worker-good.local",
"worker-bad": "worker-bad.local",
},
SSHUser: "atlas",
SSHPort: 22,
Startup: config.Startup{
APIWaitSeconds: 1,
APIPollSeconds: 1,
RequireTimeSync: false,
RequireNodeInventoryReach: false,
RequireNodeSSHAuth: true,
ReconcileAccessOnBoot: false,
AutoEtcdRestoreOnAPIFailure: false,
StuckPodGraceSeconds: 1,
},
State: config.State{
Dir: tmpDir,
ReportsDir: filepath.Join(tmpDir, "reports"),
RunHistoryPath: filepath.Join(tmpDir, "runs.json"),
LockPath: filepath.Join(tmpDir, "ananke.lock"),
IntentPath: filepath.Join(tmpDir, "intent.json"),
},
}
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
{match: matchContains("kubectl", "version", "--request-timeout=5s"), out: "ok"},
{match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"},
{match: matchContains("kubectl", "-n", "flux-system", "get", "gitrepository", "flux-system", "-o", "jsonpath={.spec.url}"), out: ""},
{match: matchContains("kubectl", "-n", "flux-system", "get", "gitrepository", "flux-system", "-o", "jsonpath={.spec.ref.branch}"), out: ""},
{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: podsJSON},
{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: eventsJSON},
{match: matchContains("kubectl", "annotate", "node", "worker-bad"), out: ""},
{match: matchContains("kubectl", "cordon", "worker-bad"), out: ""},
{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: ""},
{match: matchContains("ssh", "worker-good", "echo __ANANKE_SSH_AUTH_OK__"), err: errors.New("permission denied publickey")},
})
err := orch.Startup(ctx, StartupOptions{Reason: "runtime quarantine coverage"})
if err == nil || !strings.Contains(err.Error(), "ssh auth gate failed") {
t.Fatalf("expected controlled ssh auth gate failure after quarantine, got %v", err)
}
if !containsNode(orch.cfg.Startup.IgnoreUnavailableNodes, "worker-bad") {
t.Fatalf("expected runtime-wedged worker to be added to ignore_unavailable_nodes, got %v", orch.cfg.Startup.IgnoreUnavailableNodes)
}
}
// writeTestExecutable writes a temporary command fixture for tests.
// Signature: writeTestExecutable(t *testing.T, path string, body string).
// Why: fake ssh/ssh-keygen binaries let retry paths run without touching host
// SSH state or invoking the real cluster operator tooling.
func writeTestExecutable(t *testing.T, path string, body string) {
t.Helper()
if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
t.Fatalf("write test executable %s: %v", path, err)
}
}

480
internal/cluster/orchestrator_quality_gate_more_closeout_test.go Normal file
View File

@ -0,0 +1,480 @@
package cluster
import (
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"scm.bstein.dev/bstein/ananke/internal/config"
)
// TestCordonLeaseEnforcementRecoveryCloseoutBranches runs one orchestration or CLI step.
// Signature: TestCordonLeaseEnforcementRecoveryCloseoutBranches(t *testing.T).
// Why: lease enforcement must release recovered Ananke cordons while escalating
// stale or unknown cordons without overriding operator-owned state.
func TestCordonLeaseEnforcementRecoveryCloseoutBranches(t *testing.T) {
now := time.Now().UTC()
future := now.Add(time.Hour).Format(time.RFC3339)
past := now.Add(-time.Hour).Format(time.RFC3339)
oldTaint := now.Add(-2 * time.Hour).Format(time.RFC3339)
nodesJSON := fmt.Sprintf(`{"items":[
{"metadata":{"name":"","annotations":{}}},
{"metadata":{"name":"uncordoned","annotations":{%q:"ananke"}},"spec":{"unschedulable":false}},
{"metadata":{"name":"manual","annotations":{}},"spec":{"unschedulable":true,"taints":[{"key":"node.kubernetes.io/unschedulable","timeAdded":%q}]}},
{"metadata":{"name":"crypt","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}},
{"metadata":{"name":"runtime","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}},
{"metadata":{"name":"proxy","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}},
{"metadata":{"name":"unknown","annotations":{%q:"ananke",%q:"mystery",%q:%q}},"spec":{"unschedulable":true}}
]}`,
anankeCordonOwnerAnnotation,
oldTaint,
anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonMissingCryptsetup, anankeCordonDeadlineAnnotation, future,
anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonRuntimeWedge, anankeCordonDeadlineAnnotation, future,
anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonKubeletProxy, anankeCordonDeadlineAnnotation, future,
anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, anankeCordonDeadlineAnnotation, past)
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: nodesJSON},
{match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_PRESENT__"},
{match: matchContains("kubectl", "get", "pods", "-A"), out: `{"items":[]}`},
{match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`},
{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/proxy/proxy/healthz"), out: "ok"},
{match: matchContains("kubectl", "uncordon"), out: ""},
{match: matchContains("kubectl", "annotate", "node"), out: ""},
})
released, err := orch.enforceRecoveryCordonLeases(context.Background())
if err == nil || released != 3 {
t.Fatalf("expected three recovered cordons plus manual-action error, released=%d err=%v", released, err)
}
if !strings.Contains(err.Error(), "manual action required") || !strings.Contains(err.Error(), "unknown") {
t.Fatalf("expected manual action details, got %v", err)
}
}
// TestCriticalEndpointServiceHealCloseoutBranches runs one orchestration or CLI step.
// Signature: TestCriticalEndpointServiceHealCloseoutBranches(t *testing.T).
// Why: configured service backend repair should distinguish healthy endpoints,
// missing endpoints, backend scale failures, and startup-probe repair success.
func TestCriticalEndpointServiceHealCloseoutBranches(t *testing.T) {
workloadJSON := `{"spec":{"template":{"spec":{"containers":[{"name":"app","livenessProbe":{"httpGet":{"path":"/healthz"}}}]}}}}`
cfg := config.Config{Startup: config.Startup{
CriticalServiceEndpoints: []string{"bad-entry", "ready/svc", "missing/svc", "zero/svc", "probe/svc", "error/svc"},
CriticalServiceStartupProbeRepair: true,
}}
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
{match: matchContains("kubectl", "-n", "ready", "get", "endpoints", "svc"), out: `{"subsets":[{"addresses":[{"ip":"10.0.0.10"}]}]}`},
{match: matchContains("kubectl", "-n", "missing", "get", "endpoints", "svc"), err: errors.New(`Error from server (NotFound): endpoints "svc" not found`)},
{match: matchContains("kubectl", "-n", "zero", "get", "endpoints", "svc"), out: `{"subsets":[]}`},
{match: matchContains("kubectl", "-n", "probe", "get", "endpoints", "svc"), out: `{"subsets":[]}`},
{match: matchContains("kubectl", "-n", "error", "get", "endpoints", "svc"), err: errors.New("api down")},
{match: matchContains("kubectl", "-n", "zero", "scale", "deployment", "svc"), out: ""},
{match: matchContains("kubectl", "-n", "zero", "rollout", "status", "deployment/svc"), out: ""},
{match: matchContains("kubectl", "-n", "zero", "scale", "statefulset", "svc"), err: errors.New(`Error from server (NotFound): statefulsets.apps "svc" not found`)},
{match: matchContains("kubectl", "-n", "missing", "scale"), err: errors.New(`Error from server (NotFound): deployments.apps "svc" not found`)},
{match: matchContains("kubectl", "-n", "probe", "scale", "deployment", "svc"), err: errors.New("scale denied")},
{match: matchContains("kubectl", "-n", "probe", "get", "deployment", "svc"), out: workloadJSON},
{match: matchContains("kubectl", "-n", "probe", "patch", "deployment", "svc"), out: ""},
{match: matchContains("kubectl", "-n", "probe", "get", "statefulset", "svc"), err: errors.New(`Error from server (NotFound): statefulsets.apps "svc" not found`)},
})
healed, err := orch.healUnreadyConfiguredServiceBackends(context.Background())
if err == nil {
t.Fatalf("expected aggregated critical endpoint errors")
}
joined := strings.Join(healed, ",")
if !strings.Contains(joined, "zero/deployment/svc") || !strings.Contains(joined, "probe/deployment/svc") {
t.Fatalf("expected zero and probe repairs, healed=%v", healed)
}
if !strings.Contains(err.Error(), "bad-entry") || !strings.Contains(err.Error(), "error/svc") {
t.Fatalf("expected invalid and query errors, got %v", err)
}
if _, _, err := parseCriticalServiceEndpoint(" /svc"); err == nil {
t.Fatalf("expected empty namespace endpoint parse error")
}
}
// TestLonghornAndStaleRWOAdditionalCloseoutBranches runs one orchestration or CLI step.
// Signature: TestLonghornAndStaleRWOAdditionalCloseoutBranches(t *testing.T).
// Why: Longhorn parser errors and stale RWO helper predicates protect storage
// repair from false positives and should be covered without live objects.
func TestLonghornAndStaleRWOAdditionalCloseoutBranches(t *testing.T) {
ctx := context.Background()
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), err: errors.New(`Error from server (NotFound): nodes.longhorn.io not found`)},
})
if nodes, err := orch.queryLonghornNodes(ctx); err != nil || len(nodes.Items) != 0 {
t.Fatalf("expected missing Longhorn CRD to be empty, nodes=%v err=%v", nodes, err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: "{"},
})
if _, err := orch.queryLonghornNodes(ctx); err == nil {
t.Fatalf("expected longhorn node decode error")
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: "{"},
})
if _, err := orch.queryLonghornManagerDaemonSet(ctx); err == nil {
t.Fatalf("expected daemonset decode error")
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json"), out: "{"},
})
if _, err := orch.longhornManagerPodsByNode(ctx, map[string]string{"app": "longhorn"}); err == nil {
t.Fatalf("expected manager pod decode error")
}
if cond := longhornConditionByType(longhornNode{}, "Ready"); cond != nil {
t.Fatalf("expected missing condition to return nil")
}
checkCalls := 0
orch = buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"titan-23"}}, nil)
orch.runOverride = func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
joined := strings.Join(args, " ")
if name == "ssh" && strings.Contains(joined, "command -v cryptsetup") {
checkCalls++
if checkCalls == 1 {
return "__ANANKE_CRYPTSETUP_MISSING__", errors.New("exit 41")
}
return "__ANANKE_CRYPTSETUP_PRESENT__", nil
}
if name == "ssh" && strings.Contains(joined, "apt-get update") {
return "installed but no marker", nil
}
return "", nil
}
orch.runSensitiveOverride = orch.runOverride
if err := orch.ensureHostCryptsetup(ctx, "titan-23"); err != nil {
t.Fatalf("expected cryptsetup verify success after install, got %v", err)
}
if checkCalls < 2 {
t.Fatalf("expected install verification check, calls=%d", checkCalls)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "pvc", "-A"), out: "{"},
})
if _, err := orch.queryPVCs(ctx); err == nil {
t.Fatalf("expected pvc decode error")
}
pvc := persistentVolumeClaimResource{}
pvc.Metadata.Namespace = "apps"
pvc.Metadata.Name = "data"
pvc.Spec.AccessModes = []string{"ReadWriteOncePod"}
pod := closeoutPendingControllerPod("apps", "app", "node", "ReplicaSet", "ContainerCreating", time.Now().Add(-time.Hour))
if names := podRWOPVCVolumeNames(pod, map[string]persistentVolumeClaimResource{"apps/data": pvc}); names["data"] != "data" {
t.Fatalf("expected RWO pod volume mapping, got %v", names)
}
initPod := podResource{}
initPod.Status.InitContainerStatuses = []podContainerStatus{{Name: "init", State: podContainerState{Waiting: &podContainerWaitingState{Reason: "PodInitializing"}}}}
if !replacementPodPendingForAttach(initPod) {
t.Fatalf("expected init waiting pod to count as attach-pending")
}
event := eventList{Items: []eventResource{{Type: "Warning", Reason: "SomethingElse", Message: "volume is already used by pod"}}}
event.Items[0].InvolvedObject.Kind = "Pod"
event.Items[0].InvolvedObject.Namespace = "apps"
event.Items[0].InvolvedObject.Name = "app"
pod.Metadata.Namespace = "apps"
pod.Metadata.Name = "app"
if !podHasBlockedAttachEvent(pod, event) {
t.Fatalf("expected blocked attach message to match")
}
if podsShareAnyClaim(podResource{}, pod, map[string]string{"vol": "other"}) {
t.Fatalf("unrelated claims should not match")
}
unsafePod := podResource{}
unsafePod.Status.ContainerStatuses = []podContainerStatus{{Name: "ghost", State: podContainerState{Running: &podContainerRunningState{StartedAt: time.Now()}}}}
if !runningContainersMountAnyVolume(unsafePod, map[string]string{"data": "data"}) {
t.Fatalf("unknown running container should be treated unsafe")
}
}
// TestTCPDrainAndPostStartAdditionalCloseoutBranches runs one orchestration or CLI step.
// Signature: TestTCPDrainAndPostStartAdditionalCloseoutBranches(t *testing.T).
// Why: near-threshold TCP, drain, and post-start helpers need local coverage for
// TLS probing, error throttling, SSH fallback, and diagnostics summaries.
func TestTCPDrainAndPostStartAdditionalCloseoutBranches(t *testing.T) {
ctx := context.Background()
certServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = w.Write([]byte("ok"))
}))
cert := certServer.TLS.Certificates[0]
certServer.Close()
listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}})
if err != nil {
t.Fatalf("listen TLS fixture: %v", err)
}
t.Cleanup(func() { _ = listener.Close() })
go func() {
for {
conn, err := listener.Accept()
if err != nil {
return
}
go func() {
defer conn.Close()
_, _ = conn.Write([]byte("220 ready\r\n"))
_ = conn.SetDeadline(time.Now().Add(2 * time.Second))
buf := make([]byte, 32)
_, _ = conn.Read(buf)
_, _ = conn.Write([]byte("250 tls-pong\r\n"))
}()
}
}()
host, portText, err := net.SplitHostPort(listener.Addr().String())
if err != nil {
t.Fatalf("split TLS server address: %v", err)
}
var port int
if _, err := fmt.Sscanf(portText, "%d", &port); err != nil {
t.Fatalf("parse TLS server port: %v", err)
}
ok, detail := buildOrchestratorWithStubs(t, config.Config{}, nil).tcpServiceCheckReady(ctx, config.TCPServiceChecklistCheck{
Host: host, Port: port, TLS: true, InsecureSkipTLS: true, Send: "PING\r\n", ExpectContains: "tls-pong",
})
if !ok || detail != "connected" {
t.Fatalf("expected TLS tcp check success, ok=%v detail=%q", ok, detail)
}
orch := buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{TCPServiceChecklist: []config.TCPServiceChecklistCheck{{Host: "127.0.0.1", Port: 1, Namespace: "mail", Service: "smtp"}}}}, []commandStub{
{match: matchContains("kubectl", "-n", "mail", "scale", "deployment", "smtp"), err: errors.New("scale denied")},
})
var zero time.Time
orch.maybeAutoHealTCPServiceBackends(ctx, &zero)
if zero.IsZero() {
t.Fatalf("expected tcp heal attempt timestamp")
}
dry := buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{TCPServiceChecklist: []config.TCPServiceChecklistCheck{{Host: "127.0.0.1", Port: 1}}}}, nil)
dry.runner.DryRun = true
dry.maybeAutoHealTCPServiceBackends(ctx, nil)
tmpConfig := t.TempDir() + "/ssh_config"
tmpIdentity := t.TempDir() + "/id_ed25519"
orch = buildOrchestratorWithStubs(t, config.Config{SSHConfigFile: tmpConfig, SSHIdentityFile: tmpIdentity}, nil)
if orch.resolveSSHConfigFile() != tmpConfig || orch.resolveSSHIdentityFile() != tmpIdentity {
t.Fatalf("expected explicit ssh config and identity paths")
}
orch.runSensitiveOverride = nil
if out, err := orch.runSensitive(ctx, time.Second, "sh", "-c", "echo noisy; exit 4"); err == nil || out != "noisy" {
t.Fatalf("expected runSensitive output-backed error, out=%q err=%v", out, err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "cordon", "worker-a"), err: errors.New("cordon warning")},
{match: matchContains("kubectl", "drain", "worker-a"), err: errors.New("drain denied")},
{match: matchContains("kubectl", "get", "pods", "-A"), out: "apps api Running ReplicaSet\nkube ds Running DaemonSet\nold done Succeeded Job"},
})
if err := orch.drainWorkers(ctx, []string{"worker-a"}); err == nil || !strings.Contains(err.Error(), "apps/api") {
t.Fatalf("expected drain diagnostics in error, got %v", err)
}
if diag := orch.drainNodeDiagnostics(ctx, "worker-a"); !strings.Contains(diag, "apps/api") {
t.Fatalf("expected blocking pod diagnostic, got %q", diag)
}
}
// TestWorkloadConvergenceAndAutoHealCloseoutBranches runs one orchestration or CLI step.
// Signature: TestWorkloadConvergenceAndAutoHealCloseoutBranches(t *testing.T).
// Why: workload convergence and post-start auto-heal are broad coordinators, so
// focused fixtures keep their skip, warning, and no-op paths from going dark.
func TestWorkloadConvergenceAndAutoHealCloseoutBranches(t *testing.T) {
ctx := context.Background()
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), err: errors.New("api down")},
})
if _, _, err := orch.workloadConvergenceReady(ctx); err == nil {
t.Fatalf("expected workload controller query error")
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: "{"},
})
if _, _, err := orch.workloadConvergenceReady(ctx); err == nil {
t.Fatalf("expected workload controller decode error")
}
workloads := `{"items":[
{"kind":"","metadata":{"namespace":"apps","name":"skip"}},
{"kind":"Deployment","metadata":{"namespace":"apps","name":"api"},"spec":{"replicas":2},"status":{"readyReplicas":1}},
{"kind":"StatefulSet","metadata":{"namespace":"ignored","name":"db"},"spec":{"replicas":1},"status":{"readyReplicas":0}},
{"kind":"DaemonSet","metadata":{"namespace":"apps","name":"agent"},"status":{"desiredNumberScheduled":2,"numberReady":1}}
]}`
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
IgnoreWorkloadNamespaces: []string{"ignored"},
IgnoreUnavailableNodes: []string{"down-a"},
}}, []commandStub{
{match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: workloads},
})
ready, detail, err := orch.workloadConvergenceReady(ctx)
if err != nil || ready || !strings.Contains(detail, "apps/deployment/api") {
t.Fatalf("expected pending deployment detail, ready=%v detail=%q err=%v", ready, detail, err)
}
if _, _, ok := desiredReady(workloadResource{Kind: "Job"}); ok {
t.Fatalf("unsupported workload kind should not report desired/ready")
}
old := time.Now().Add(-10 * time.Minute)
podA := closeoutPendingControllerPod("apps", "wedge-a", "bad-node", "ReplicaSet", "CreateContainerError", old)
podB := closeoutPendingControllerPod("apps", "wedge-b", "bad-node", "ReplicaSet", "RunContainerError", old)
podA.Spec.Volumes = nil
podB.Spec.Volumes = nil
reasons := map[string]string{"apps/wedge-a": "ContainerRuntimeWedge:bad-node", "apps/wedge-b": "ContainerRuntimeWedge:bad-node"}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "annotate", "node", "bad-node"), out: ""},
{match: matchContains("kubectl", "cordon", "bad-node"), out: ""},
})
if nodes := orch.quarantineContainerRuntimeWedgeNodes(ctx, podList{Items: []podResource{podA, podB}}, reasons, time.Second, nil, nil, nil); len(nodes) != 1 || nodes[0] != "bad-node" {
t.Fatalf("expected wedged node quarantine, nodes=%v", nodes)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "annotate", "node", "bad-node"), out: ""},
{match: matchContains("kubectl", "cordon", "bad-node"), err: errors.New("cordon denied")},
})
if nodes := orch.quarantineContainerRuntimeWedgeNodes(ctx, podList{Items: []podResource{podA, podB}}, reasons, time.Second, nil, nil, nil); len(nodes) != 0 {
t.Fatalf("cordon failure should not report quarantined nodes, nodes=%v", nodes)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "pods", "-A"), out: "{"},
})
if _, err := orch.quarantineContainerRuntimeWedgeNodesFromCluster(ctx); err == nil {
t.Fatalf("expected runtime wedge cluster decode error")
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), err: errors.New("events down")},
})
if _, err := orch.containerRuntimeWedgePodReasons(ctx, podList{Items: []podResource{podA}}, time.Second); err == nil {
t.Fatalf("expected runtime wedge event query error")
}
invalidPod := closeoutPendingControllerPod("", "invalid", "node-a", "ReplicaSet", "CreateContainerError", old)
runningPod := closeoutPendingControllerPod("apps", "running", "node-a", "ReplicaSet", "CreateContainerError", old)
runningPod.Status.Phase = "Running"
noOwnerPod := closeoutPendingControllerPod("apps", "no-owner", "node-a", "", "CreateContainerError", old)
noOwnerPod.Metadata.OwnerReferences = nil
newPod := closeoutPendingControllerPod("apps", "new", "node-a", "ReplicaSet", "CreateContainerError", time.Now())
runtimePod := closeoutPendingControllerPod("apps", "runtime", "node-a", "ReplicaSet", "CreateContainerError", old)
runtimeEvents := eventList{Items: []eventResource{
{Type: "Normal", Reason: "Failed", Message: "failed to reserve container name"},
{Type: "Warning", Reason: "Other", Message: "failed to reserve container name"},
{Type: "Warning", Reason: "Failed", Message: "failed to reserve container name"},
{Type: "Warning", Reason: "Failed", Message: "failed to reserve container name"},
{Type: "Warning", Reason: "Failed", EventTime: old.Add(-time.Minute), Message: "failed to reserve container name"},
{Type: "Warning", Reason: "Failed", Message: "ordinary failure"},
}}
for i := range runtimeEvents.Items {
runtimeEvents.Items[i].InvolvedObject.Kind = "Pod"
runtimeEvents.Items[i].InvolvedObject.Namespace = "apps"
runtimeEvents.Items[i].InvolvedObject.Name = "runtime"
}
runtimeEvents.Items[2].InvolvedObject.Kind = "Service"
runtimeEvents.Items[3].InvolvedObject.Name = "missing"
runtimeRaw, err := json.Marshal(runtimeEvents)
if err != nil {
t.Fatalf("marshal runtime events: %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), out: string(runtimeRaw)},
})
if reasons, err := orch.containerRuntimeWedgePodReasons(ctx, podList{Items: []podResource{invalidPod, runningPod, noOwnerPod, newPod, runtimePod}}, time.Second); err != nil || len(reasons) != 0 {
t.Fatalf("expected runtime skip-only reasons, got reasons=%v err=%v", reasons, err)
}
ignoredPod := closeoutPendingControllerPod("ignored", "pod", "node-a", "ReplicaSet", "CreateContainerError", old)
rulePod := closeoutPendingControllerPod("apps", "skip-me", "node-a", "ReplicaSet", "CreateContainerError", old)
ignoredNodePod := closeoutPendingControllerPod("apps", "node-skip", "node-skip", "ReplicaSet", "CreateContainerError", old)
pvcPod := closeoutPendingControllerPod("apps", "pvc", "node-a", "ReplicaSet", "CreateContainerError", old)
noReasonPod := closeoutPendingControllerPod("apps", "no-reason", "node-a", "ReplicaSet", "CreateContainerError", old)
reasonMap := map[string]string{
"ignored/pod": "ContainerRuntimeWedge:node-a",
"apps/skip-me": "ContainerRuntimeWedge:node-a",
"apps/node-skip": "ContainerRuntimeWedge:node-skip",
"apps/no-owner": "ContainerRuntimeWedge:node-a",
"apps/new": "ContainerRuntimeWedge:node-a",
"apps/pvc": "ContainerRuntimeWedge:node-a",
}
if nodes := orch.quarantineContainerRuntimeWedgeNodes(ctx, podList{Items: []podResource{invalidPod, noReasonPod, ignoredPod, rulePod, ignoredNodePod, noOwnerPod, newPod, pvcPod}}, reasonMap, time.Second, map[string]struct{}{"ignored": struct{}{}}, map[string]struct{}{"node-skip": struct{}{}}, []workloadIgnoreRule{{Namespace: "apps", Name: "skip-me"}}); len(nodes) != 0 {
t.Fatalf("skip-only quarantine should return no nodes, got %v", nodes)
}
stuck := closeoutPendingControllerPod("apps", "crashy", "node-a", "ReplicaSet", "CrashLoopBackOff", old)
stuck.Spec.Volumes = nil
podsJSON := fmt.Sprintf(`{"items":[{"metadata":{"namespace":"apps","name":"crashy","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"node-a"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CrashLoopBackOff"}}}]}}]}`, old.Format(time.RFC3339))
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{
{match: matchContains("kubectl", "get", "pods", "-A"), out: podsJSON},
{match: matchContains("kubectl", "get", "events", "-A"), err: errors.New("events down")},
{match: matchContains("kubectl", "get", "pvc", "-A"), err: errors.New("pvc down")},
{match: matchContains("kubectl", "-n", "apps", "delete", "pod", "crashy"), out: ""},
})
if err := orch.recycleStuckControllerPods(ctx); err != nil {
t.Fatalf("expected stuck pod recycle despite scan warnings: %v", err)
}
auto := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: ""},
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[]}`},
{match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"},
{match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`},
})
if err := auto.postStartAutoHeal(ctx); err != nil {
t.Fatalf("expected post-start auto-heal no-op success, got %v", err)
}
}
// TestStartupWaitLoopCloseoutBranches runs one orchestration or CLI step.
// Signature: TestStartupWaitLoopCloseoutBranches(t *testing.T).
// Why: startup wait loops should cover immediate success and cancellation paths
// without sleeping or requiring any live Kubernetes objects.
func TestStartupWaitLoopCloseoutBranches(t *testing.T) {
ctx := context.Background()
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: `{"items":[]}`},
})
if err := orch.waitForWorkloadConvergence(ctx); err != nil {
t.Fatalf("expected workload convergence success: %v", err)
}
canceled, cancel := context.WithCancel(ctx)
cancel()
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
WorkloadConvergenceWaitSeconds: 1,
WorkloadConvergencePollSeconds: 1,
}}, []commandStub{
{match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: `{"items":[{"kind":"Deployment","metadata":{"namespace":"apps","name":"api"},"spec":{"replicas":1},"status":{"readyReplicas":0}}]}`},
})
if err := orch.waitForWorkloadConvergence(canceled); !errors.Is(err, context.Canceled) {
t.Fatalf("expected canceled workload convergence, got %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, nil)
if err := orch.waitForCriticalServiceEndpoints(ctx); err != nil {
t.Fatalf("expected critical endpoint success: %v", err)
}
canceled, cancel = context.WithCancel(ctx)
cancel()
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
CriticalServiceEndpoints: []string{"apps/api"},
CriticalServiceEndpointWaitSec: 1,
CriticalServiceEndpointPollSec: 1,
}}, []commandStub{
{match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), out: `{"subsets":[]}`},
})
orch.runner.DryRun = true
if err := orch.waitForCriticalServiceEndpoints(canceled); !errors.Is(err, context.Canceled) {
t.Fatalf("expected canceled critical endpoint wait, got %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, nil)
orch.runner.DryRun = true
if err := orch.waitForPostStartProbes(ctx); err != nil {
t.Fatalf("dry-run post-start probes should pass: %v", err)
}
canceled, cancel = context.WithCancel(ctx)
cancel()
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
PostStartProbes: []string{"https://example.invalid"},
PostStartProbeWaitSeconds: 1,
PostStartProbePollSeconds: 1,
}}, []commandStub{
{match: matchContains("curl", "https://example.invalid"), err: errors.New("curl down")},
})
if err := orch.waitForPostStartProbes(canceled); !errors.Is(err, context.Canceled) {
t.Fatalf("expected canceled post-start probe wait, got %v", err)
}
}

370
internal/cluster/orchestrator_quality_gate_recovery_closeout_test.go Normal file
View File

@ -0,0 +1,370 @@
package cluster
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
"scm.bstein.dev/bstein/ananke/internal/config"
)
// TestCordonLeaseCriticalEndpointAndProbeCloseoutBranches runs one orchestration or CLI step.
// Signature: TestCordonLeaseCriticalEndpointAndProbeCloseoutBranches(t *testing.T).
// Why: recovery cordons and endpoint probe repair carry the cluster-safety
// contract for automatic startup repairs, so their failure paths need coverage.
func TestCordonLeaseCriticalEndpointAndProbeCloseoutBranches(t *testing.T) {
ctx := context.Background()
orch := buildOrchestratorWithStubs(t, config.Config{}, nil)
if err := orch.cordonNodeWithLease(ctx, " ", "", ""); err == nil {
t.Fatalf("expected empty cordon node to fail")
}
if ok := manualActionAlreadyMarked(nil, "action"); ok {
t.Fatalf("nil annotations should not already mark action")
}
if expired, detail := cordonLeaseExpired(map[string]string{}, time.Now()); !expired || detail != "missing deadline" {
t.Fatalf("expected missing deadline to expire, expired=%v detail=%q", expired, detail)
}
if expired, detail := cordonLeaseExpired(map[string]string{anankeCordonDeadlineAnnotation: "bad-time"}, time.Now()); !expired || detail != "bad-time" {
t.Fatalf("expected malformed deadline to expire, expired=%v detail=%q", expired, detail)
}
if since := nodeUnschedulableSince(nodeReadyItem{}); !since.IsZero() {
t.Fatalf("expected zero unschedulable timestamp, got %s", since)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "annotate", "node", "titan-23"), err: errors.New("annotate denied")},
})
if err := orch.cordonNodeWithLease(ctx, "titan-23", "repair", "detail"); err == nil || !strings.Contains(err.Error(), "annotate cordon lease") {
t.Fatalf("expected annotate cordon lease error, got %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "cordon", "titan-23"), err: errors.New("cordon denied")},
{match: matchContains("kubectl", "annotate", "node", "titan-23"), out: ""},
})
if err := orch.cordonNodeWithLease(ctx, "titan-23", "repair", "detail"); err == nil || !strings.Contains(err.Error(), "cordon denied") {
t.Fatalf("expected cordon failure, got %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "annotate", "node", "gone"), err: errors.New(`Error from server (NotFound): nodes "gone" not found`)},
{match: matchContains("kubectl", "annotate", "node", "blocked"), err: errors.New("api denied")},
})
if err := orch.clearCordonLease(ctx, "gone"); err != nil {
t.Fatalf("not-found clear should be ignored: %v", err)
}
if err := orch.clearCordonLease(ctx, "blocked"); err == nil || !strings.Contains(err.Error(), "clear cordon lease") {
t.Fatalf("expected clear cordon lease error, got %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "uncordon", "titan-23"), out: ""},
{match: matchContains("kubectl", "annotate", "node", "titan-23"), err: errors.New("cleanup denied")},
})
if err := orch.uncordonAndClearCordonLease(ctx, "titan-23", "test"); err != nil {
t.Fatalf("lease cleanup warning should not fail uncordon: %v", err)
}
if _, err := orch.recoverLeasedCordon(ctx, "titan-23", map[string]string{anankeCordonReasonAnnotation: "mystery"}); err == nil {
t.Fatalf("expected unknown cordon reason error")
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/titan-23/proxy/healthz"), err: errors.New("proxy down")},
})
if recovered, err := orch.recoverLeasedCordon(ctx, "titan-23", map[string]string{anankeCordonReasonAnnotation: cordonReasonKubeletProxy}); err == nil || recovered {
t.Fatalf("expected unhealthy proxy to keep cordon, recovered=%v err=%v", recovered, err)
}
old := time.Now().Add(-10 * time.Minute)
podJSON := fmt.Sprintf(`{"items":[{"metadata":{"namespace":"apps","name":"stuck","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"titan-23"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CreateContainerError"}}}]}}]}`, old.Format(time.RFC3339))
eventJSON := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"stuck"},"message":"failed to reserve container name for container app"}]}`, time.Now().Format(time.RFC3339))
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{
{match: matchContains("kubectl", "get", "pods", "-A"), out: podJSON},
{match: matchContains("kubectl", "get", "events", "-A"), out: eventJSON},
})
if wedged, err := orch.nodeStillHasRuntimeWedge(ctx, "titan-23"); err != nil || !wedged {
t.Fatalf("expected runtime wedge evidence, wedged=%v err=%v", wedged, err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "pods", "-A"), out: "{"},
})
if _, err := orch.nodeStillHasRuntimeWedge(ctx, "titan-23"); err == nil {
t.Fatalf("expected bad pod json error")
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: "{"},
})
if _, err := orch.queryReadyNodes(ctx); err == nil {
t.Fatalf("expected bad node json error")
}
endpointsJSON := `{"subsets":[{"addresses":[{"ip":"10.0.0.1"},{"ip":"10.0.0.2"}]},{"addresses":[{"ip":""}]}]}`
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{
{match: matchContains("kubectl", "-n", "grafana", "get", "endpoints", "grafana"), out: endpointsJSON},
{match: matchContains("kubectl", "-n", "bad", "get", "endpoints", "svc"), out: "{", err: nil},
})
if count, err := orch.endpointAddressCount(ctx, "grafana", "grafana"); err != nil || count != 2 {
t.Fatalf("unexpected endpoint count=%d err=%v", count, err)
}
if _, err := orch.endpointAddressCount(ctx, "bad", "svc"); err == nil {
t.Fatalf("expected endpoint decode error")
}
if ready, detail, _, _, err := orch.criticalServiceEndpointsReady(ctx); err != nil || !ready || detail != "no critical service endpoints configured" {
t.Fatalf("expected empty critical endpoint checklist ready, ready=%v detail=%q err=%v", ready, detail, err)
}
workloadJSON := `{"spec":{"template":{"spec":{"containers":[{"name":"app","livenessProbe":{"httpGet":{"path":"/healthz"}}},{"name":"","livenessProbe":{"tcpSocket":{"port":8080}}},{"name":"has-startup","livenessProbe":{"httpGet":{"path":"/ready"}},"startupProbe":{"httpGet":{"path":"/ready"}}}]}}}}`
var patched bool
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{
{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: workloadJSON},
{match: func(name string, args []string) bool {
if name == "kubectl" && strings.Contains(strings.Join(args, " "), "-n apps patch deployment api") {
patched = true
return true
}
return false
}, out: ""},
})
repaired, err := orch.repairWorkloadStartupProbes(ctx, "apps", "deployment", "api")
if err != nil || len(repaired) != 1 || repaired[0] != "apps/deployment/api" || !patched {
t.Fatalf("unexpected startup probe repair repaired=%v patched=%v err=%v", repaired, patched, err)
}
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{
{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: "{", err: nil},
})
if _, err := orch.repairWorkloadStartupProbes(ctx, "apps", "deployment", "api"); err == nil {
t.Fatalf("expected workload decode error")
}
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{
{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: workloadJSON},
{match: matchContains("kubectl", "-n", "apps", "patch", "deployment", "api"), err: errors.New("patch denied")},
})
if _, err := orch.repairWorkloadStartupProbes(ctx, "apps", "deployment", "api"); err == nil {
t.Fatalf("expected workload patch error")
}
if repaired, err := orch.maybeRepairCriticalBackendStartupProbes(ctx, " ", "api"); err != nil || repaired != nil {
t.Fatalf("blank critical backend repair should no-op, repaired=%v err=%v", repaired, err)
}
}
// TestLonghornRuntimeStaleRWOAndPostStartCloseoutBranches runs one orchestration or CLI step.
// Signature: TestLonghornRuntimeStaleRWOAndPostStartCloseoutBranches(t *testing.T).
// Why: storage handoff, host prerequisite repair, runtime readiness, and
// post-start shell helpers are high-risk automation seams that should stay
// covered by local fixtures.
func TestLonghornRuntimeStaleRWOAndPostStartCloseoutBranches(t *testing.T) {
ctx := context.Background()
old := time.Now().Add(-10 * time.Minute)
pod := closeoutPendingControllerPod("apps", "crypt", "titan-23", "ReplicaSet", "CreateContainerError", old)
events := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"crypt"},"message":"MountVolume.SetUp failed: cryptsetup: no such file or directory"}]}`, time.Now().Format(time.RFC3339))
cryptsetupCalls := 0
orch := buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"titan-23"}}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), out: events},
{match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")},
{match: matchContains("kubectl", "annotate", "node", "titan-23"), out: ""},
{match: matchContains("kubectl", "cordon", "titan-23"), out: ""},
})
reasons, err := orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{pod}}, time.Second)
if err != nil || reasons["apps/crypt"] != "EncryptedVolumeCryptsetupNodeCordoned:titan-23" {
t.Fatalf("unexpected encrypted mount repair reasons=%v err=%v", reasons, err)
}
orch = buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"titan-23"}}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), out: events},
{match: func(name string, args []string) bool {
if name == "ssh" && strings.Contains(strings.Join(args, " "), "apt-get update") {
cryptsetupCalls++
return true
}
return false
}, out: "__ANANKE_CRYPTSETUP_INSTALLED__"},
{match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_MISSING__", err: errors.New("exit 41")},
})
reasons, err = orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{pod, pod}}, time.Second)
if err != nil || reasons["apps/crypt"] != "EncryptedVolumeCryptsetupRepaired:titan-23" || cryptsetupCalls != 1 {
t.Fatalf("unexpected repaired cryptsetup reasons=%v calls=%d err=%v", reasons, cryptsetupCalls, err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), out: "{"},
})
if _, err := orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{pod}}, time.Second); err == nil {
t.Fatalf("expected encrypted mount event decode error")
}
longhornNodes := "titan-23\tFalse\nready\tTrue\npartial\n"
attachEvents := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc-1: node titan-23 is not ready: longhorn-backend unavailable"}]}`, time.Now().Format(time.RFC3339))
attachPod := closeoutPendingControllerPod("apps", "attach", "titan-23", "ReplicaSet", "ContainerCreating", old)
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: longhornNodes},
{match: matchContains("kubectl", "get", "events", "-A"), out: attachEvents},
})
blocked, err := orch.longhornAttachBlockedPodReasons(ctx, podList{Items: []podResource{attachPod}}, time.Second)
if err != nil || blocked["apps/attach"] != "LonghornAttachBlockedOnUnreadyNode:titan-23" {
t.Fatalf("unexpected attach-blocked reasons=%v err=%v", blocked, err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), err: errors.New("api down")},
})
if _, err := orch.longhornAttachBlockedPodReasons(ctx, podList{Items: []podResource{attachPod}}, time.Second); err == nil {
t.Fatalf("expected longhorn node query error")
}
pvcJSON := `{"items":[{"metadata":{"namespace":"apps","name":"data"},"spec":{"accessModes":["ReadWriteOnce"],"volumeName":"pv-data"}}]}`
replacement := closeoutPendingControllerPod("apps", "app-new", "titan-24", "ReplicaSet", "ContainerCreating", old)
oldOwner := closeoutDeletingRWOPod("apps", "app-old", "titan-23", old, false)
blockEvents := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"app-new"},"message":"Multi-Attach error for volume pv-data"}]}`, time.Now().Format(time.RFC3339))
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), out: blockEvents},
{match: matchContains("kubectl", "get", "pvc", "-A"), out: pvcJSON},
})
decisions, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{Items: []podResource{oldOwner, replacement}}, time.Second)
if err != nil || !decisions["apps/app-old"].ForceDelete || decisions["apps/app-old"].Unsafe {
t.Fatalf("unexpected safe stale RWO decision=%v err=%v", decisions, err)
}
oldUnsafe := closeoutDeletingRWOPod("apps", "app-old", "titan-23", old, true)
decisions, err = orch.staleRWOPVCOwnerDecisions(ctx, podList{Items: []podResource{oldUnsafe, replacement}}, time.Second)
if err != nil || !decisions["apps/app-old"].Unsafe {
t.Fatalf("unexpected unsafe stale RWO decision=%v err=%v", decisions, err)
}
if pvcSingleWriter(persistentVolumeClaimResource{}) {
t.Fatalf("empty pvc should not be single-writer")
}
if replacementPodPendingForAttach(podResource{}) {
t.Fatalf("empty pod should not be pending for attach")
}
if podKey(podResource{}) != "" {
t.Fatalf("empty pod should not have key")
}
runtimeCfg := config.Config{SSHManagedNodes: []string{"titan-23"}, Startup: config.Startup{NodeRuntimeRestartWaitSeconds: 1}}
orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
{match: matchContains("ssh", "systemctl --no-block restart k3s-agent"), out: ""},
{match: matchContains("kubectl", "wait", "node/titan-23"), out: ""},
{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/titan-23/proxy/healthz"), out: "ok"},
{match: matchContains("ssh", "systemctl is-active k3s-agent"), out: "active"},
})
if err := orch.recoverManagedNodeRuntime(ctx, "titan-23", true, "already cordoned"); err != nil {
t.Fatalf("expected runtime recovery success, got %v", err)
}
orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
{match: matchContains("kubectl", "wait", "node/titan-23"), out: ""},
{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/titan-23/proxy/healthz"), out: "ok"},
{match: matchContains("ssh", "systemctl is-active k3s-agent"), out: "inactive"},
})
if err := orch.waitForManagedNodeRuntimeReady(ctx, "titan-23", 0); err == nil || !strings.Contains(err.Error(), "inactive") {
t.Fatalf("expected inactive k3s-agent readiness failure, got %v", err)
}
if got := orch.nodeRuntimeRebootWait(); got != 420*time.Second {
t.Fatalf("expected default reboot wait, got %s", got)
}
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{PostStartProbes: []string{" https://ok ", " "}}}, []commandStub{
{match: matchContains("curl", "https://ok"), out: "403"},
})
if ok, detail := orch.postStartProbesReady(ctx); !ok || detail != "all probes successful" {
t.Fatalf("expected post-start probe success, ok=%v detail=%q", ok, detail)
}
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{PostStartProbes: []string{"https://bad"}}}, []commandStub{
{match: matchContains("curl", "https://bad"), out: "oops"},
})
if ok, detail := orch.postStartProbesReady(ctx); ok || !strings.Contains(detail, "parse http status") {
t.Fatalf("expected post-start parse failure, ok=%v detail=%q", ok, detail)
}
if probeStatusAccepted("x", 500) {
t.Fatalf("500 should not be an accepted probe status")
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "--raw", "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health"), err: errors.New("transport closed")},
})
if _, err := orch.vaultSealedViaHTTP(ctx); err == nil || !strings.Contains(err.Error(), "vault HTTP health check failed") {
t.Fatalf("expected empty-output vault HTTP error, got %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "--raw", "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health"), out: "{", err: errors.New("partial")},
})
if _, err := orch.vaultSealedViaHTTP(ctx); err == nil || !strings.Contains(err.Error(), "parse") {
t.Fatalf("expected vault HTTP parse error with transport failure, got %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, nil)
orch.runner.Kubeconfig = filepath.Join(t.TempDir(), "kubeconfig")
if out, err := orch.runSensitiveWithInput(ctx, time.Second, "secret", "sh", "-c", "cat"); err != nil || out != "secret" {
t.Fatalf("expected runSensitiveWithInput echo, out=%q err=%v", out, err)
}
if out, err := orch.runSensitiveWithInput(ctx, time.Second, "", "sh", "-c", "exit 3"); err == nil || out != "" {
t.Fatalf("expected empty-output command failure, out=%q err=%v", out, err)
}
binDir := t.TempDir()
fakeSSH := filepath.Join(binDir, "ssh")
if err := os.WriteFile(fakeSSH, []byte("#!/bin/sh\ncase \" $* \" in *\" -J \"*) echo jump failed; exit 7;; esac\ncat >/dev/null\necho direct-ok\n"), 0o755); err != nil {
t.Fatalf("write fake ssh: %v", err)
}
fakeFlux := filepath.Join(binDir, "flux")
if err := os.WriteFile(fakeFlux, []byte("#!/bin/sh\necho flux-ok\n"), 0o755); err != nil {
t.Fatalf("write fake flux: %v", err)
}
t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
orch = buildOrchestratorWithStubs(t, config.Config{SSHJumpHost: "bastion", SSHJumpUser: "jump", SSHUser: "atlas", SSHNodeHosts: map[string]string{"titan-23": "10.0.0.23"}}, nil)
out, err := orch.sshWithInput(ctx, "titan-23", "sudo -S true", "pw\n", time.Second)
if err != nil || out != "direct-ok" {
t.Fatalf("expected fake ssh direct fallback success, out=%q err=%v", out, err)
}
var sawFlux bool
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "patch"), out: ""},
{match: matchContains("kubectl", "annotate"), out: ""},
{match: func(name string, args []string) bool {
if name == "flux" && strings.Contains(strings.Join(args, " "), "reconcile source git flux-system") {
sawFlux = true
return true
}
return false
}, out: ""},
})
if err := orch.resumeFluxAndReconcile(ctx); err != nil || !sawFlux {
t.Fatalf("expected flux reconcile path, sawFlux=%v err=%v", sawFlux, err)
}
}
// closeoutPendingControllerPod runs one orchestration or CLI step.
// Signature: closeoutPendingControllerPod(ns, name, node, ownerKind, waitReason string, created time.Time) podResource.
// Why: coverage closeout tests need compact pod fixtures for storage and runtime
// classifiers without repeating nested Kubernetes JSON structs.
func closeoutPendingControllerPod(ns, name, node, ownerKind, waitReason string, created time.Time) podResource {
var pod podResource
pod.Metadata.Namespace = ns
pod.Metadata.Name = name
pod.Metadata.CreationTimestamp = created
pod.Metadata.OwnerReferences = []ownerReference{{Kind: ownerKind, Name: "owner"}}
pod.Spec.NodeName = node
pod.Spec.Volumes = []podVolume{{Name: "data", PersistentVolumeClaim: &podPersistentVolumeClaim{ClaimName: "data"}}}
pod.Status.Phase = "Pending"
pod.Status.ContainerStatuses = []podContainerStatus{{Name: "app", State: podContainerState{Waiting: &podContainerWaitingState{Reason: waitReason}}}}
return pod
}
// closeoutDeletingRWOPod runs one orchestration or CLI step.
// Signature: closeoutDeletingRWOPod(ns, name, node string, deletedAt time.Time, runningWriter bool) podResource.
// Why: stale RWO tests need paired safe and unsafe terminating owner pods with
// the same PVC semantics but different live-writer evidence.
func closeoutDeletingRWOPod(ns, name, node string, deletedAt time.Time, runningWriter bool) podResource {
pod := closeoutPendingControllerPod(ns, name, node, "ReplicaSet", "", deletedAt.Add(-time.Hour))
pod.Metadata.DeletionTimestamp = &deletedAt
pod.Status.Phase = "Running"
pod.Spec.Containers = []podContainer{
{Name: "app", VolumeMounts: []podVolumeMount{{Name: "data", MountPath: "/data"}}},
{Name: "sidecar"},
}
statusName := "sidecar"
if runningWriter {
statusName = "app"
}
pod.Status.ContainerStatuses = []podContainerStatus{{Name: statusName, State: podContainerState{Running: &podContainerRunningState{StartedAt: deletedAt.Add(-time.Minute)}}}}
return pod
}

429
internal/cluster/orchestrator_quality_gate_storage_closeout_test.go Normal file
View File

@ -0,0 +1,429 @@
package cluster
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"scm.bstein.dev/bstein/ananke/internal/config"
)
// TestLonghornRecoveryClassifierCloseoutBranches runs one orchestration or CLI step.
// Signature: TestLonghornRecoveryClassifierCloseoutBranches(t *testing.T).
// Why: Longhorn mount and attach recovery should skip noisy non-matches and
// only act on current, pod-scoped storage evidence.
func TestLonghornRecoveryClassifierCloseoutBranches(t *testing.T) {
ctx := context.Background()
old := time.Now().Add(-10 * time.Minute)
now := time.Now()
invalid := closeoutPendingControllerPod("", "invalid", "node-a", "ReplicaSet", "CreateContainerError", old)
running := closeoutPendingControllerPod("apps", "running", "node-a", "ReplicaSet", "CreateContainerError", old)
running.Status.Phase = "Running"
noOwner := closeoutPendingControllerPod("apps", "no-owner", "node-a", "", "CreateContainerError", old)
noOwner.Metadata.OwnerReferences = nil
newPod := closeoutPendingControllerPod("apps", "new", "node-a", "ReplicaSet", "CreateContainerError", time.Now())
unmanaged := closeoutPendingControllerPod("apps", "unmanaged", "node-unmanaged", "ReplicaSet", "CreateContainerError", old)
reuseA := closeoutPendingControllerPod("apps", "reuse-a", "node-ok", "ReplicaSet", "CreateContainerError", old)
reuseB := closeoutPendingControllerPod("apps", "reuse-b", "node-ok", "ReplicaSet", "CreateContainerError", old)
fail := closeoutPendingControllerPod("apps", "fail", "node-fail", "ReplicaSet", "CreateContainerError", old)
events := fmt.Sprintf(`{"items":[
{"type":"Normal","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"},
{"type":"Warning","reason":"Other","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"},
{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Service","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"},
{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"missing"},"message":"cryptsetup: no such file or directory"},
{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"},
{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"different mount failure"},
{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"unmanaged"},"message":"cryptsetup: no such file or directory"},
{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"},
{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-b"},"message":"cryptsetup: no such file or directory"},
{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"fail"},"message":"cryptsetup: no such file or directory"}
]}`,
now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339),
old.Add(-time.Minute).Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339),
now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339))
orch := buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"node-ok", "node-fail"}}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), out: events},
{match: matchContains("ssh", "node-ok", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_PRESENT__"},
{match: matchContains("ssh", "node-fail", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")},
{match: matchContains("kubectl", "annotate", "node", "node-fail"), err: errors.New("annotate denied")},
})
reasons, err := orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{invalid, running, noOwner, newPod, unmanaged, reuseA, reuseB, fail}}, time.Second)
if err != nil {
t.Fatalf("repairEncryptedVolumeMountPrereqs: %v", err)
}
if reasons["apps/reuse-a"] != "EncryptedVolumeCryptsetupRepaired:node-ok" || reasons["apps/reuse-b"] != "EncryptedVolumeCryptsetupRepaired:node-ok" {
t.Fatalf("expected reused cryptsetup repair reasons, got %v", reasons)
}
if reasons["apps/fail"] != "" {
t.Fatalf("cordon failure should not mark failed pod repaired, reasons=%v", reasons)
}
attachEvents := fmt.Sprintf(`{"items":[
{"type":"Normal","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"},
{"type":"Warning","reason":"Other","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"},
{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Service","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"},
{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"missing"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"},
{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"},
{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"some other attach message"},
{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node other is not ready: longhorn-backend unavailable"}
]}`,
now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339),
old.Add(-time.Minute).Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339))
attach := closeoutPendingControllerPod("apps", "attach", "node-bad", "ReplicaSet", "ContainerCreating", old)
notPending := closeoutPendingControllerPod("apps", "not-pending", "node-bad", "ReplicaSet", "ContainerCreating", old)
notPending.Status.Phase = "Running"
readyNode := closeoutPendingControllerPod("apps", "ready-node", "node-ready", "ReplicaSet", "ContainerCreating", old)
noAttachOwner := closeoutPendingControllerPod("apps", "no-owner", "node-bad", "", "ContainerCreating", old)
noAttachOwner.Metadata.OwnerReferences = nil
newAttach := closeoutPendingControllerPod("apps", "new", "node-bad", "ReplicaSet", "ContainerCreating", time.Now())
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: "node-bad\tFalse\nnode-ready\tTrue\n"},
{match: matchContains("kubectl", "get", "events", "-A"), out: attachEvents},
})
blocked, err := orch.longhornAttachBlockedPodReasons(ctx, podList{Items: []podResource{attach, notPending, readyNode, noAttachOwner, newAttach}}, time.Second)
if err != nil {
t.Fatalf("longhornAttachBlockedPodReasons: %v", err)
}
if blocked["apps/attach"] != "" {
t.Fatalf("all attach events should have been skipped, got %v", blocked)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), err: errors.New(`Error from server (NotFound): nodes.longhorn.io not found`)},
})
if unready, err := orch.longhornUnreadyNodes(ctx); err != nil || len(unready) != 0 {
t.Fatalf("expected missing longhorn nodes to be empty, unready=%v err=%v", unready, err)
}
}
// TestStaleRWOClassifierCloseoutBranches runs one orchestration or CLI step.
// Signature: TestStaleRWOClassifierCloseoutBranches(t *testing.T).
// Why: stale RWO ownership must reject incomplete handoff evidence before it
// considers force-deleting a terminating pod.
func TestStaleRWOClassifierCloseoutBranches(t *testing.T) {
ctx := context.Background()
old := time.Now().Add(-10 * time.Minute)
pvcJSON := `{"items":[{"metadata":{"namespace":"apps","name":"data"},"spec":{"accessModes":["ReadWriteOnce"]}}]}`
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), out: "{"},
})
if _, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{}, time.Second); err == nil {
t.Fatalf("expected stale RWO event decode error")
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), out: ""},
{match: matchContains("kubectl", "get", "pvc", "-A"), err: errors.New("pvc down")},
})
if _, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{}, time.Second); err == nil {
t.Fatalf("expected stale RWO pvc query error")
}
oldPod := closeoutDeletingRWOPod("apps", "old", "node-a", old, false)
sameNode := closeoutPendingControllerPod("apps", "same-node", "node-a", "ReplicaSet", "ContainerCreating", old)
noEvent := closeoutPendingControllerPod("apps", "no-event", "node-b", "ReplicaSet", "ContainerCreating", old)
noClaim := closeoutPendingControllerPod("apps", "no-claim", "node-b", "ReplicaSet", "ContainerCreating", old)
noClaim.Spec.Volumes = nil
events := `{"items":[{"type":"Warning","reason":"FailedAttachVolume","involvedObject":{"kind":"Pod","namespace":"apps","name":"no-claim"},"message":"Multi-Attach error"}]}`
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), out: events},
{match: matchContains("kubectl", "get", "pvc", "-A"), out: pvcJSON},
})
decisions, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{Items: []podResource{oldPod, sameNode, noEvent, noClaim}}, time.Second)
if err != nil {
t.Fatalf("staleRWOPVCOwnerDecisions: %v", err)
}
if len(decisions) != 0 {
t.Fatalf("incomplete replacement evidence should not decide, got %v", decisions)
}
emptyPVCs, err := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "pvc", "-A"), out: ""},
}).queryPVCs(ctx)
if err != nil || len(emptyPVCs) != 0 {
t.Fatalf("expected empty pvc output to return empty map, pvcs=%v err=%v", emptyPVCs, err)
}
if runningContainersMountAnyVolume(podResource{}, nil) {
t.Fatalf("empty volume set should be safe")
}
if podHasBlockedAttachEvent(podResource{}, eventList{Items: []eventResource{{Type: "Normal"}}}) {
t.Fatalf("normal events should not block attach")
}
if strings.TrimSpace(podControllerKey(podResource{})) != "" {
t.Fatalf("empty pod should not have controller key")
}
}
// TestRecoveryCoordinatorMiscCloseoutBranches runs one orchestration or CLI step.
// Signature: TestRecoveryCoordinatorMiscCloseoutBranches(t *testing.T).
// Why: a handful of coordinator branches only fire on unusual repair failures;
// local stubs keep them covered without mutating the cluster.
func TestRecoveryCoordinatorMiscCloseoutBranches(t *testing.T) {
ctx := context.Background()
orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "annotate", "node", "ok"), out: ""},
{match: matchContains("kubectl", "cordon", "ok"), out: ""},
})
if err := orch.cordonNodeWithLease(ctx, "ok", "", strings.Repeat("detail ", 80)); err != nil {
t.Fatalf("expected cordon lease success: %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"crypt"}}, []commandStub{
{match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")},
})
if recovered, err := orch.recoverLeasedCordon(ctx, "crypt", map[string]string{anankeCordonReasonAnnotation: cordonReasonMissingCryptsetup}); err == nil || recovered {
t.Fatalf("expected cryptsetup recovery to remain failed, recovered=%v err=%v", recovered, err)
}
old := time.Now().Add(-10 * time.Minute)
wedgedPod := closeoutPendingControllerPod("apps", "wedged", "runtime", "ReplicaSet", "CreateContainerError", old)
wedgedPod.Spec.Volumes = nil
podsJSON := fmt.Sprintf(`{"items":[{"metadata":{"namespace":"apps","name":"wedged","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"runtime"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CreateContainerError"}}}]}}]}`, old.Format(time.RFC3339))
eventsJSON := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"wedged"},"message":"context deadline exceeded while creating container"}]}`, time.Now().Format(time.RFC3339))
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{
{match: matchContains("kubectl", "get", "pods", "-A"), out: podsJSON},
{match: matchContains("kubectl", "get", "events", "-A"), out: eventsJSON},
})
if recovered, err := orch.recoverLeasedCordon(ctx, "runtime", map[string]string{anankeCordonReasonAnnotation: cordonReasonRuntimeWedge}); err == nil || recovered {
t.Fatalf("expected runtime wedge to keep cordon, recovered=%v err=%v", recovered, err)
}
workloadJSON := `{"spec":{"template":{"spec":{"containers":[{"name":"app","livenessProbe":{"httpGet":{"path":"/healthz"}}}]}}}}`
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
CriticalServiceEndpoints: []string{"apps/api"},
CriticalServiceEndpointWaitSec: 1,
CriticalServiceEndpointPollSec: 1,
CriticalServiceStartupProbeRepair: true,
CriticalServiceStartupProbeThreshold: 2,
}}, []commandStub{
{match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), out: `{"subsets":[]}`},
{match: matchContains("kubectl", "-n", "apps", "scale", "deployment", "api"), err: errors.New("scale denied")},
{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: workloadJSON},
{match: matchContains("kubectl", "-n", "apps", "patch", "deployment", "api"), out: ""},
{match: matchContains("kubectl", "-n", "apps", "get", "statefulset", "api"), err: errors.New(`Error from server (NotFound): statefulsets.apps "api" not found`)},
})
canceled, cancel := context.WithCancel(ctx)
cancel()
if err := orch.waitForCriticalServiceEndpoints(canceled); !errors.Is(err, context.Canceled) {
t.Fatalf("expected canceled critical wait after repair attempt, got %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
CriticalServiceEndpoints: []string{"apps/api"},
CriticalServiceStartupProbeRepair: true,
AutoRecycleStuckPods: true,
}}, []commandStub{
{match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), out: `{"subsets":[]}`},
{match: matchContains("kubectl", "-n", "apps", "scale", "deployment", "api"), err: errors.New("scale denied")},
{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: `{"spec":{"template":{"spec":{"containers":[{"name":"app"}]}}}}`},
{match: matchContains("kubectl", "-n", "apps", "get", "statefulset", "api"), err: errors.New(`Error from server (NotFound): statefulsets.apps "api" not found`)},
{match: matchContains("kubectl", "get", "pods", "-A"), out: `{"items":[]}`},
{match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`},
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: ""},
{match: matchContains("kubectl", "get", "pvc", "-A"), out: ""},
})
if _, err := orch.healUnreadyConfiguredServiceBackends(ctx); err == nil {
t.Fatalf("expected heal error when probe repair finds no containers")
}
}
// TestRecycleDecisionCloseoutBranches runs one orchestration or CLI step.
// Signature: TestRecycleDecisionCloseoutBranches(t *testing.T).
// Why: pod recycling must skip external blockers and force-delete only the
// stale-owner cases that have enough storage handoff evidence.
func TestRecycleDecisionCloseoutBranches(t *testing.T) {
ctx := context.Background()
old := time.Now().Add(-10 * time.Minute)
dnsPod := closeoutPendingControllerPod("apps", "dns", "node-a", "ReplicaSet", "ImagePullBackOff", old)
credPod := closeoutPendingControllerPod("apps", "cred", "node-a", "ReplicaSet", "ErrImagePull", old)
unsafeOld := closeoutDeletingRWOPod("apps", "unsafe-old", "node-a", old, true)
unsafeNew := closeoutPendingControllerPod("apps", "unsafe-new", "node-b", "ReplicaSet", "ContainerCreating", old)
forceOld := closeoutDeletingRWOPod("apps", "force-old", "node-a", old, false)
forceNew := closeoutPendingControllerPod("apps", "force-new", "node-b", "ReplicaSet", "ContainerCreating", old)
staleDeleting := closeoutDeletingRWOPod("apps", "stale-delete", "node-a", old, false)
staleDeleting.Spec.Volumes = nil
deleteFail := closeoutPendingControllerPod("apps", "delete-fail", "node-a", "ReplicaSet", "CrashLoopBackOff", old)
deleteFail.Spec.Volumes = nil
podsRaw, err := json.Marshal(podList{Items: []podResource{dnsPod, credPod, unsafeOld, unsafeNew, forceOld, forceNew, staleDeleting, deleteFail}})
if err != nil {
t.Fatalf("marshal pods: %v", err)
}
events := eventList{Items: []eventResource{
{Type: "Warning", Reason: "Failed", Message: "failed to pull image: lookup registry.local: no such host"},
{Type: "Warning", Reason: "FailedPull", Message: "unauthorized: authentication required"},
{Type: "Warning", Reason: "FailedAttachVolume", Message: "Multi-Attach error for unsafe"},
{Type: "Warning", Reason: "FailedAttachVolume", Message: "Multi-Attach error for force"},
}}
events.Items[0].InvolvedObject.Kind = "Pod"
events.Items[0].InvolvedObject.Namespace = "apps"
events.Items[0].InvolvedObject.Name = "dns"
events.Items[1].InvolvedObject.Kind = "Pod"
events.Items[1].InvolvedObject.Namespace = "apps"
events.Items[1].InvolvedObject.Name = "cred"
events.Items[2].InvolvedObject.Kind = "Pod"
events.Items[2].InvolvedObject.Namespace = "apps"
events.Items[2].InvolvedObject.Name = "unsafe-new"
events.Items[3].InvolvedObject.Kind = "Pod"
events.Items[3].InvolvedObject.Namespace = "apps"
events.Items[3].InvolvedObject.Name = "force-new"
eventsRaw, err := json.Marshal(events)
if err != nil {
t.Fatalf("marshal events: %v", err)
}
pvcJSON := `{"items":[{"metadata":{"namespace":"apps","name":"data"},"spec":{"accessModes":["ReadWriteOnce"]}}]}`
orch := buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{
{match: matchContains("kubectl", "get", "pods", "-A"), out: string(podsRaw)},
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: ""},
{match: matchContains("kubectl", "get", "events", "-A"), out: string(eventsRaw)},
{match: matchContains("kubectl", "get", "pvc", "-A"), out: pvcJSON},
{match: matchContains("kubectl", "-n", "apps", "delete", "pod", "delete-fail"), err: errors.New("delete denied")},
{match: matchContains("kubectl", "-n", "apps", "delete", "pod"), out: ""},
})
if err := orch.recycleStuckControllerPods(ctx); err != nil {
t.Fatalf("recycleStuckControllerPods: %v", err)
}
}
// TestSmallCoordinatorErrorCloseoutBranches runs one orchestration or CLI step.
// Signature: TestSmallCoordinatorErrorCloseoutBranches(t *testing.T).
// Why: small coordinator branches should stay covered even when they are only
// reached by aggregated repair failures.
func TestSmallCoordinatorErrorCloseoutBranches(t *testing.T) {
ctx := context.Background()
cfg := config.Config{Startup: config.Startup{CriticalServiceEndpoints: []string{"apps/api"}}}
orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), err: errors.New("longhorn down")},
{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: "{"},
{match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"},
{match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), err: errors.New("endpoints down")},
{match: matchContains("kubectl", "get", "events", "-A"), err: errors.New("events down")},
})
if err := orch.postStartAutoHeal(ctx); err == nil {
t.Fatalf("expected post-start auto-heal aggregation error")
}
if healthy, err := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/flaky/proxy/healthz"), err: errors.New("i/o timeout")},
}).kubeletProxyHealthy(ctx, "flaky"); healthy || err == nil {
t.Fatalf("expected transient kubelet proxy retries to fail, healthy=%v err=%v", healthy, err)
}
if isTransientKubeletProxyCheckErr(nil) {
t.Fatalf("nil error is not transient")
}
if repaired, err := buildOrchestratorWithStubs(t, config.Config{}, nil).maybeRepairCriticalBackendStartupProbes(ctx, "apps", "api"); err != nil || repaired != nil {
t.Fatalf("disabled startup probe repair should no-op, repaired=%v err=%v", repaired, err)
}
orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{
{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: "server said no", err: errors.New("get denied")},
{match: matchContains("kubectl", "-n", "apps", "get", "statefulset", "api"), out: "server said no", err: errors.New("get denied")},
})
if _, err := orch.maybeRepairCriticalBackendStartupProbes(ctx, "apps", "api"); err == nil {
t.Fatalf("expected startup probe repair aggregation error")
}
if err := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "annotate", "node", "manual"), err: errors.New("annotate denied")},
}).clearCordonLease(ctx, "manual"); err == nil {
t.Fatalf("expected clear cordon lease error")
}
buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "annotate", "node", "manual"), err: errors.New("annotate denied")},
}).markManualActionRequired(ctx, "manual", "operator please")
}
// TestFinalSmallCoverageCloseoutBranches runs one orchestration or CLI step.
// Signature: TestFinalSmallCoverageCloseoutBranches(t *testing.T).
// Why: final near-threshold files need concise coverage for rare but important
// recovery error paths.
func TestFinalSmallCoverageCloseoutBranches(t *testing.T) {
ctx := context.Background()
oldDelay := etcdRestorePostStartDelay
etcdRestorePostStartDelay = 0
t.Cleanup(func() { etcdRestorePostStartDelay = oldDelay })
hash := strings.Repeat("a", 64)
orch := buildOrchestratorWithStubs(t, config.Config{
ControlPlanes: []string{"cp1", "cp2"},
SSHManagedNodes: []string{"cp1", "cp2"},
}, []commandStub{
{match: matchContains("ssh", "sudo systemctl cat k3s"), out: "ExecStart=/usr/local/bin/k3s server"},
{match: matchContains("ssh", "test -s '/snap.db'"), out: "2097152"},
{match: matchContains("ssh", "etcd-snapshot ls"), out: "/snap.db"},
{match: matchContains("ssh", "sha256sum '/snap.db'"), out: hash},
{match: matchContains("ssh", "sudo systemctl stop k3s"), out: ""},
{match: matchContains("ssh", "server --cluster-reset --cluster-reset-restore-path /snap.db"), out: ""},
{match: matchContains("ssh", "sudo systemctl start k3s"), out: ""},
})
if err := orch.EtcdRestore(ctx, EtcdRestoreOptions{ControlPlane: "cp1", SnapshotPath: "/snap.db"}); err != nil {
t.Fatalf("expected etcd restore success: %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: ""},
})
if repaired, err := orch.reconcileLonghornKubernetesReadiness(ctx); err != nil || repaired != 0 {
t.Fatalf("empty longhorn nodes should no-op, repaired=%d err=%v", repaired, err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[{"metadata":{"name":"n"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`},
{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{}}}}`},
})
if _, err := orch.reconcileLonghornKubernetesReadiness(ctx); err == nil {
t.Fatalf("expected empty longhorn selector error")
}
orch = buildOrchestratorWithStubs(t, config.Config{
SSHManagedNodes: []string{"managed"},
Startup: config.Startup{
IgnoreUnavailableNodes: []string{"ignored"},
LonghornCryptsetupExemptNodes: []string{"exempt"},
RequiredNodeLabels: map[string]map[string]string{"fallback": {"longhorn-host": "true"}},
},
}, []commandStub{
{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: "ignored\nexempt\nunmanaged\n"},
{match: matchContains("kubectl", "annotate", "node", "unmanaged"), out: ""},
{match: matchContains("kubectl", "cordon", "unmanaged"), out: ""},
})
guarded, err := orch.ensureLonghornEncryptedHostPrereqs(ctx, []string{"ignored", "exempt", "unmanaged", "managed"})
if err != nil || strings.Join(guarded, ",") != "ignored,exempt,managed" {
t.Fatalf("unexpected longhorn prereq guard guarded=%v err=%v", guarded, err)
}
runtimeCfg := config.Config{SSHManagedNodes: []string{"node"}, Startup: config.Startup{NodeRuntimeRestartWaitSeconds: 1}}
orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
{match: matchContains("ssh", "systemctl --no-block restart k3s-agent"), out: ""},
{match: matchContains("kubectl", "wait", "node/node"), err: errors.New("not ready")},
{match: matchContains("ssh", "systemctl show k3s-agent"), err: errors.New("show failed")},
})
if err := orch.recoverManagedNodeRuntime(ctx, "node", true, "test"); err == nil || !strings.Contains(err.Error(), "state check failed") {
t.Fatalf("expected runtime show error, got %v", err)
}
orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
{match: matchContains("ssh", "systemctl --no-block restart k3s-agent"), out: ""},
{match: matchContains("kubectl", "wait", "node/node"), err: errors.New("not ready")},
{match: matchContains("ssh", "systemctl show k3s-agent"), out: "ActiveState=deactivating"},
})
if err := orch.recoverManagedNodeRuntime(ctx, "node", true, "test"); err == nil || !strings.Contains(err.Error(), "host_repair_allow_reboot is false") {
t.Fatalf("expected reboot disallowed error, got %v", err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
{match: matchContains("kubectl", "get", "events", "-A"), out: "{"},
})
if _, err := orch.imagePullCredentialBlockerReasons(ctx); err == nil {
t.Fatalf("expected image-pull credential event decode error")
}
dry := buildOrchestratorWithStubs(t, config.Config{}, nil)
dry.runner.DryRun = true
if repaired, err := dry.healImagePullCredentialSync(ctx); err != nil || repaired != nil {
t.Fatalf("dry-run image credential sync should no-op, repaired=%v err=%v", repaired, err)
}
orch = buildOrchestratorWithStubs(t, config.Config{}, nil)
orch.runSensitiveOverride = nil
orch.runner.Kubeconfig = t.TempDir() + "/kubeconfig"
if out, err := orch.runSensitive(ctx, time.Second, "sh", "-c", "echo ok"); err != nil || out != "ok" {
t.Fatalf("expected runSensitive kubeconfig success, out=%q err=%v", out, err)
}
pod := closeoutPendingControllerPod("apps", "app", "node", "ReplicaSet", "ContainerCreating", time.Now())
pod.Spec.Volumes = []podVolume{{Name: "empty", PersistentVolumeClaim: &podPersistentVolumeClaim{ClaimName: " "}}}
if got := podRWOPVCVolumeNames(pod, nil); len(got) != 0 {
t.Fatalf("blank claim should not map, got %v", got)
}
}

4
internal/config/apply_defaults.go
View File

@ -43,6 +43,10 @@ func (c *Config) applyDefaults() {
},
}
}
c.Startup.RequiredNodeLabelsMode = strings.ToLower(strings.TrimSpace(c.Startup.RequiredNodeLabelsMode))
if c.Startup.RequiredNodeLabelsMode == "" {
c.Startup.RequiredNodeLabelsMode = "enforce"
}
if c.Startup.TimeSyncWaitSeconds <= 0 {
c.Startup.TimeSyncWaitSeconds = 240
}

1
internal/config/defaults.go
View File

@ -71,6 +71,7 @@ func defaults() Config {
"ananke.bstein.dev/harbor-bootstrap": "true",
},
},
RequiredNodeLabelsMode: "enforce",
RequirePostStartProbes: true,
PostStartProbeWaitSeconds: 240,
PostStartProbePollSeconds: 5,

85
internal/config/load.go
View File

@ -7,10 +7,35 @@ import (
"gopkg.in/yaml.v3"
)
type inventoryFragment struct {
SSHUser *string `yaml:"ssh_user"`
SSHPort *int `yaml:"ssh_port"`
SSHConfigFile *string `yaml:"ssh_config_file"`
SSHIdentityFile *string `yaml:"ssh_identity_file"`
SSHNodeHosts *map[string]string `yaml:"ssh_node_hosts"`
SSHNodeUsers *map[string]string `yaml:"ssh_node_users"`
SSHManagedNodes *[]string `yaml:"ssh_managed_nodes"`
ControlPlanes *[]string `yaml:"control_planes"`
Workers *[]string `yaml:"workers"`
Startup *struct {
RequiredNodeLabels *map[string]map[string]string `yaml:"required_node_labels"`
RequiredNodeLabelsMode *string `yaml:"required_node_labels_mode"`
IgnoreUnavailableNodes *[]string `yaml:"ignore_unavailable_nodes"`
} `yaml:"startup"`
}
// Load runs one orchestration or CLI step.
// Signature: Load(path string) (Config, error).
// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
func Load(path string) (Config, error) {
return LoadWithFragments(path)
}
// LoadWithFragments runs one orchestration or CLI step.
// Signature: LoadWithFragments(path string, fragments ...string) (Config, error).
// Why: Terraform can generate a narrow inventory fragment while Ananke keeps
// loading the ordinary host runtime config for imperative recovery behavior.
func LoadWithFragments(path string, fragments ...string) (Config, error) {
cfg := defaults()
b, err := os.ReadFile(path)
@ -24,6 +49,15 @@ func Load(path string) (Config, error) {
if err := yaml.Unmarshal(b, &cfg); err != nil {
return Config{}, fmt.Errorf("decode config %s: %w", path, err)
}
for _, fragment := range fragments {
fb, err := os.ReadFile(fragment)
if err != nil {
return Config{}, fmt.Errorf("read config fragment %s: %w", fragment, err)
}
if err := applyInventoryFragment(fb, &cfg); err != nil {
return Config{}, fmt.Errorf("decode config fragment %s: %w", fragment, err)
}
}
cfg.applyDefaults()
if !startupProbeRepairConfigured {
@ -35,6 +69,57 @@ func Load(path string) (Config, error) {
return cfg, nil
}
// applyInventoryFragment runs one orchestration or CLI step.
// Signature: applyInventoryFragment(raw []byte, cfg *Config) error.
// Why: restricts generated Terraform fragments to declarative inventory fields
// so they cannot accidentally take over Ananke's recovery behavior.
func applyInventoryFragment(raw []byte, cfg *Config) error {
var fragment inventoryFragment
if err := yaml.Unmarshal(raw, &fragment); err != nil {
return err
}
if fragment.SSHUser != nil {
cfg.SSHUser = *fragment.SSHUser
}
if fragment.SSHPort != nil {
cfg.SSHPort = *fragment.SSHPort
}
if fragment.SSHConfigFile != nil {
cfg.SSHConfigFile = *fragment.SSHConfigFile
}
if fragment.SSHIdentityFile != nil {
cfg.SSHIdentityFile = *fragment.SSHIdentityFile
}
if fragment.SSHNodeHosts != nil {
cfg.SSHNodeHosts = *fragment.SSHNodeHosts
}
if fragment.SSHNodeUsers != nil {
cfg.SSHNodeUsers = *fragment.SSHNodeUsers
}
if fragment.SSHManagedNodes != nil {
cfg.SSHManagedNodes = *fragment.SSHManagedNodes
}
if fragment.ControlPlanes != nil {
cfg.ControlPlanes = *fragment.ControlPlanes
}
if fragment.Workers != nil {
cfg.Workers = *fragment.Workers
}
if fragment.Startup == nil {
return nil
}
if fragment.Startup.RequiredNodeLabels != nil {
cfg.Startup.RequiredNodeLabels = *fragment.Startup.RequiredNodeLabels
}
if fragment.Startup.RequiredNodeLabelsMode != nil {
cfg.Startup.RequiredNodeLabelsMode = *fragment.Startup.RequiredNodeLabelsMode
}
if fragment.Startup.IgnoreUnavailableNodes != nil {
cfg.Startup.IgnoreUnavailableNodes = *fragment.Startup.IgnoreUnavailableNodes
}
return nil
}
// yamlPathExists runs one orchestration or CLI step.
// Signature: yamlPathExists(raw []byte, path ...string) (bool, error).
// Why: selected boolean defaults need to distinguish an omitted YAML key from an

183
internal/config/load_additional_test.go
View File

@ -89,3 +89,186 @@ ups:
t.Fatalf("expected explicit gitea-api check, got %q", cfg.Startup.ServiceChecklist[0].Name)
}
}
// TestLoadWithFragmentsReplacesGeneratedInventory runs one orchestration or CLI step.
// Signature: TestLoadWithFragmentsReplacesGeneratedInventory(t *testing.T).
// Why: Terraform-generated inventory fragments must be authoritative for the
// declarative bootstrap fields they own, without merging stale host config.
func TestLoadWithFragmentsReplacesGeneratedInventory(t *testing.T) {
tmp := t.TempDir()
basePath := filepath.Join(tmp, "ananke.yaml")
fragmentPath := filepath.Join(tmp, "ananke.inventory.yaml")
base := `
control_planes: [old-cp]
workers: [old-worker]
ssh_user: atlas
ssh_port: 2277
ssh_node_hosts:
old-cp: 192.0.2.10
ssh_managed_nodes: [old-cp, old-worker]
expected_flux_branch: main
expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
iac_repo_path: /opt/titan-iac
startup:
required_node_labels:
old-worker:
old-label: "true"
ignore_unavailable_nodes: [old-worker]
ups:
enabled: false
`
fragment := `
ssh_user: atlas
ssh_port: 2277
ssh_config_file: /home/atlas/.ssh/config
ssh_identity_file: /home/atlas/.ssh/id_ed25519
ssh_node_hosts:
titan-0a: 192.168.22.11
titan-04: 192.168.22.30
ssh_node_users:
titan-04: workeradmin
ssh_managed_nodes: [titan-0a, titan-04]
control_planes: [titan-0a]
workers: [titan-04]
startup:
required_node_labels_mode: validate
required_node_labels:
titan-04:
node-role.kubernetes.io/worker: "true"
ignore_unavailable_nodes: []
`
if err := os.WriteFile(basePath, []byte(base), 0o644); err != nil {
t.Fatalf("write base config: %v", err)
}
if err := os.WriteFile(fragmentPath, []byte(fragment), 0o644); err != nil {
t.Fatalf("write fragment config: %v", err)
}
cfg, err := LoadWithFragments(basePath, fragmentPath)
if err != nil {
t.Fatalf("load config with fragment: %v", err)
}
if len(cfg.ControlPlanes) != 1 || cfg.ControlPlanes[0] != "titan-0a" {
t.Fatalf("expected fragment control planes, got %v", cfg.ControlPlanes)
}
if _, ok := cfg.SSHNodeHosts["old-cp"]; ok {
t.Fatalf("expected fragment ssh_node_hosts to replace base map, got %#v", cfg.SSHNodeHosts)
}
if cfg.SSHNodeUsers["titan-04"] != "workeradmin" {
t.Fatalf("expected fragment ssh_node_users override, got %#v", cfg.SSHNodeUsers)
}
if _, ok := cfg.Startup.RequiredNodeLabels["old-worker"]; ok {
t.Fatalf("expected fragment required_node_labels to replace base map, got %#v", cfg.Startup.RequiredNodeLabels)
}
if cfg.Startup.RequiredNodeLabelsMode != "validate" {
t.Fatalf("expected fragment label mode validate, got %q", cfg.Startup.RequiredNodeLabelsMode)
}
if len(cfg.Startup.IgnoreUnavailableNodes) != 0 {
t.Fatalf("expected fragment ignored unavailable nodes to replace base slice, got %v", cfg.Startup.IgnoreUnavailableNodes)
}
}
// TestLoadWithFragmentsAllowsTopLevelInventoryOnly runs one orchestration or CLI step.
// Signature: TestLoadWithFragmentsAllowsTopLevelInventoryOnly(t *testing.T).
// Why: generated inventory fragments may omit startup labels during staged
// rollout while still replacing host/node inventory.
func TestLoadWithFragmentsAllowsTopLevelInventoryOnly(t *testing.T) {
tmp := t.TempDir()
basePath := filepath.Join(tmp, "ananke.yaml")
fragmentPath := filepath.Join(tmp, "ananke.inventory.yaml")
base := `
control_planes: [old-cp]
workers: [old-worker]
ssh_user: atlas
ssh_port: 2277
ssh_node_hosts:
old-cp: 192.0.2.10
ssh_managed_nodes: [old-cp, old-worker]
expected_flux_branch: main
expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
iac_repo_path: /opt/titan-iac
startup:
required_node_labels_mode: enforce
required_node_labels:
old-worker:
old-label: "true"
ups:
enabled: false
`
fragment := `
ssh_node_hosts:
titan-0a: 192.168.22.11
ssh_managed_nodes: [titan-0a]
control_planes: [titan-0a]
workers: []
`
if err := os.WriteFile(basePath, []byte(base), 0o644); err != nil {
t.Fatalf("write base config: %v", err)
}
if err := os.WriteFile(fragmentPath, []byte(fragment), 0o644); err != nil {
t.Fatalf("write fragment config: %v", err)
}
cfg, err := LoadWithFragments(basePath, fragmentPath)
if err != nil {
t.Fatalf("load config with top-level fragment: %v", err)
}
if len(cfg.ControlPlanes) != 1 || cfg.ControlPlanes[0] != "titan-0a" {
t.Fatalf("expected top-level fragment control plane, got %v", cfg.ControlPlanes)
}
if _, ok := cfg.Startup.RequiredNodeLabels["old-worker"]; !ok {
t.Fatalf("expected startup labels to remain when fragment omits startup, got %#v", cfg.Startup.RequiredNodeLabels)
}
if cfg.Startup.RequiredNodeLabelsMode != "enforce" {
t.Fatalf("expected base label mode to remain enforce, got %q", cfg.Startup.RequiredNodeLabelsMode)
}
}
// TestLoadWithFragmentsFailsOnMissingFragment runs one orchestration or CLI step.
// Signature: TestLoadWithFragmentsFailsOnMissingFragment(t *testing.T).
// Why: installer and systemd mistakes should fail closed instead of silently
// running with stale embedded inventory.
func TestLoadWithFragmentsFailsOnMissingFragment(t *testing.T) {
cfgPath := filepath.Join(t.TempDir(), "ananke.yaml")
raw := `
control_planes: [titan-0a]
expected_flux_branch: main
expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
iac_repo_path: /opt/titan-iac
ups:
enabled: false
`
if err := os.WriteFile(cfgPath, []byte(raw), 0o644); err != nil {
t.Fatalf("write config: %v", err)
}
if _, err := LoadWithFragments(cfgPath, filepath.Join(t.TempDir(), "missing.yaml")); err == nil {
t.Fatalf("expected missing fragment error")
}
}
// TestLoadWithFragmentsFailsOnBadFragmentYAML runs one orchestration or CLI step.
// Signature: TestLoadWithFragmentsFailsOnBadFragmentYAML(t *testing.T).
// Why: malformed generated inventory must fail closed instead of running with
// partially decoded declarative state.
func TestLoadWithFragmentsFailsOnBadFragmentYAML(t *testing.T) {
tmp := t.TempDir()
cfgPath := filepath.Join(tmp, "ananke.yaml")
fragmentPath := filepath.Join(tmp, "ananke.inventory.yaml")
raw := `
control_planes: [titan-0a]
expected_flux_branch: main
expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
iac_repo_path: /opt/titan-iac
ups:
enabled: false
`
if err := os.WriteFile(cfgPath, []byte(raw), 0o644); err != nil {
t.Fatalf("write config: %v", err)
}
if err := os.WriteFile(fragmentPath, []byte(":\n- invalid"), 0o644); err != nil {
t.Fatalf("write bad fragment: %v", err)
}
if _, err := LoadWithFragments(cfgPath, fragmentPath); err == nil {
t.Fatalf("expected bad fragment decode error")
}
}

25
internal/config/startup_service_catalog_closeout_test.go Normal file
View File

@ -0,0 +1,25 @@
package config
import "testing"
// TestMergeStartupCatalogDefaultsSkipsBlankNames runs one orchestration or CLI step.
// Signature: TestMergeStartupCatalogDefaultsSkipsBlankNames(t *testing.T).
// Why: blank generated or hand-written checklist names should not poison
// default-catalog merging or create duplicate unnamed runtime checks.
func TestMergeStartupCatalogDefaultsSkipsBlankNames(t *testing.T) {
httpMerged := mergeServiceChecklistDefaults(
[]ServiceChecklistCheck{{Name: " "}, {Name: "custom"}},
[]ServiceChecklistCheck{{Name: " "}, {Name: "baseline"}},
)
if len(httpMerged) != 3 || httpMerged[0].Name != " " || httpMerged[1].Name != "baseline" || httpMerged[2].Name != "custom" {
t.Fatalf("unexpected HTTP checklist merge: %#v", httpMerged)
}
tcpMerged := mergeTCPServiceChecklistDefaults(
[]TCPServiceChecklistCheck{{Name: " "}, {Name: "smtp-custom"}},
[]TCPServiceChecklistCheck{{Name: " "}, {Name: "smtp-baseline"}},
)
if len(tcpMerged) != 3 || tcpMerged[0].Name != " " || tcpMerged[1].Name != "smtp-baseline" || tcpMerged[2].Name != "smtp-custom" {
t.Fatalf("unexpected TCP checklist merge: %#v", tcpMerged)
}
}

1
internal/config/types.go
View File

@ -38,6 +38,7 @@ type Startup struct {
NodeInventoryReachRequiredNodes []string `yaml:"node_inventory_reachability_required_nodes"`
LonghornCryptsetupExemptNodes []string `yaml:"longhorn_cryptsetup_exempt_nodes"`
RequiredNodeLabels map[string]map[string]string `yaml:"required_node_labels"`
RequiredNodeLabelsMode string `yaml:"required_node_labels_mode"`
RequireTimeSync bool `yaml:"require_time_sync"`
TimeSyncWaitSeconds int `yaml:"time_sync_wait_seconds"`
TimeSyncPollSeconds int `yaml:"time_sync_poll_seconds"`

3
internal/config/validate.go
View File

@ -87,6 +87,9 @@ func (c Config) Validate() error {
}
}
}
if c.Startup.RequiredNodeLabelsMode != "enforce" && c.Startup.RequiredNodeLabelsMode != "validate" {
return fmt.Errorf("config.startup.required_node_labels_mode must be enforce or validate")
}
if c.Startup.TimeSyncWaitSeconds <= 0 {
return fmt.Errorf("config.startup.time_sync_wait_seconds must be > 0")
}

4
internal/config/validate_matrix_test.go
View File

@ -32,6 +32,7 @@ func TestValidateRejectsInvalidFieldsMatrix(t *testing.T) {
{"bad_node_inventory_poll", func(c *Config) { c.Startup.NodeInventoryReachPollSeconds = 0 }},
{"bad_empty_node_inventory_required_node", func(c *Config) { c.Startup.NodeInventoryReachRequiredNodes = []string{"titan-0a", ""} }},
{"bad_empty_longhorn_cryptsetup_exempt_node", func(c *Config) { c.Startup.LonghornCryptsetupExemptNodes = []string{"titan-23", ""} }},
{"bad_required_node_labels_mode", func(c *Config) { c.Startup.RequiredNodeLabelsMode = "repair" }},
{"bad_time_sync_wait", func(c *Config) { c.Startup.TimeSyncWaitSeconds = 0 }},
{"bad_time_sync_poll", func(c *Config) { c.Startup.TimeSyncPollSeconds = 0 }},
{"bad_time_sync_quorum", func(c *Config) { c.Startup.TimeSyncMode = "quorum"; c.Startup.TimeSyncQuorum = 0 }},
@ -161,6 +162,9 @@ func TestApplyDefaultsPopulatesZeroConfig(t *testing.T) {
if cfg.Startup.TimeSyncMode == "" || cfg.Startup.EtcdRestoreControlPlane == "" || cfg.Startup.VaultUnsealKeyFile == "" {
t.Fatalf("expected startup defaults to be set")
}
if cfg.Startup.RequiredNodeLabelsMode != "enforce" {
t.Fatalf("expected required node labels mode default enforce, got %q", cfg.Startup.RequiredNodeLabelsMode)
}
if cfg.Startup.PostStartAutoHealSeconds <= 0 || cfg.Startup.DeadNodeCleanupGraceSeconds <= 0 {
t.Fatalf("expected post-start auto-heal defaults to be set")
}

12
scripts/install-config-migration.sh
View File

@ -338,3 +338,15 @@ sanitize_migrated_ananke_config() {
chmod 0640 "${cfg}" || true
fi
}
install_inventory_fragment() {
if [[ -z "${INVENTORY_FRAGMENT:-}" ]]; then
return 0
fi
if [[ ! -f "${INVENTORY_FRAGMENT}" ]]; then
echo "[install] ANANKE_INVENTORY_FRAGMENT does not exist: ${INVENTORY_FRAGMENT}" >&2
exit 1
fi
install -m 0640 "${INVENTORY_FRAGMENT}" "${CONF_DIR}/ananke.inventory.yaml"
echo "[install] installed Terraform-generated inventory fragment to ${CONF_DIR}/ananke.inventory.yaml"
}

2
scripts/install.sh
View File

@ -22,6 +22,7 @@ NUT_PRODUCT_ID="${ANANKE_NUT_PRODUCT_ID:-0601}"
NUT_MONITOR_USER="${ANANKE_NUT_MONITOR_USER:-monuser}"
NUT_MONITOR_PASSWORD="${ANANKE_NUT_MONITOR_PASSWORD:-anankeupsmon}"
FORCE_CONFIG_TEMPLATE="${ANANKE_FORCE_CONFIG_TEMPLATE:-}"
INVENTORY_FRAGMENT="${ANANKE_INVENTORY_FRAGMENT:-}"
ENFORCE_QUALITY_GATE="${ANANKE_ENFORCE_QUALITY_GATE:-1}"
while [[ $# -gt 0 ]]; do
@ -103,6 +104,7 @@ else
fi
migrate_ananke_config
sanitize_migrated_ananke_config
install_inventory_fragment
ensure_ananke_ssh_identity
ensure_ananke_kubeconfig

6
testing/hygiene/in_tree_test_allowlist.txt
View File

@ -20,6 +20,11 @@ internal/cluster/orchestrator_autorepair_proxy_test.go
internal/cluster/orchestrator_autorepair_service_test.go
internal/cluster/orchestrator_critical_endpoint_additional_test.go
internal/cluster/orchestrator_cordon_lease_test.go
internal/cluster/orchestrator_quality_gate_more_closeout_test.go
internal/cluster/orchestrator_quality_gate_storage_closeout_test.go
internal/cluster/orchestrator_quality_gate_closeout_test.go
internal/cluster/orchestrator_quality_gate_final_closeout_test.go
internal/cluster/orchestrator_quality_gate_recovery_closeout_test.go
internal/cluster/orchestrator_test.go
internal/cluster/orchestrator_hardening_test.go
internal/cluster/orchestrator_image_pull_credentials_test.go
@ -28,6 +33,7 @@ internal/cluster/orchestrator_workload_recovery_test.go
internal/cluster/orchestrator_vault_test.go
internal/config/config_test.go
internal/config/load_additional_test.go
internal/config/startup_service_catalog_closeout_test.go
internal/config/validate_matrix_test.go
internal/service/daemon_additional_test.go
internal/service/daemon_coverage_closeout_test.go

62
testing/orchestrator/hooks_ingress_service_matrix_test.go
View File

@ -95,6 +95,68 @@ func TestHookIngressServiceMatrix(t *testing.T) {
}
})
t.Run("required-node-labels-validate-mode-reports-drift-without-applying", func(t *testing.T) {
cfg := lifecycleConfig(t)
cfg.Startup.RequiredNodeLabelsMode = "validate"
cfg.Startup.RequiredNodeLabels = map[string]map[string]string{
"titan-23": {
"topology.kubernetes.io/zone": "lab-a",
},
}
labelCalled := false
run := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
command := name + " " + strings.Join(args, " ")
switch {
case name == "kubectl" && strings.Contains(command, "label node titan-23 --overwrite"):
labelCalled = true
return "", nil
case name == "kubectl" && strings.Contains(command, "get node titan-23 -o json"):
return `{"metadata":{"labels":{"topology.kubernetes.io/zone":"lab-a"}}}`, nil
default:
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
}
}
orch, _ := newHookOrchestrator(t, cfg, run, run)
if err := orch.TestHookEnsureRequiredNodeLabels(context.Background()); err != nil {
t.Fatalf("expected validate mode success, got %v", err)
}
if labelCalled {
t.Fatalf("validate mode must not apply node labels")
}
runDrift := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
command := name + " " + strings.Join(args, " ")
if name == "kubectl" && strings.Contains(command, "label node titan-23 --overwrite") {
t.Fatalf("validate mode drift must not apply node labels, got %q", command)
}
if name == "kubectl" && strings.Contains(command, "get node titan-23 -o json") {
return `{"metadata":{"labels":{"topology.kubernetes.io/zone":"lab-b"}}}`, nil
}
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
}
orchDrift, _ := newHookOrchestrator(t, cfg, runDrift, runDrift)
err := orchDrift.TestHookEnsureRequiredNodeLabels(context.Background())
if err == nil || !strings.Contains(err.Error(), "required node label drift") {
t.Fatalf("expected validate mode drift error, got %v", err)
}
cfg.Startup.NodeInventoryReachRequiredNodes = []string{"titan-db"}
runAbsent := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
command := name + " " + strings.Join(args, " ")
if name == "kubectl" && strings.Contains(command, "label node titan-23 --overwrite") {
t.Fatalf("validate mode absent node must not apply node labels, got %q", command)
}
if name == "kubectl" && strings.Contains(command, "get node titan-23 -o json") {
return "", errors.New("Error from server (NotFound): nodes \"titan-23\" not found")
}
return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
}
orchAbsent, _ := newHookOrchestrator(t, cfg, runAbsent, runAbsent)
if err := orchAbsent.TestHookEnsureRequiredNodeLabels(context.Background()); err != nil {
t.Fatalf("expected validate mode absent non-core node to be skipped, got %v", err)
}
})
t.Run("ingress-discovery-checklist-and-heal", func(t *testing.T) {
tlsServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)