From 72ce4a942c24de09c19bb5b1888bf99b82f232bc Mon Sep 17 00:00:00 2001 From: codex Date: Tue, 7 Jul 2026 18:29:20 -0300 Subject: [PATCH] recovery: harden post-start auto-heal --- cmd/ananke/command_handlers.go | 22 + cmd/ananke/command_handlers_autoheal_test.go | 40 ++ cmd/ananke/command_handlers_injection_test.go | 2 + cmd/ananke/main_dispatch.go | 8 + cmd/ananke/main_dispatch_additional_test.go | 12 +- docs/ananke-hardening-prompt-2026-07-07.md | 465 ++++++++++++++++++ docs/ananke-recovery-2026-07-07.md | 207 ++++++++ .../cluster/orchestrator_access_fluxsource.go | 9 +- internal/cluster/orchestrator_autorepair.go | 51 +- .../orchestrator_autorepair_proxy_test.go | 13 +- .../orchestrator_autorepair_service_test.go | 64 +++ .../cluster/orchestrator_autorepair_test.go | 71 +++ internal/cluster/orchestrator_core.go | 9 + .../orchestrator_critical_endpoints.go | 136 ++++- .../orchestrator_critical_probe_repair.go | 144 ++++++ .../cluster/orchestrator_critical_vault.go | 25 +- .../cluster/orchestrator_hardening_test.go | 455 +++++++++++++++++ .../cluster/orchestrator_host_privilege.go | 331 +++++++++++++ .../cluster/orchestrator_image_pull_dns.go | 88 ++++ .../orchestrator_longhorn_reconcile.go | 265 ++++++++++ .../cluster/orchestrator_longhorn_recovery.go | 32 +- .../orchestrator_node_runtime_recovery.go | 162 ++++++ .../cluster/orchestrator_poststart_exec.go | 129 +++-- .../cluster/orchestrator_service_stability.go | 9 +- .../cluster/orchestrator_stale_rwo_owner.go | 319 ++++++++++++ .../cluster/orchestrator_storage_types.go | 20 +- .../cluster/orchestrator_tcp_service_check.go | 164 ++++++ internal/cluster/orchestrator_vault_health.go | 74 +++ .../orchestrator_workload_convergence.go | 28 ++ internal/config/apply_defaults.go | 25 + internal/config/config_test.go | 65 +++ internal/config/defaults.go | 7 + internal/config/load.go | 39 ++ internal/config/startup_service_catalog.go | 91 ++++ internal/config/testing_hooks.go | 16 + internal/config/types.go | 24 + internal/config/validate.go | 45 ++ internal/config/validate_matrix_test.go | 18 + testing/config/config_quality_matrix_test.go | 61 ++- testing/hygiene/in_tree_test_allowlist.txt | 3 + testing/orchestrator/drill_lifecycle_test.go | 4 +- testing/orchestrator/drill_recovery_test.go | 7 +- .../hooks_access_deep_matrix_test.go | 2 +- ...ooks_access_vault_lifecycle_matrix_test.go | 2 +- .../hooks_branch_saturation_test.go | 2 +- .../hooks_health_failure_matrix_test.go | 8 + .../orchestrator/hooks_health_storage_test.go | 5 +- .../hooks_lowfile_coverage_test.go | 8 +- .../orchestrator/hooks_lowfunc_matrix_test.go | 2 +- ...ooks_vault_lifecycle_branch_matrix_test.go | 3 +- ...ooks_vault_poststart_branch_matrix_test.go | 3 +- 51 files changed, 3669 insertions(+), 125 deletions(-) create mode 100644 cmd/ananke/command_handlers_autoheal_test.go create mode 100644 docs/ananke-hardening-prompt-2026-07-07.md create mode 100644 docs/ananke-recovery-2026-07-07.md create mode 100644 internal/cluster/orchestrator_autorepair_service_test.go create mode 100644 internal/cluster/orchestrator_critical_probe_repair.go create mode 100644 internal/cluster/orchestrator_hardening_test.go create mode 100644 internal/cluster/orchestrator_host_privilege.go create mode 100644 internal/cluster/orchestrator_image_pull_dns.go create mode 100644 internal/cluster/orchestrator_longhorn_reconcile.go create mode 100644 internal/cluster/orchestrator_node_runtime_recovery.go create mode 100644 internal/cluster/orchestrator_stale_rwo_owner.go create mode 100644 internal/cluster/orchestrator_tcp_service_check.go create mode 100644 internal/cluster/orchestrator_vault_health.go diff --git a/cmd/ananke/command_handlers.go b/cmd/ananke/command_handlers.go index dfef4fb..1c8299d 100644 --- a/cmd/ananke/command_handlers.go +++ b/cmd/ananke/command_handlers.go @@ -56,6 +56,9 @@ var ( etcdRestoreOrchestratorCommand = func(ctx context.Context, orch *cluster.Orchestrator, opts cluster.EtcdRestoreOptions) error { return orch.EtcdRestore(ctx, opts) } + autoHealOrchestratorCommand = func(ctx context.Context, orch *cluster.Orchestrator) error { + return orch.RunPostStartAutoHeal(ctx) + } daemonRunCommand = func(ctx context.Context, daemon *service.Daemon) error { return daemon.Run(ctx) } readIntentCommand = state.ReadIntent writeIntentCommand = state.MustWriteIntent @@ -132,6 +135,25 @@ func runStartup(logger *log.Logger, args []string) error { }) } +// runAutoHeal runs one orchestration or CLI step. +// Signature: runAutoHeal(logger *log.Logger, args []string) error. +// Why: operators need a narrow way to run Ananke's daemon repair pass once +// without invoking a full startup or depending on the UPS daemon loop. +func runAutoHeal(logger *log.Logger, args []string) error { + fs := flag.NewFlagSet("auto-heal", flag.ExitOnError) + configPath := fs.String("config", "/etc/ananke/ananke.yaml", "Path to config file") + execute := fs.Bool("execute", false, "Actually execute changes (default dry-run)") + _ = fs.Parse(args) + + _, orch, err := buildOrchestratorCommand(logger, *configPath, !*execute) + if err != nil { + return err + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + return autoHealOrchestratorCommand(ctx, orch) +} + // runShutdown runs one orchestration or CLI step. // Signature: runShutdown(logger *log.Logger, args []string) error. // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve. diff --git a/cmd/ananke/command_handlers_autoheal_test.go b/cmd/ananke/command_handlers_autoheal_test.go new file mode 100644 index 0000000..92e9185 --- /dev/null +++ b/cmd/ananke/command_handlers_autoheal_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "context" + "io" + "log" + "testing" + + "scm.bstein.dev/bstein/ananke/internal/cluster" + "scm.bstein.dev/bstein/ananke/internal/config" +) + +// TestRunAutoHealInvokesPostStartRepair runs one orchestration or CLI step. +// Signature: TestRunAutoHealInvokesPostStartRepair(t *testing.T). +// Why: the one-shot auto-heal command must call the same repair entrypoint as +// the daemon loop so operators can run Ananke's post-start repairs directly. +func TestRunAutoHealInvokesPostStartRepair(t *testing.T) { + restore := stubCommandHandlerHooks() + defer restore() + + called := false + buildOrchestratorCommand = func(_ *log.Logger, _ string, dryRun bool) (config.Config, *cluster.Orchestrator, error) { + cfg := minimalHandlerConfig(t) + return cfg, newTestOrchestrator(cfg, dryRun), nil + } + autoHealOrchestratorCommand = func(_ context.Context, orch *cluster.Orchestrator) error { + called = true + if orch == nil { + t.Fatalf("expected orchestrator") + } + return nil + } + logger := log.New(io.Discard, "", 0) + if err := runAutoHeal(logger, []string{"--config", "/ignored.yaml", "--execute"}); err != nil { + t.Fatalf("expected auto-heal success, got %v", err) + } + if !called { + t.Fatalf("expected auto-heal orchestrator hook to be called") + } +} diff --git a/cmd/ananke/command_handlers_injection_test.go b/cmd/ananke/command_handlers_injection_test.go index 454b5f8..5b8390b 100644 --- a/cmd/ananke/command_handlers_injection_test.go +++ b/cmd/ananke/command_handlers_injection_test.go @@ -403,6 +403,7 @@ func stubCommandHandlerHooks() func() { prevStartup := startupOrchestratorCommand prevShutdown := shutdownOrchestratorCommand prevRestore := etcdRestoreOrchestratorCommand + prevAutoHeal := autoHealOrchestratorCommand prevDaemon := daemonRunCommand prevReadIntent := readIntentCommand prevWriteIntent := writeIntentCommand @@ -416,6 +417,7 @@ func stubCommandHandlerHooks() func() { startupOrchestratorCommand = prevStartup shutdownOrchestratorCommand = prevShutdown etcdRestoreOrchestratorCommand = prevRestore + autoHealOrchestratorCommand = prevAutoHeal daemonRunCommand = prevDaemon readIntentCommand = prevReadIntent writeIntentCommand = prevWriteIntent diff --git a/cmd/ananke/main_dispatch.go b/cmd/ananke/main_dispatch.go index 66ef0ab..851bf12 100644 --- a/cmd/ananke/main_dispatch.go +++ b/cmd/ananke/main_dispatch.go @@ -11,6 +11,7 @@ var ( shutdownCommand = runShutdown etcdRestoreCommand = runEtcdRestore daemonCommand = runDaemon + autoHealCommand = runAutoHeal statusCommand = runStatus intentCommand = runIntent ) @@ -54,6 +55,11 @@ func runMain(logger *log.Logger, args []string) int { logger.Printf("daemon failed: %v", err) return 1 } + case "auto-heal": + if err := autoHealCommand(logger, args[2:]); err != nil { + logger.Printf("auto-heal failed: %v", err) + return 1 + } case "status": if err := statusCommand(logger, args[2:]); err != nil { logger.Printf("status failed: %v", err) @@ -88,6 +94,7 @@ Commands: shutdown Perform graceful cluster shutdown etcd-restore Restore etcd from snapshot on a control plane daemon Monitor UPS and auto-trigger shutdown + auto-heal Run one post-start auto-heal pass status Print current ananke status and estimates intent Read or manually set intent state @@ -96,6 +103,7 @@ Examples: ananke shutdown --config /etc/ananke/ananke.yaml --execute --mode cluster-only --reason "manual-maintenance" ananke etcd-restore --config /etc/ananke/ananke.yaml --execute ananke daemon --config /etc/ananke/ananke.yaml + ananke auto-heal --config /etc/ananke/ananke.yaml --execute ananke status --config /etc/ananke/ananke.yaml ananke intent --config /etc/ananke/ananke.yaml --set normal --reason "manual-clear" --execute diff --git a/cmd/ananke/main_dispatch_additional_test.go b/cmd/ananke/main_dispatch_additional_test.go index 4d3662b..1c464ff 100644 --- a/cmd/ananke/main_dispatch_additional_test.go +++ b/cmd/ananke/main_dispatch_additional_test.go @@ -71,10 +71,11 @@ func TestRunMainDispatchSuccessPaths(t *testing.T) { func(_ *log.Logger, _ []string) error { return nil }, func(_ *log.Logger, _ []string) error { return nil }, func(_ *log.Logger, _ []string) error { return nil }, + func(_ *log.Logger, _ []string) error { return nil }, ) defer restore() - commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "status", "intent", "help", "-h", "--help"} + commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "auto-heal", "status", "intent", "help", "-h", "--help"} for _, command := range commands { code := runMain(logger, []string{"ananke", command}) if code != 0 { @@ -96,10 +97,11 @@ func TestRunMainDispatchErrorPaths(t *testing.T) { func(_ *log.Logger, _ []string) error { return fail }, func(_ *log.Logger, _ []string) error { return fail }, func(_ *log.Logger, _ []string) error { return fail }, + func(_ *log.Logger, _ []string) error { return fail }, ) defer restore() - commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "status", "intent"} + commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "auto-heal", "status", "intent"} for _, command := range commands { code := runMain(logger, []string{"ananke", command}) if code != 1 { @@ -122,13 +124,14 @@ func TestRunMainUsageAndUnknownPaths(t *testing.T) { } // stubDispatchCommands runs one orchestration or CLI step. -// Signature: stubDispatchCommands(startup, shutdown, etcdRestore, daemon, status, intent func(*log.Logger, []string) error) func(). +// Signature: stubDispatchCommands(startup, shutdown, etcdRestore, daemon, autoHeal, status, intent func(*log.Logger, []string) error) func(). // Why: keeps command-var overrides scoped and safely restorable across tests. func stubDispatchCommands( startup, shutdown, etcdRestore, daemon, + autoHeal, status, intent func(*log.Logger, []string) error, ) func() { @@ -136,6 +139,7 @@ func stubDispatchCommands( prevShutdown := shutdownCommand prevEtcdRestore := etcdRestoreCommand prevDaemon := daemonCommand + prevAutoHeal := autoHealCommand prevStatus := statusCommand prevIntent := intentCommand @@ -143,6 +147,7 @@ func stubDispatchCommands( shutdownCommand = shutdown etcdRestoreCommand = etcdRestore daemonCommand = daemon + autoHealCommand = autoHeal statusCommand = status intentCommand = intent @@ -151,6 +156,7 @@ func stubDispatchCommands( shutdownCommand = prevShutdown etcdRestoreCommand = prevEtcdRestore daemonCommand = prevDaemon + autoHealCommand = prevAutoHeal statusCommand = prevStatus intentCommand = prevIntent } diff --git a/docs/ananke-hardening-prompt-2026-07-07.md b/docs/ananke-hardening-prompt-2026-07-07.md new file mode 100644 index 0000000..c370bbb --- /dev/null +++ b/docs/ananke-hardening-prompt-2026-07-07.md @@ -0,0 +1,465 @@ +# Prompt For Ananke Recovery Hardening + +You are a fresh, stateless Codex session working in the Ananke repository. Your goal is to implement, test, and validate generalized recovery improvements in Ananke based on a real Titan cluster power-loss incident on 2026-07-07. Treat this as production reliability work: read the repo first, follow existing code style and architecture, keep changes cohesive, and solve failure classes categorically rather than hard-coding node names, namespaces, or one-off incidents. + +## Mission + +Make Ananke resilient and automatic during post-power-loss cluster recovery. After your changes, one Ananke startup/recovery process should be able to: + +- Bring Vault and critical dependencies back safely. +- Repair or escalate wedged k3s/containerd nodes using existing Ananke SSH/host access. +- Use sudo credentials from Vault, securely and noninteractively, for host repair and diagnostics. +- Avoid leaving Longhorn storage nodes cordoned indefinitely. +- Detect Kubernetes/Longhorn readiness mismatches and repair label/manager/VolumeAttachment drift. +- Resolve stale pod/PVC ownership safely, with different behavior for sidecar-only stale owners versus still-running application containers that mount the PVC. +- Continue post-success convergence monitoring long enough to catch late node/runtime/storage wedges. +- Report concise, actionable blockers without noisy repeated pod recycling. + +Do not implement special cases for `titan-04`, `titan-05`, `titan-15`, `titan-17`, `titan-22`, Firefly, Jellyfin, or any specific app. Use those names only as incident fixtures/test names. The code should operate on generic managed nodes, generic Longhorn nodes, generic Kubernetes workloads, and generic RWO PVCs. + +## Incident Context + +The Titan cluster lost power for a few minutes. Ariadne may have started shutdown behavior. The operator asked for a single Ananke process to bring the cluster back up and learn from any manual help required. + +Ananke did a lot correctly: + +- It recovered Vault and unsealed it. +- It recovered critical workloads, Vault injector, Postgres, Keycloak, OpenSearch, oauth2 logs, CoreDNS, metrics-server, and many app pods. +- It ran a single startup workflow and eventually completed successfully. +- It recycled many stuck pods and ran several checks. + +But Ananke needed manual assistance in multiple reusable failure classes: + +- Node runtime wedges where Kubernetes readiness did not fully reflect kubelet/containerd health. +- Host repairs blocked by sudo privilege gaps. +- Longhorn nodes cordoned by Kubernetes remained unavailable for storage scheduling. +- Longhorn readiness diverged from Kubernetes readiness. +- Missing node labels caused Longhorn manager pods to disappear. +- Stale terminating pods held RWO volumes and blocked replacement pods. +- Repeated pod deletes did not resolve storage ownership or runtime reservation issues. +- Ananke declared startup success before all late runtime/storage fallout was resolved. + +## Live Incident Facts To Encode As Tests + +Use these as fixtures for tests and documentation, not as hard-coded behavior: + +- `titan-05`: k3s-agent restart hung. Host SSH worked. `k3s-agent.service` entered `deactivating`, `final-sigkill`, `activating/start`, and `Result=timeout`. Kubelet port `10250` stayed closed. Kubernetes heartbeat was stale with `node.kubernetes.io/unreachable` taints. In-cluster helper pod could not start. Manual controlled reboot recovered it. +- `titan-04`: after Ananke reported startup success, the node was Kubernetes `Ready` and kubelet `10250` initially open, but new pods failed with `FailedCreatePodSandBox`, `failed to reserve sandbox name`, and `context deadline exceeded`. A non-blocking `k3s-agent` restart then stuck in `deactivating/stop-sigkill`, kubelet `10250` closed, Kubernetes moved to `Ready=Unknown`, and a controlled reboot recovered it. +- `titan-07`: a pod looked running in the API, but `kubectl exec` failed with `cannot exec in a deleted state`; restarting k3s-agent and recreating the pod fixed it. +- `titan-14`: a replacement pod hit `failed to reserve container name` / `CreateContainerError`; k3s-agent restart and rerunning the in-cluster helper fixed it. Later another pod on the same node showed the same sandbox reservation pattern, but then progressed after enough time. Tests should distinguish transient progress from a stall. +- `titan-15` and `titan-17`: Ananke cordoned them after encrypted Longhorn mounts exposed missing `cryptsetup`. Longhorn itself had `allowScheduling=true` and ready disks, but Longhorn `Schedulable=False` because Kubernetes had cordoned the nodes. Manual `kubectl uncordon titan-15 titan-17` restored Kubernetes scheduling and Longhorn schedulability. Ananke had attempted `cryptsetup-bin` repair but failed because `atlas` sudo required a password. +- `titan-22`: Kubernetes `Ready=True`, CSI plugin and engine images were running, but Longhorn node `Ready=False` because `longhorn-manager` pod was missing. The root cause was missing node label `longhorn-host=true`, while the `longhorn-manager` DaemonSet selected `longhorn-host=true`. Manual `kubectl label node titan-22 longhorn-host=true --overwrite` started `longhorn-manager` and Longhorn node Ready became true. A pending wallet pod still carried an old failed VolumeAttachment until the pod was deleted/recreated. +- Firefly stale owner, safe-to-force variant: `finance/firefly-...-q8rtq` was terminating for more than 25 minutes, had no finalizers, and only a Vault sidecar remained running. Replacement pod was blocked by `Multi-Attach` for an RWO PVC. Force-deleting the stale pod cleared ownership and allowed Longhorn attach to the replacement. +- Firefly stale owner, unsafe-to-force variant: a later terminating Firefly pod still had the main application container running and mounting the RWO PVC. It was not safe to blindly force-delete. Waiting allowed kubelet to finish termination, then Longhorn attached the volume to the replacement. +- Manual runtime inspection attempt on `titan-06`: `sudo -n crictl ...` failed because `atlas` required a sudo password. Ananke is expected to have access to sudo passwords in Vault, so this must be designed as a first-class Ananke capability rather than manual operator shell access. +- Vault health check robustness: at one point `kubectl exec vault-0 -- vault status` was killed, but HTTP health via Kubernetes pod proxy showed `initialized=true`, `sealed=false`, `standby=false`. Startup should not fail solely on transient `kubectl exec` failure when endpoint and HTTP health prove Vault is available. +- Image/DNS issues: several image pulls failed with `lookup registry-1.docker.io`, `lookup auth.docker.io`, or `lookup production.cloudfront.docker.com: Try again`, then later succeeded. Ananke repeatedly recycling unchanged ImagePullBackOff pods created noise. + +## Required Implementation Areas + +### 1. Secure Sudo Capability From Vault + +Ananke should be able to run bounded host diagnostics and repairs that require sudo. + +Requirements: + +- Discover and use the existing Vault integration and secret naming conventions already present in the repo/config. Do not invent a new hard-coded secret path if the repo has one. +- Support noninteractive sudo for configured managed nodes. +- Never log sudo passwords, Vault tokens, rendered secrets, command input containing secrets, or full environment dumps. +- Provide a preflight check: for each managed node, can Ananke retrieve host sudo material and can it run a harmless command such as `sudo -S -p '' true` with a timeout? +- If credentials are missing, wrong, or Vault is unavailable, Ananke should report `host-privilege-unavailable` with node, intended action, and secret lookup class, not spin. +- Use strict command allowlists for privileged commands. The recovery path should not become arbitrary remote root execution. +- Use bounded timeouts for all host commands and classify timeout versus auth failure versus command failure. +- Prefer safe wrappers for: + - `systemctl show/is-active/restart --no-block k3s-agent` + - `systemctl reboot` when configured and escalated + - `crictl ps/pods/inspect/stop` for recovery diagnostics and controlled container stop + - package checks/install for `cryptsetup-bin`, `open-iscsi`, `nfs-common`, `dmsetup` + - `modprobe dm_crypt` or equivalent kernel-module validation where appropriate + +Tests: + +- Vault lookup succeeds and sudo command succeeds. +- Vault lookup succeeds but sudo password is wrong. +- Vault lookup missing. +- Vault unavailable while Vault HTTP health is also degraded. +- Vault unavailable but action does not require sudo. +- Command times out. +- Command returns nonzero. +- Logs are scanned to ensure no password appears. + +### 2. Managed Node Runtime Recovery Primitive + +Implement a generic state machine for node runtime recovery. + +Detection signals: + +- Kubernetes node `Ready=Unknown` or stale heartbeat. +- Kubernetes node `Ready=True` but pods on that node show repeated: + - `FailedCreatePodSandBox` + - `failed to reserve sandbox name` + - `failed to reserve container name` + - `CreateContainerError` + - `FailedKillPod` + - long-lived `ContainerCreating` or `PodInitializing` with no progress + - `kubectl exec`/logs failures indicating deleted runtime state +- Host SSH works but kubelet port `127.0.0.1:10250` is closed or slow. +- `systemctl show k3s-agent` reports `deactivating`, `stop-sigterm`, `stop-sigkill`, `final-sigterm`, `final-sigkill`, `activating/start`, or `Result=timeout`. +- Existing in-cluster maintenance helper pod for the node cannot start, stays `ContainerCreating`, or cannot be scheduled because the node runtime is already unhealthy. + +Escalation ladder: + +- Open a single node recovery incident and suppress noisy repeated pod recycling for pods on that node while host repair is in progress. +- Cordon the node before host-level intervention unless it is already unreachable/tainted. +- Try in-cluster maintenance helper if kubelet/runtime can create pods. +- If helper cannot run, use Ananke SSH route and sudo from Vault. +- Prefer `systemctl --no-block restart k3s-agent`; never use a blocking restart over SSH as the primary recovery action. +- Wait a bounded interval and require both: + - Kubernetes Ready=True with fresh heartbeat + - local kubelet `10250` open +- If k3s-agent remains stuck in stop/kill/activating states or heartbeats remain stale, escalate to configured controlled reboot. +- After reboot, wait for SSH, k3s-agent active/running, kubelet port open, Kubernetes Ready=True, and unreachable taints cleared. +- Uncordon only after the node is healthy and after storage-specific checks pass. +- Continue this monitoring after startup success for a configurable post-success convergence window. + +Tests: + +- Ready node with repeated sandbox reservation events triggers recovery. +- Ready node with one transient sandbox event and then progress does not trigger restart. +- Unknown node with SSH unavailable reports external blocker without repeated action. +- Unknown node with SSH available, kubelet closed, k3s-agent stuck, helper unavailable escalates to reboot. +- Non-blocking restart recovers node. +- Non-blocking restart moves service to `deactivating/stop-sigkill`; reboot escalation occurs. +- Reboot command accepted but SSH does not return within timeout. +- Reboot returns and all readiness conditions clear. +- False positive prevention: node Ready=True, kubelet open, events old/stale only, no action. +- Idempotency: repeated Ananke runs do not issue repeated restarts/reboots while an incident is active. + +### 3. Longhorn And Kubernetes Readiness Reconciliation + +Ananke must not treat Kubernetes node Ready as sufficient for storage workloads. + +Implement a Longhorn readiness reconciler: + +- Read Kubernetes Nodes, Longhorn Node CRs, Longhorn manager DaemonSet, Longhorn manager pods, CSI pods, engine-image pods, Volume CRs, VolumeAttachments, and relevant PVC-bound pods. +- Compare: + - Kubernetes Ready/SchedulingDisabled/taints + - Node labels required by Longhorn manager DaemonSet, especially `longhorn-host=true` + - Longhorn Node `Ready` and `Schedulable` conditions + - Longhorn Node `spec.allowScheduling` + - Longhorn manager pod presence on nodes where Longhorn Node exists or where PVC workloads are allowed + - CSI plugin readiness + - VolumeAttachment errors that reference stale Longhorn node readiness +- Detect label drift: Longhorn node exists and CSI/engine pods exist, but manager pod is missing because the node no longer matches the DaemonSet selector. +- Repair label drift only if it is safe: + - There is a known desired label source in repo/config/state, or + - Longhorn Node CR exists for that Kubernetes node and existing Longhorn metadata indicates the node is intended to be a Longhorn node. + - Emit exactly what label will be restored and why. +- If safe repair cannot be proven, cordon or avoid scheduling PVC workloads to the node and report a blocker. +- After restoring labels or manager pods, wait for Longhorn Node Ready=True. +- For VolumeAttachments that failed because Longhorn thought a now-ready node was not ready, trigger a bounded retry: + - Prefer deleting/recreating the controller-owned Pending pod if that is the established pattern. + - Delete stale failed VolumeAttachment only if Kubernetes controller semantics and repo policy allow it. + - Never delete PV/PVC/Longhorn Volume data. + +Specific expectations from incident: + +- `titan-22` class: Kubernetes Ready, missing `longhorn-host=true`, no manager pod, Longhorn Node Ready=False ManagerPodMissing. Ananke should restore label or report exact unsafe reason, then wait for manager pod and Longhorn Ready. +- `titan-15`/`titan-17` class: Kubernetes cordon makes Longhorn Schedulable=False despite Longhorn `allowScheduling=true` and ready disks. Ananke should not leave them cordoned after the blocking repair condition is resolved or after operator-approved override. + +Tests: + +- Longhorn manager selector changes are handled generically. +- Missing Longhorn label restored from desired state. +- Missing Longhorn label not restored if node lacks Longhorn Node CR and no desired-state evidence exists. +- Longhorn Node Ready=False ManagerPodMissing becomes Ready after label restoration and manager pod startup. +- VolumeAttachment error due stale Longhorn node readiness is retried after Longhorn Ready. +- Kubernetes cordoned node causes Longhorn Schedulable=False and is uncordoned after repair. +- Kubernetes cordoned node remains cordoned if required package/kernel preflight still fails and no override is configured. +- `allowScheduling=false` is respected; Ananke should not enable Longhorn disk scheduling unless explicitly configured, but it may still ensure manager/CSI readiness for attachment if workloads are allowed there. + +### 4. Cordon/Uncordon And Host Package Repair Policy + +Ananke cordoned `titan-15` and `titan-17` because encrypted Longhorn mounts exposed missing `cryptsetup`; it failed to install because sudo required a password. This must become a closed-loop repair. + +Requirements: + +- Distinguish: + - Kubernetes scheduling cordon + - Longhorn disk scheduling (`allowScheduling`) + - Longhorn node readiness/schedulability + - ordinary `PreferNoSchedule` taints used as soft placement hints +- If Ananke cordons a node, it must record why, when, what remediation is required, and what condition will uncordon it. +- If the remediation is package/module installation, use Vault-backed sudo and verify the result. +- If package repair succeeds, uncordon automatically and verify Longhorn Schedulable=True. +- If package repair fails due missing sudo secret, wrong sudo password, package manager lock, apt/dpkg failure, or network outage, report the exact class and do not leave the node silently disabled. +- Support an operator override policy for "uncordon for Longhorn availability even if encrypted workload preflight is not perfect", but this must be explicit/configured and logged. +- Do not remove existing soft taints such as `atlas.bstein.dev/spillover=true:PreferNoSchedule` or `longhorn=true:PreferNoSchedule` unless the repo’s desired-state policy says to. + +Tests: + +- Missing cryptsetup with valid sudo installs package, loads module if needed, uncordons node. +- Missing cryptsetup with missing sudo secret reports privilege blocker and leaves cordon with reason. +- Node already has package but missing kernel module: module load path tested. +- Node has Longhorn ready disks but is Kubernetes cordoned: Longhorn schedulability blocker is reported. +- Uncordon preserves unrelated soft taints. +- Re-running recovery is idempotent. + +### 5. Stale RWO PVC Owner Recovery + +Implement a generic stale-owner recovery flow for controller-owned pods and RWO PVCs. + +Detection: + +- Old pod has `metadata.deletionTimestamp` older than threshold. +- Replacement pod from same controller exists and is pending/initializing. +- Replacement events include `Multi-Attach`, `FailedAttachVolume`, `Volume is already used by pod(s)`, or `Volume is already exclusively attached to one node`. +- PVC is `ReadWriteOnce` or equivalent single-writer mode. +- Old and new pods are on different nodes. +- Longhorn Volume/VolumeAttachment shows old node still owns attachment or new attach blocked. +- Ordinary pod deletes have been attempted repeatedly without changing ownership. + +Recovery grouping: + +- Group old pod, replacement pod, controller, PVC, PV, VolumeAttachment(s), Longhorn Volume, old node, new node, and relevant events into one incident. +- Emit one concise operator record. +- Stop repeated blind pod deletes while the incident is active. + +Safety decisions: + +- Inspect old pod container statuses and volume mounts. +- Classify containers: + - PVC-writing application containers: mount the blocked PVC. + - Sidecars that do not mount the blocked PVC: Vault agent, projected service account, emptyDir-only, etc. + - Unknown containers: treat as unsafe. +- If only sidecars remain and no container mounts the PVC, force-delete the stale pod after a bounded wait. +- If an application container still runs and mounts the PVC, do not blindly force-delete. Instead: + - Wait for kubelet termination if it is making progress. + - If stalled and host runtime inspection is available through Vault-backed sudo, inspect and optionally stop the specific container through a controlled allowlisted path. + - If Ananke cannot prove the PVC-writing container is stopped, report `unsafe-stale-owner` and do not clear the API object. +- If old node kubelet/runtime is unhealthy, route through managed node recovery before stale-owner cleanup. + +Tests: + +- Sidecar-only stale pod force-clears and replacement attaches. +- Live app container with blocked PVC waits and does not force-delete. +- Live app container eventually exits; replacement attaches without force-delete. +- Host runtime inspection unavailable due sudo privilege reports unsafe blocker. +- Host runtime inspection available and controlled container stop succeeds. +- Longhorn detach/attach follows cleanup. +- Replacement readiness is rechecked after attach. +- Multiple PVCs in one pod are handled. +- Non-RWO PVC is not processed by this recovery path. + +### 6. Vault Health Robustness + +Startup should not fail because one Vault probe path is flaky if other authoritative health signals prove Vault is usable. + +Requirements: + +- Prefer multiple health signals: + - pod Running/Ready + - service endpoints populated + - HTTP `/v1/sys/health` through service or pod proxy + - `vault status` through exec as one signal, not the only signal +- If `kubectl exec vault status` is killed/transiently fails but HTTP health reports initialized/unsealed/active or standby-acceptable per config, classify as transient exec failure and continue with warning. +- If HTTP health and exec disagree, report clearly and retry with bounded backoff. +- Never log tokens. + +Tests: + +- exec killed but HTTP health unsealed => startup continues with warning. +- exec says sealed and HTTP says sealed => startup blocks/unseals. +- endpoints missing => startup waits. +- HTTP unavailable but exec succeeds => startup may proceed if policy allows. + +### 7. Image Pull And DNS Failure Classification + +Ananke should not repeatedly recycle pods whose only issue is unchanged image pull or DNS failure. + +Requirements: + +- Detect ImagePullBackOff/ErrImagePull events caused by DNS/registry lookup. +- Run separate DNS checks: + - CoreDNS pods ready + - in-cluster DNS query from a known diagnostic pod if available + - node host DNS or kubelet image pull DNS failures grouped by node + - registry reachability by hostname class, not one app +- If the image pull failure is unchanged and recent, do not repeatedly delete the pod. +- Emit a compact blocker: image, registry host, node, last error, and whether DNS is cluster-wide or node-specific. +- Retry only after DNS/registry health changes or after bounded interval. + +Tests: + +- Docker Hub auth DNS failure is classified. +- Private registry DNS failure is classified. +- One-off pull failure that later succeeds is not over-escalated. +- Repeated pod recycle is suppressed. + +### 8. Startup Completion And Post-Success Monitoring + +Ananke declared `startup_status=success` while late runtime/storage issues still surfaced. Make completion smarter. + +Requirements: + +- Startup can reach "cluster core recovered" while still monitoring app convergence. +- Do not mark total recovery complete until: + - critical services pass + - all managed nodes are either healthy, intentionally excluded, or explicitly blocked + - Longhorn node readiness/schedulability matches desired policy + - no active node-runtime incidents exist + - no unsafe stale-owner incidents exist +- Add a post-success watch window for late events: sandbox reservation, FailedKillPod, kubelet metrics scrape failures, Longhorn manager missing, VolumeAttachment errors. +- Distinguish statuses: + - `core_recovered` + - `converging` + - `success` + - `success_with_noncritical_blockers` + - `blocked_requires_operator` +- Report app-level blockers separately from cluster-level blockers. + +Tests: + +- Critical services pass but node runtime wedge appears during post-success window => not final success until resolved. +- Noncritical app image pull blocker can be `success_with_noncritical_blockers`. +- Unsafe stale RWO owner blocks full success. +- All clear => success. + +## Status And Operator Reporting + +Improve operator output. The user should not need to read journals to know what is happening. + +Status should include: + +- Whether a startup/recovery workflow is already running. +- Current phase and elapsed time. +- Current incidents grouped by type: + - node runtime recovery + - Longhorn/Kubernetes drift + - stale RWO owner + - Vault health + - image/DNS pull + - host privilege/preflight +- For each incident: resource, trigger signals, action taken, next action, elapsed time, and whether operator input is needed. +- Counts should be accurate and stable. +- Repeated noisy details should be summarized with "same blocker still present for X minutes" instead of repeated lines. + +## Security Constraints + +- Do not log sudo passwords, Vault tokens, unsealed keys, or rendered secret contents. +- Redact secrets in command arguments and environment. +- Use command allowlists for privileged host actions. +- Make destructive operations explicit and guarded: + - reboot only after escalation criteria + - force-delete pod only after stale-owner safety checks + - deleting VolumeAttachments only if repo policy allows and after safeguards +- Keep audit logs of actions: node/resource, reason, command class, result, elapsed time, but not secrets. + +## Implementation Guidance + +Before editing: + +- Read Ananke’s existing startup/recovery code, Kubernetes client abstractions, Vault access layer, SSH/host command layer, Longhorn helpers if any, status/progress writer, and tests. +- Reuse existing abstractions. Do not invent a parallel framework unless necessary. +- Use fake clients and table-driven tests where the repo already uses them. +- Keep changes modular: + - host privilege provider + - node runtime incident detector/reconciler + - Longhorn readiness reconciler + - stale RWO owner reconciler + - image/DNS classifier + - status/progress summarizer +- Ensure all reconcilers are idempotent and bounded. + +Suggested model: + +- Define incident structs with stable keys, timestamps, observed signals, action history, next action, and terminal status. +- Every recovery action should require: + - detection evidence + - policy permission + - safety gate + - timeout + - postcondition check +- Avoid "delete pod until healthy" loops. Replace with evidence-driven remediation. + +## Validation Plan + +Run the repository’s standard checks first, then add targeted tests. + +Required validation: + +- Unit tests for each new classifier/reconciler. +- Tests for redaction/no secret leakage. +- Tests for idempotency on repeated reconcile calls. +- Tests for progress/status output. +- Integration-style tests with fake Kubernetes/Longhorn objects: + - nodes, pods, events, PVCs, PVs, VolumeAttachments, Longhorn Node CRs, Longhorn Volume CRs, DaemonSets, and pods. +- Host command fake tests: + - SSH unavailable + - sudo unavailable + - sudo succeeds + - command timeout + - k3s-agent service states + - reboot accepted/not accepted +- Vault fake tests: + - secret found + - secret missing + - Vault sealed/unavailable + - HTTP health fallback +- End-to-end scenario tests composed from the incident: + - `titan-05` hard runtime wedge -> reboot -> Ready + - `titan-04` post-success sandbox wedge -> restart fails -> reboot -> Ready + - `titan-15`/`titan-17` cryptsetup/cordon flow -> sudo repair or explicit blocker -> uncordon when safe + - `titan-22` missing `longhorn-host` label -> restore label -> manager pod appears -> Longhorn Ready + - Firefly sidecar-only stale owner -> force delete -> attach replacement + - Firefly live app container stale owner -> wait/report unsafe, no force delete + - Vault exec killed but HTTP healthy -> continue + - DNS image pull errors -> classify without repeated recycle + +Run commands appropriate for the repo, likely including: + +```bash +go test ./... +go test -race ./... +go vet ./... +``` + +If the repo uses other tooling, discover and run it. If any expensive/integration tests require cluster credentials, document how to run them and ensure fake-client tests cover the logic by default. + +## Acceptance Criteria + +The work is complete only when: + +- Ananke can retrieve and use Vault-backed sudo credentials securely or clearly reports why it cannot. +- Node runtime wedges are detected and recovered by a bounded escalation ladder. +- Ananke can handle Ready nodes that cannot create sandboxes. +- Ananke can recover from k3s-agent stop-sigkill/timeout by controlled reboot when configured. +- Ananke does not leave Longhorn nodes cordoned indefinitely after repair/override conditions are satisfied. +- Ananke detects and fixes or reports Longhorn/Kubernetes drift, including missing manager-pod labels. +- Ananke verifies Longhorn node readiness before trusting Kubernetes node readiness for PVC workloads. +- Ananke handles stale RWO ownership safely, distinguishing sidecar-only stale pods from live PVC-writing app containers. +- Ananke avoids noisy repeated deletion of unchanged image-pull failures. +- Vault health checks tolerate transient exec failures when HTTP health is authoritative. +- Startup success reflects real cluster recovery, with explicit `success_with_noncritical_blockers` or `blocked_requires_operator` states where appropriate. +- All changes are covered by rigorous tests and pass the repo’s standard test suite. + +## What Not To Do + +- Do not hard-code Titan node names, app names, PVC names, or namespaces. +- Do not log secrets. +- Do not delete PVs, PVCs, or Longhorn Volume data. +- Do not force-delete pods with live app containers mounting RWO PVCs unless a controlled, audited stop path has proven storage safety. +- Do not repeatedly restart/reboot nodes without incident state and cooldowns. +- Do not treat Kubernetes Ready as equivalent to Longhorn Ready. +- Do not declare full success while node-runtime or unsafe stale-owner incidents are active. + +## Useful Terminology From The Incident + +- "Node runtime wedge": Kubernetes or host state where kubelet/containerd cannot create/kill sandboxes even if the node appears partially Ready. +- "Stale RWO owner": a terminating pod or runtime survivor that still holds an exclusive PVC and blocks replacement attach. +- "Sidecar-only stale owner": old pod still exists, but only non-PVC sidecars remain; force deletion may be safe after checks. +- "Unsafe stale owner": app container still running and mounting the PVC; wait or controlled stop required. +- "Longhorn/Kubernetes drift": Kubernetes node readiness/labels and Longhorn node readiness/manager state disagree. +- "Host privilege unavailable": Ananke cannot run necessary sudo actions despite needing host-level repair. + +Use this prompt to implement production-grade, categorical recovery logic in Ananke, not one-off incident handling. diff --git a/docs/ananke-recovery-2026-07-07.md b/docs/ananke-recovery-2026-07-07.md new file mode 100644 index 0000000..c68f75e --- /dev/null +++ b/docs/ananke-recovery-2026-07-07.md @@ -0,0 +1,207 @@ +# Ananke Recovery Notes - 2026-07-07 + +## Context + +- Titan cluster lost power briefly; Ariadne may have started shutdown behavior. +- User goal: let one Ananke recovery process bring the cluster back to health, monitor it, and record any manual help or automation gaps. +- Operator view is from `/home/brad/Development/bstein_dev_home`; Kubernetes API is reachable through `/home/brad/titan/kubeconfig`. + +## Current Ananke Run + +- Coordinator host: `titan-db`. +- `ananke.service` daemon is active. +- `ananke-bootstrap.service` is active and running: + - command: `/usr/local/bin/ananke startup --config /etc/ananke/ananke.yaml --execute --force-flux-branch main --auto-peer-failover --peer-wait-seconds 180` + - current run started: `2026-07-07T17:19:50Z` + - current phase observed: `critical-workloads` + - checklist observed: `10 passed / 0 failed / 2 running` + - auto-heals observed: pod recycling for `VaultInitStuck`, `ContainerRuntimeWedge`, `CrashLoopBackOff`, stale terminating pods, and Longhorn attach blocked on unready nodes. + - previous run failed at `2026-07-07T17:19:19Z` waiting on `logging/deployment/oauth2-proxy-logs`; systemd restarted the bootstrap service. + +## Observed Recovery Progress + +- Latest snapshot at approximately `2026-07-07T17:58Z`: + - Ananke is still running in `convergence-checks` with `17/19` checks passed, `0` failed, and `2` running. + - Ananke has recorded `31` auto-heal actions, mostly repeated stuck-pod recycling. + - `titan-22` and `titan-24` both recovered to `Ready=True` with fresh kubelet transitions (`titan-22` at `2026-07-07T17:56:29Z`, `titan-24` at `2026-07-07T17:55:11Z`). + - Metrics server is running and `kubectl top nodes` works for recovered nodes; `titan-22` had just rejoined and metrics may lag briefly. + - Remaining convergence blocker is mostly app workload readiness, led by `finance/firefly`. +- Ananke completed the startup workflow successfully at `2026-07-07T17:59:00Z`: + - `startup_status=success` + - `startup_phase=complete` + - `startup_checklist_total=20 passed=19 failed=0 running=1` + - `startup_auto_heals=33` + - `intent=normal` +- Vault recovered: + - `vault/vault-0` is `1/1 Running` on `titan-18`. + - Vault status reports `initialized=true`, `sealed=false`, `ha_enabled=true`. + - Vault became active at `2026-07-07T17:05:46Z`. + - `vault-k8s-auth-config-autoheal-*` completed. +- Vault injector recovered after `titan-04` kubelet recovery: + - `vault-injector-agent-injector-*` became `1/1 Running`. + - Vault service endpoints and injector webhook endpoints are populated. +- Vault-backed workloads started moving again: + - Ariadne's Vault init authenticated successfully and rendered `/vault/secrets/ariadne-env.sh`. + - Ariadne is now `2/2 Running` on `titan-08`. +- Flux is mostly reconciled, but several app kustomizations are still waiting on app readiness or dependencies. +- Longhorn no longer shows the earlier broad faulted volume picture; observed state later was `41 attached`, `1 attaching`, `47 detached`. + +## Current Known Issues + +- `titan-22` and `titan-24` are still `NotReady` / `Unknown` from Kubernetes. +- `titan-05` is `Unknown` after power-event recovery and k3s-agent restart attempts: + - Ananke SSH route on port `2277` reaches the host. + - `systemctl restart k3s-agent` timed out while stopping the old process. + - systemd reported `Processes still around after SIGKILL`, then spawned a new `/usr/local/bin/k3s agent`. + - `k3s-agent.service` remained `activating/start`; local `127.0.0.1:6444` was open but kubelet `127.0.0.1:10250` was closed. + - Kubernetes node heartbeat stayed stale at `2026-07-07T17:21:57Z` with `node.kubernetes.io/unreachable` taints. + - `maintenance/k3s-agent-restart-*` on `titan-05` stayed `ContainerCreating`, which confirms the in-cluster helper cannot repair this class once the node runtime is wedged. +- `titan-15` and `titan-17` are `Ready,SchedulingDisabled`; Ananke cordoned them because encrypted Longhorn mounts exposed missing `cryptsetup`. +- Ananke tried to install `cryptsetup-bin` on `titan-15` and `titan-17`, but the SSH repair failed because `sudo` required a password for `atlas`. +- Several workloads are still unready while Ananke recycles stale `VaultInitStuck`, `CrashLoopBackOff`, and `ImagePullBackOff` pods. + +## Manual Actions Taken + +- Verified cluster health with `kubectl` and Flux commands. +- Verified Vault directly with `vault status` inside `vault-0`. +- Verified Ananke coordinator service state through `titan-db`. +- Did not start a second Ananke recovery process after discovering the active `ananke-bootstrap.service`. +- Used the existing `maintenance/k3s-agent-restart` DaemonSet as a targeted host-level nudge by deleting its pods on `titan-04`, `titan-05`, `titan-06`, `titan-08`, and `titan-18`; this reruns the DaemonSet's `k3s-agent` restart command on those nodes. +- The in-cluster helper successfully reran on `titan-08`, but helper startup/termination was itself wedged on other affected nodes. Queued direct `systemctl --no-block restart k3s-agent` over Ananke SSH for `titan-05`, `titan-06`, and `titan-18`. A direct restart attempt against `titan-04` hung and was stopped. +- After discovering Ananke's SSH profile uses `atlas@node:2277`, queued `systemctl --no-block restart k3s-agent` on `titan-04` and `titan-05` through the same route. +- `titan-04` recovered to `Ready` after the non-blocking restart and its `maintenance/k3s-agent-restart-*` pod became `Running`. +- `titan-05` did not recover: systemd timed out killing the old `k3s-agent`, left containerd shims behind, and started a new agent process without reopening kubelet port `10250` or restoring node heartbeats. +- A controlled reboot was issued for `titan-05` through Ananke's SSH route at approximately `2026-07-07T17:32Z` because the node met the proposed escalation criteria: stale Kubernetes heartbeat, kubelet `10250` closed, `k3s-agent.service` stuck in `activating/start`, and the in-cluster helper stuck `ContainerCreating`. +- After the reboot command returned, `titan-05` temporarily stopped responding to ping and SSH, then recovered to Kubernetes `Ready=True` with fresh heartbeats and no `unreachable` taints by approximately `2026-07-07T17:35Z`. +- Ananke later detected and logged `cordoned container-runtime-wedged node(s): titan-07 pods=4`; direct host checks showed `titan-07` reachable, `k3s-agent` active/running, and kubelet `10250` open. This should become a false-positive/auto-uncordon test case if Kubernetes readiness remains healthy. +- Later `titan-07` showed a real runtime/API-state split: `oauth2-proxy-logs` reported container state `running`, but `kubectl exec` failed with `cannot exec in a deleted state` and `kubectl logs --previous` could not retrieve the container logs. Manual assist: queued `systemctl --no-block restart k3s-agent` on `titan-07` and deleted the stale `logging/oauth2-proxy-logs-*` pod. +- `titan-07` recovered to `Ready=True` with `k3s-agent` active and kubelet `10250` open. `logging/oauth2-proxy-logs` was recreated on `titan-12` and became `2/2 Running`. +- Ananke repeatedly failed and restarted on `logging/deployment/oauth2-proxy-logs` because the replacement pod on `titan-14` hit the same `failed to reserve container name` / `CreateContainerError` runtime wedge signature. +- Manual assist: queued `systemctl --no-block restart k3s-agent` on `titan-14` through Ananke SSH and deleted the stuck `logging/oauth2-proxy-logs-5b4d4f87db-2tdzd` pod with `--wait=false` so the controller can recreate it after the runtime reset. +- Follow-up on `titan-14`: the direct SSH command timed out from the operator view, but a later status check showed `k3s-agent.service` had a new MainPID and was `activating/start`; the existing `maintenance/k3s-agent-restart` DaemonSet pod for `titan-14` was also deleted to rerun the in-cluster helper. +- `titan-14` recovered to fresh `Ready=True` heartbeats and the replacement `maintenance/k3s-agent-restart-*` pod became `Running`. +- After the `titan-05` reboot, `logging/opensearch-0` recovered to `1/1 Running`; `logging/oauth2-proxy-logs` was recreated on `titan-07` and was still initializing at the time of note. +- Ananke also entered a short retry loop where `kubectl exec vault-0 ... vault status` was killed even though Vault HTTP health through the Kubernetes pod proxy reported `initialized=true`, `sealed=false`, `standby=false`. This should become a separate robustness fix: startup should not fail outright on a transient killed `kubectl exec` when service endpoints/HTTP health prove Vault is active. +- After Keycloak completed its slow post-attach startup at approximately `2026-07-07T17:45:55Z`, OIDC-dependent OAuth proxies began recovering. +- Ananke advanced past `critical-workloads` into `convergence-checks` by approximately `2026-07-07T17:50:50Z`. +- Manual assist: force-deleted stale pod `finance/firefly-6669f46874-q8rtq` after Ananke repeatedly issued ordinary deletes but the pod remained `Terminating` with no finalizers and continued to block `firefly-storage` attachment. The Firefly Longhorn volume moved from `titan-04` to `titan-06` and reported `attached`/`healthy` on the replacement node. +- Post-Ananke-completion manual assist: `titan-04` was still `Ready` with kubelet `10250` open, but several newly scheduled pods showed `FailedCreatePodSandBox`, `failed to reserve sandbox name`, and `context deadline exceeded`. Queued `systemctl --no-block restart k3s-agent` on `titan-04` at approximately `2026-07-07T18:02:55Z`. +- `titan-04` did not recover from the non-blocking `k3s-agent` restart. It moved to Kubernetes `Ready=Unknown` with `node.kubernetes.io/unreachable` taints; host checks showed `k3s-agent.service` stuck in `deactivating/stop-sigkill`, `Result=timeout`, and kubelet `10250` closed. Manual assist: issued a controlled reboot for `titan-04` through Ananke's SSH route shortly after `2026-07-07T18:06Z`. +- `titan-04` recovered after reboot by approximately `2026-07-07T18:10Z`: Kubernetes reported `Ready=True`, taints cleared, `k3s-agent` was `active/running`, and kubelet `10250` was open. +- Firefly hit a second stale-owner handoff after the first cleanup. Replacement pod `firefly-6669f46874-pjhg6` on `titan-07` was blocked while terminating pod `firefly-6669f46874-lrz5d` on `titan-06` still had the main `firefly` container running and mounted `firefly-storage`. No force delete was issued for this second handoff because the application container was live and PVC-writing. The old pod later disappeared on its own; Longhorn then attached `pvc-358b4319-60cb-4322-b948-776c34a414a9` to `titan-07` as `attached`/`healthy`. +- Attempted to inspect container runtime state on `titan-06` with `crictl`, but `atlas` required a sudo password. This should be part of the automation fix: Ananke either needs a noninteractive, bounded runtime-inspection primitive or must report that it cannot prove a PVC-writing container is stopped. +- Manual assist: uncordoned `titan-15` and `titan-17` after confirming Kubernetes `Ready=True`, Longhorn `allowScheduling=true`, and Longhorn disks ready. The remaining blocker for Longhorn on both nodes was Kubernetes `SchedulingDisabled`; after uncordon, Longhorn `Schedulable=True` on both nodes. +- Manual assist: resolved the `titan-22` Kubernetes/Longhorn readiness discrepancy. Kubernetes reported `Ready=True`, but Longhorn node `Ready=False` with `ManagerPodMissing`; the Longhorn manager DaemonSet selected `longhorn-host=true`, and `titan-22` was missing that label. Restored `longhorn-host=true`, which started `longhorn-manager-jtsjw` on `titan-22` and brought Longhorn node `Ready=True`. +- Manual assist: deleted pending controller-owned pod `crypto/wallet-monero-temp-844bd949c7-28r4p` to force a fresh volume-attach retry after the `titan-22` Longhorn manager came up. +- Manual assist: deleted stale unattached VolumeAttachment `csi-0bac99e18b687f6a50647dd959c26d6e2a1b2bf1a4f844de4f646cf3408d41c5` after it retained the pre-fix "node titan-22 is not ready" Longhorn error. Longhorn then attached `pvc-c6a4bf9b-17c7-4bdf-8a88-ca460ad0da5b` to `titan-22` as `attached`/`healthy`. + +## Automation Growth Opportunities + +- Ananke should expose from a remote operator machine whether the coordinator bootstrap is already running, so an operator does not risk starting a duplicate recovery. +- The local Ananke config path assumes `/etc/ananke/ananke.yaml` and `/var/lib/ananke`; operator-side `status` failed without root. A read-only remote status wrapper would make this smoother. +- Ananke should handle the `cryptsetup-bin` host repair on managed nodes without requiring interactive sudo, or clearly preflight and report the missing node privilege before a power event. +- Ananke should separate `Vault pod/data volume unavailable` from `Vault sealed` in status. It did recover correctly, but early status said the unseal check passed while deferring because `vault-0` was pending. +- Ananke should avoid repeatedly recycling the same ImagePullBackOff pods if the underlying registry/image problem is unchanged; this creates noisy recovery logs. +- Ananke should surface current blocking workloads in a compact operator summary rather than requiring journal inspection. +- Ananke should detect pods stuck in `PodInitializing` after init completion, especially when events show `FailedCreatePodSandbox`, `failed to reserve sandbox name`, `FailedKillPod`, or image pulls that never transition to `ImagePullBackOff`. +- Ananke should be able to rerun the existing `k3s-agent-restart` maintenance primitive on affected nodes when containerd/kubelet sandbox state is wedged after a power event. +- Ananke should fall back from the in-cluster restart helper to SSH `systemctl --no-block restart k3s-agent` when the helper pod cannot start because kubelet/containerd is already wedged. Blocking `systemctl restart` over SSH can hang the operator path. +- Ananke should check host and CoreDNS upstream DNS health separately from node SSH reachability. During this incident, node IP reachability passed while hostname resolution and image pulls were intermittently failing. +- Ananke should stop repeatedly issuing ordinary pod deletes when the same controller pod remains `Terminating` and continues to own exclusive resources. It should escalate to a stale-owner cleanup flow that proves the old workload is not writing, force-removes the orphaned API object when safe, then verifies Longhorn detach/attach and replacement pod startup. +- Ananke should reconcile Kubernetes node labels against Longhorn manager scheduling requirements. During this incident `titan-22` was Kubernetes `Ready=True`, had CSI/engine Longhorn pods, and had a Longhorn Node CR, but no `longhorn-manager` pod because `longhorn-host=true` was missing. Kubernetes Ready alone was not sufficient for PVC workloads. +- Ananke should understand that Kubernetes cordon directly affects Longhorn node schedulability. `titan-15` and `titan-17` had Longhorn-ready disks and `allowScheduling=true`, but Longhorn `Schedulable=False` solely because the Kubernetes nodes were cordoned. + +### Generalized Node Runtime Wedge Case + +This incident exposed a reusable failure mode that Ananke should treat as a node-runtime recovery problem, not just an app-pod recovery problem. + +Detection signals: + +- Node is `Ready=Unknown` or has a stale heartbeat while SSH on the managed Ananke route still works. +- Host is pingable and local k3s load balancer port `127.0.0.1:6444` may be open, but kubelet port `127.0.0.1:10250` is closed or API proxy calls fail. +- `systemctl restart k3s-agent` hangs, or `systemctl show k3s-agent` reports `deactivating`, `final-sigterm`, `final-sigkill`, `activating/start`, or `Result=timeout` for longer than a bounded recovery window. +- `journalctl -u k3s-agent` includes `Processes still around after SIGKILL`, `State 'stop-sigterm' timed out`, or `State 'final-sigterm' timed out`. +- Pods on the node show `FailedCreatePodSandBox`, `failed to reserve sandbox name`, `CreateContainerError`, `FailedKillPod`, or long-lived `ContainerCreating`/`PodInitializing`. +- The `maintenance/k3s-agent-restart` DaemonSet pod for that node cannot start or stays `ContainerCreating`, which means Kubernetes-based repair is unavailable. + +Suggested automated escalation ladder: + +- Mark the node as in node-runtime recovery and pause noisy repeated pod recycling for pods pinned to that node. +- Cordon the node or confirm it already has `unreachable` taints before host-level repair. +- Try a bounded SSH health probe using Ananke's configured SSH user, port, config, and identity. +- Prefer `systemctl --no-block restart k3s-agent` over blocking `systemctl restart k3s-agent`. +- After a short wait, require both Kubernetes heartbeat freshness and local kubelet port `10250` to recover. +- If the service remains `deactivating`, `final-sigkill`, or `activating/start` with stale heartbeats, escalate to a configured host reboot path. +- After reboot, wait for SSH, k3s-agent active/running, kubelet `10250`, Kubernetes `Ready=True`, and removal of `unreachable` taints before resuming app-pod recycling for that node. + +Implementation notes: + +- This should be a generic managed-node repair primitive, not hard-coded to `titan-05`. +- The primitive should emit one compact operator summary: node, phase, trigger signals, action taken, elapsed time, and next escalation. +- Tests should cover a node with SSH available but kubelet closed, a blocking restart timeout, leftover shim evidence, a helper DaemonSet stuck in `ContainerCreating`, and successful recovery through reboot. +- The post-completion `titan-04` recurrence confirms the primitive must continue monitoring after Ananke declares cluster startup success. A Ready node with fresh-enough status can still be unable to create new sandboxes, and a later restart can transition it into the same hard `deactivating/stop-sigkill` state seen on `titan-05`. + +### Generalized Stale Pod / Volume Ownership Case + +This incident also exposed a reusable app-convergence failure mode that Ananke should treat as stale Kubernetes/runtime ownership rather than simple pod unhealthiness. + +Concrete example: + +- `finance/firefly-6669f46874-q8rtq` entered deletion at `2026-07-07T17:43:55Z` and remained `Terminating` for more than 25 minutes. +- The pod had no finalizers but still reported a running `vault-agent` sidecar on `titan-04`. +- The replacement pod `finance/firefly-6669f46874-lrz5d` on `titan-06` stayed `Init:0/2`. +- Events showed `FailedAttachVolume` / `Multi-Attach` for PVC `firefly-storage` because volume `pvc-358b4319-60cb-4322-b948-776c34a414a9` was still attached to `titan-04`. +- Longhorn reported the volume `attached`, `healthy`, current node `titan-04`. +- Ananke repeatedly logged ordinary recycling of the same stuck pod, but did not escalate to clearing the stale owner. +- Manual force deletion of the stale pod cleared the stale API owner and allowed Longhorn to attach `firefly-storage` to the replacement pod's node. + +Detection signals: + +- A controller-owned pod has `metadata.deletionTimestamp` older than a bounded threshold and no finalizers. +- A replacement pod from the same controller is pending or initializing on another node. +- Events include `Multi-Attach`, `FailedAttachVolume`, `Volume is already used by pod(s)`, or `Volume is already exclusively attached to one node`. +- The PVC is `ReadWriteOnce` and the old and new pod nodes differ. +- Longhorn/CNI/runtime state still shows the old node as the attachment owner. +- Ordinary `kubectl delete pod --wait=false` is repeated without changing the deletion timestamp or clearing the volume attachment. + +Suggested automated escalation ladder: + +- Group the old pod, replacement pod, PVC, VolumeAttachment, Longhorn volume, controller, and node into one recovery incident. +- Verify whether any application container that mounts the PVC is still running. Sidecars that do not mount the PVC should not block stale-owner cleanup by themselves. +- Check node health for the old owner node. If kubelet/runtime is unhealthy, route through the managed-node recovery primitive first. +- If the old owner node is healthy but the pod remains deleting past the threshold, issue a bounded force deletion for the stale pod. +- Wait for VolumeAttachment/Longhorn ownership to detach from the old node or attach to the replacement node. +- Recheck the replacement pod init sequence and controller readiness before declaring the app recovered. +- Emit one compact operator event instead of repeatedly logging the same pod recycle. + +Implementation notes: + +- This should be generic across any `ReadWriteOnce` PVC workload, not hard-coded to Firefly. +- The safety check should reason at container/mount level: a sidecar that only mounts service-account, Vault secret, or emptyDir volumes is materially different from an app container actively mounting the PVC. +- Tests should cover a terminating pod without finalizers holding an RWO PVC, an old node with healthy kubelet, an old node with wedged kubelet, a sidecar-only survivor, and a replacement pod blocked by `Multi-Attach`. +- Tests should also cover the unsafe variant where the terminating pod's application container is still running and mounts the PVC. In that case Ananke should wait, stop the container through a controlled runtime path if allowed, or report that it lacks the privilege to prove storage safety; it should not blindly force-delete the API object. + +### Generalized Longhorn / Kubernetes Readiness Drift Case + +This incident exposed a reusable storage readiness gap: Kubernetes `Ready=True` is not enough for PVC workloads when Longhorn's node model disagrees. + +Detection signals: + +- Kubernetes node is `Ready=True`, untainted, and schedulable, but Longhorn Node `Ready=False`. +- Longhorn Node reason is `ManagerPodMissing`, or the Longhorn manager DaemonSet has no pod on the node. +- Longhorn manager DaemonSet selects a node label such as `longhorn-host=true`, and the Kubernetes node is missing it. +- Longhorn CSI/engine pods or a Longhorn Node CR exist for the node, proving the node participates in Longhorn even though manager scheduling is broken. +- VolumeAttachment errors say Longhorn cannot attach because the node is not ready, despite Kubernetes Ready=True. + +Suggested automated escalation ladder: + +- Compare Kubernetes node readiness, labels, taints, Longhorn Node readiness/schedulability, Longhorn manager DaemonSet selector, manager pod presence, CSI pod presence, and VolumeAttachment errors. +- If desired-state evidence proves the node should be a Longhorn node, restore the missing manager selector label and wait for the manager pod and Longhorn Node `Ready=True`. +- If desired-state evidence is missing, report a clear unsafe label-drift blocker rather than guessing. +- After Longhorn Node `Ready=True`, retry stale VolumeAttachments by recreating the affected controller-owned pending pod or using a safe VolumeAttachment retry policy. +- For Kubernetes-cordoned Longhorn nodes with `allowScheduling=true` and ready disks, either complete the host repair and uncordon or report the explicit blocker; do not leave Longhorn capacity disabled silently. + +Implementation notes: + +- This must be generic across Longhorn nodes and manager DaemonSet selectors, not hard-coded to `titan-22` or `longhorn-host`. +- Tests should cover missing Longhorn selector labels, manager pod missing, Kubernetes Ready but Longhorn NotReady, Kubernetes cordon causing Longhorn Schedulable=False, and stale attach errors that clear after Longhorn readiness is restored. diff --git a/internal/cluster/orchestrator_access_fluxsource.go b/internal/cluster/orchestrator_access_fluxsource.go index 529a875..2aace7d 100644 --- a/internal/cluster/orchestrator_access_fluxsource.go +++ b/internal/cluster/orchestrator_access_fluxsource.go @@ -38,8 +38,8 @@ func (o *Orchestrator) reconcileNodeAccess(ctx context.Context, nodes []string) defer wg.Done() sem <- struct{}{} defer func() { <-sem }() - if _, err := o.ssh(ctx, node, "sudo -n /usr/bin/systemctl --version"); err != nil { - errCh <- fmt.Errorf("%s: missing sudo access to /usr/bin/systemctl (--version): %w", node, err) + if err := o.preflightHostPrivilege(ctx, node); err != nil { + errCh <- fmt.Errorf("%s: host privilege preflight failed: %w", node, err) } }() } @@ -49,13 +49,16 @@ func (o *Orchestrator) reconcileNodeAccess(ctx context.Context, nodes []string) return nil } samples := []string{} + errCount := 0 for err := range errCh { + errCount++ samples = append(samples, err.Error()) if len(samples) >= 4 { break } } - return fmt.Errorf("access validation had %d errors (first: %s)", len(errCh), strings.Join(samples, " | ")) + errCount += len(errCh) + return fmt.Errorf("access validation had %d errors (first: %s)", errCount, strings.Join(samples, " | ")) } // waitForNodeSSHAuth runs one orchestration or CLI step. diff --git a/internal/cluster/orchestrator_autorepair.go b/internal/cluster/orchestrator_autorepair.go index 3c9cb87..cb0111f 100644 --- a/internal/cluster/orchestrator_autorepair.go +++ b/internal/cluster/orchestrator_autorepair.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strings" "time" ) @@ -17,6 +18,7 @@ type nodeReadyItem struct { Metadata struct { Name string `json:"name"` Annotations map[string]string `json:"annotations"` + Labels map[string]string `json:"labels"` } `json:"metadata"` Spec struct { Unschedulable bool `json:"unschedulable"` @@ -75,6 +77,14 @@ func (o *Orchestrator) postStartAutoHeal(ctx context.Context) error { errs = append(errs, fmt.Sprintf("required node labels: %v", err)) } + longhornRepairs, err := o.reconcileLonghornKubernetesReadiness(ctx) + if err != nil { + errs = append(errs, fmt.Sprintf("longhorn/kubernetes readiness drift: %v", err)) + } else if longhornRepairs > 0 { + requestReconcile = true + o.log.Printf("post-start auto-heal repaired %d Longhorn/Kubernetes readiness drift item(s)", longhornRepairs) + } + releasedCordons, err := o.enforceRecoveryCordonLeases(ctx) if err != nil { errs = append(errs, fmt.Sprintf("recovery cordon lease check: %v", err)) @@ -99,6 +109,15 @@ func (o *Orchestrator) postStartAutoHeal(ctx context.Context) error { requestReconcile = true } + serviceRepairs, err := o.healUnreadyConfiguredServiceBackends(ctx) + if err != nil { + errs = append(errs, fmt.Sprintf("configured service backend repair: %v", err)) + } else if len(serviceRepairs) > 0 { + requestReconcile = true + sort.Strings(serviceRepairs) + o.log.Printf("post-start auto-heal repaired configured service backend(s): %s", joinLimited(serviceRepairs, 8)) + } + repairedProxies, err := o.repairBrokenKubeletProxies(ctx) if err != nil { errs = append(errs, fmt.Sprintf("kubelet proxy auto-repair: %v", err)) @@ -265,38 +284,10 @@ func (o *Orchestrator) repairBrokenKubeletProxies(ctx context.Context) (int, err continue } - if !node.Unschedulable { - if err := o.cordonNodeWithLease(ctx, node.Name, cordonReasonKubeletProxy, "broken kubelet proxy before k3s-agent restart"); err != nil { - errs = append(errs, fmt.Sprintf("%s cordon before kubelet restart: %v", node.Name, err)) - continue - } - } - - o.log.Printf("warning: detected broken kubelet proxy on Ready node %s; restarting k3s-agent", node.Name) - if _, err := o.sshWithTimeout(ctx, node.Name, "sudo -n systemctl restart k3s-agent", 90*time.Second); err != nil { - if !node.Unschedulable { - o.bestEffort("uncordon node after failed kubelet proxy repair", func() error { - return o.uncordonAndClearCordonLease(ctx, node.Name, cordonReasonKubeletProxy) - }) - } - errs = append(errs, fmt.Sprintf("%s restart k3s-agent: %v", node.Name, err)) + if err := o.recoverManagedNodeRuntime(ctx, node.Name, node.Unschedulable, "broken kubelet proxy before k3s-agent restart"); err != nil { + errs = append(errs, fmt.Sprintf("%s %v", node.Name, err)) continue } - if _, err := o.kubectl(ctx, 140*time.Second, "wait", "node/"+node.Name, "--for=condition=Ready", "--timeout=120s"); err != nil { - errs = append(errs, fmt.Sprintf("%s wait Ready after k3s-agent restart: %v", node.Name, err)) - continue - } - healthy, checkErr = o.kubeletProxyHealthy(ctx, node.Name) - if !healthy { - errs = append(errs, fmt.Sprintf("%s proxy still broken after k3s-agent restart: %v", node.Name, checkErr)) - continue - } - if !node.Unschedulable { - if err := o.uncordonAndClearCordonLease(ctx, node.Name, cordonReasonKubeletProxy); err != nil { - errs = append(errs, fmt.Sprintf("%s uncordon after kubelet proxy repair: %v", node.Name, err)) - continue - } - } repaired++ } diff --git a/internal/cluster/orchestrator_autorepair_proxy_test.go b/internal/cluster/orchestrator_autorepair_proxy_test.go index 3c985b1..06973e7 100644 --- a/internal/cluster/orchestrator_autorepair_proxy_test.go +++ b/internal/cluster/orchestrator_autorepair_proxy_test.go @@ -41,9 +41,11 @@ func TestRepairBrokenKubeletProxiesRestartsSchedulableNode(t *testing.T) { case name == "kubectl" && strings.Contains(joined, "cordon titan-07"): cordoned = true return "", nil - case name == "ssh" && strings.Contains(joined, "sudo -n systemctl restart k3s-agent"): + case name == "ssh" && strings.Contains(joined, "sudo -n systemctl --no-block restart k3s-agent"): restarted = true return "", nil + case name == "ssh" && strings.Contains(joined, "sudo -n systemctl is-active k3s-agent"): + return "active", nil case name == "kubectl" && strings.Contains(joined, "wait node/titan-07 --for=condition=Ready --timeout=120s"): waited = true return "", nil @@ -100,9 +102,11 @@ func TestRepairBrokenKubeletProxiesPreservesExistingCordon(t *testing.T) { case name == "kubectl" && (strings.Contains(joined, "cordon titan-18") || strings.Contains(joined, "uncordon titan-18")): cordonTouched = true return "", nil - case name == "ssh" && strings.Contains(joined, "sudo -n systemctl restart k3s-agent"): + case name == "ssh" && strings.Contains(joined, "sudo -n systemctl --no-block restart k3s-agent"): restarted = true return "", nil + case name == "ssh" && strings.Contains(joined, "sudo -n systemctl is-active k3s-agent"): + return "active", nil case name == "kubectl" && strings.Contains(joined, "wait node/titan-18 --for=condition=Ready --timeout=120s"): return "", nil default: @@ -243,6 +247,7 @@ func TestRepairBrokenKubeletProxiesFailureBranches(t *testing.T) { {cmd: "restart"}, {cmd: "wait"}, {cmd: "health", out: "ok"}, + {cmd: "active", out: "active"}, {cmd: "uncordon", err: errors.New("uncordon denied")}, }) repaired, err := orch.repairBrokenKubeletProxies(context.Background()) @@ -280,8 +285,10 @@ func kubeletProxyRepairStub(t *testing.T, cfg config.Config, unschedulable bool, actual = "uncordon" case name == "kubectl" && strings.Contains(joined, "cordon titan-07"): actual = "cordon" - case name == "ssh" && strings.Contains(joined, "sudo -n systemctl restart k3s-agent"): + case name == "ssh" && strings.Contains(joined, "sudo -n systemctl --no-block restart k3s-agent"): actual = "restart" + case name == "ssh" && strings.Contains(joined, "sudo -n systemctl is-active k3s-agent"): + actual = "active" case name == "kubectl" && strings.Contains(joined, "wait node/titan-07 --for=condition=Ready --timeout=120s"): actual = "wait" default: diff --git a/internal/cluster/orchestrator_autorepair_service_test.go b/internal/cluster/orchestrator_autorepair_service_test.go new file mode 100644 index 0000000..4705476 --- /dev/null +++ b/internal/cluster/orchestrator_autorepair_service_test.go @@ -0,0 +1,64 @@ +package cluster + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "scm.bstein.dev/bstein/ananke/internal/config" +) + +// TestConfiguredServiceBackendRepairContinuesPastNoopEndpoint runs one orchestration or CLI step. +// Signature: TestConfiguredServiceBackendRepairContinuesPastNoopEndpoint(t *testing.T). +// Why: one unhealthy endpoint without a matching workload must not prevent later +// critical services, especially Mailu, from getting their backend repair pass. +func TestConfiguredServiceBackendRepairContinuesPastNoopEndpoint(t *testing.T) { + cfg := config.Config{ + Startup: config.Startup{ + CriticalServiceEndpoints: []string{ + "orphan/missing-service", + "mailu-mailserver/mailu-admin", + }, + }, + } + orch := buildOrchestratorWithStubs(t, cfg, nil) + + scaledMail := false + rolledOutMail := false + dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) { + if name != "kubectl" { + return "", nil + } + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "-n orphan get endpoints missing-service"): + return "", nil + case strings.Contains(joined, "-n orphan scale deployment missing-service --replicas=1"): + return "", errors.New("not found") + case strings.Contains(joined, "-n orphan scale statefulset missing-service --replicas=1"): + return "", errors.New("not found") + case strings.Contains(joined, "-n mailu-mailserver get endpoints mailu-admin"): + return "", nil + case strings.Contains(joined, "-n mailu-mailserver scale deployment mailu-admin --replicas=1"): + scaledMail = true + return "", nil + case strings.Contains(joined, "-n mailu-mailserver rollout status deployment/mailu-admin"): + rolledOutMail = true + return "", nil + case strings.Contains(joined, "-n mailu-mailserver scale statefulset mailu-admin --replicas=1"): + return "", errors.New("not found") + default: + return "", nil + } + } + orch.SetCommandOverrides(dispatch, dispatch) + + if _, err := orch.healUnreadyConfiguredServiceBackends(context.Background()); err != nil { + t.Fatalf("healUnreadyConfiguredServiceBackends failed: %v", err) + } + if !scaledMail || !rolledOutMail { + t.Fatalf("expected later mail endpoint to be repaired, scaled=%v rolledOut=%v", scaledMail, rolledOutMail) + } +} diff --git a/internal/cluster/orchestrator_autorepair_test.go b/internal/cluster/orchestrator_autorepair_test.go index 4a80172..1652775 100644 --- a/internal/cluster/orchestrator_autorepair_test.go +++ b/internal/cluster/orchestrator_autorepair_test.go @@ -177,6 +177,77 @@ func TestPostStartAutoHealSkipsWhenClusterIsHealthy(t *testing.T) { } } +// TestPostStartAutoHealRepairsConfiguredMailBackends runs one orchestration or CLI step. +// Signature: TestPostStartAutoHealRepairsConfiguredMailBackends(t *testing.T). +// Why: Mailu readiness problems should be handled by Ananke's daemon repair +// loop after startup, not only by a fresh bootstrap run. +func TestPostStartAutoHealRepairsConfiguredMailBackends(t *testing.T) { + cfg := config.Config{ + Startup: config.Startup{ + DeadNodeCleanupGraceSeconds: 300, + CriticalServiceEndpoints: []string{"mailu-mailserver/mailu-admin"}, + NodeRuntimeRestartWaitSeconds: 1, + NodeRuntimeRebootWaitSeconds: 1, + HostPrivilegedCommandTimeoutSec: 1, + }, + } + orch := buildOrchestratorWithStubs(t, cfg, nil) + + scaled := false + rolledOut := false + reconciled := false + dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) { + if name != "kubectl" { + return "", nil + } + joined := strings.Join(args, " ") + switch { + case strings.Contains(joined, "-n longhorn-system get nodes.longhorn.io -o json"): + return `{"items":[]}`, nil + case strings.Contains(joined, "-n vault get pod vault-0 -o jsonpath={.status.phase}"): + return "", nil + case strings.Contains(joined, "get nodes -o json"): + return `{"items":[]}`, nil + case strings.Contains(joined, "get pods -A -o json"): + return `{"items":[]}`, nil + case strings.Contains(joined, "get events -A -o json"): + return `{"items":[]}`, nil + case strings.Contains(joined, "get pvc -A -o json"): + return `{"items":[]}`, nil + case strings.Contains(joined, "-n mailu-mailserver get endpoints mailu-admin"): + return "", nil + case strings.Contains(joined, "-n mailu-mailserver scale deployment mailu-admin --replicas=1"): + scaled = true + return "", nil + case strings.Contains(joined, "-n mailu-mailserver rollout status deployment/mailu-admin"): + rolledOut = true + return "", nil + case strings.Contains(joined, "-n mailu-mailserver scale statefulset mailu-admin --replicas=1"): + return "", errors.New("not found") + case strings.Contains(joined, "-n flux-system annotate gitrepository flux-system reconcile.fluxcd.io/requestedAt="): + reconciled = true + return "", nil + case strings.Contains(joined, "-n flux-system annotate kustomizations.kustomize.toolkit.fluxcd.io --all reconcile.fluxcd.io/requestedAt="): + return "", nil + case strings.Contains(joined, "annotate --all-namespaces helmreleases.helm.toolkit.fluxcd.io --all reconcile.fluxcd.io/requestedAt="): + return "", nil + default: + return "", nil + } + } + orch.SetCommandOverrides(dispatch, dispatch) + + if err := orch.postStartAutoHeal(context.Background()); err != nil { + t.Fatalf("postStartAutoHeal failed: %v", err) + } + if !scaled || !rolledOut { + t.Fatalf("expected mailu-admin deployment to be scaled and rolled out, scaled=%v rolledOut=%v", scaled, rolledOut) + } + if !reconciled { + t.Fatalf("expected flux reconcile after mail backend repair") + } +} + // TestRunPostStartAutoHealDryRun runs one orchestration or CLI step. // Signature: TestRunPostStartAutoHealDryRun(t *testing.T). // Why: covers the exported wrapper and the top-level dry-run guard so daemon diff --git a/internal/cluster/orchestrator_core.go b/internal/cluster/orchestrator_core.go index 667397f..53a4934 100644 --- a/internal/cluster/orchestrator_core.go +++ b/internal/cluster/orchestrator_core.go @@ -20,6 +20,7 @@ type Orchestrator struct { log *log.Logger runOverride func(timeoutCtx context.Context, timeout time.Duration, name string, args ...string) (string, error) runSensitiveOverride func(timeoutCtx context.Context, timeout time.Duration, name string, args ...string) (string, error) + sshInputOverride func(timeoutCtx context.Context, timeout time.Duration, node string, command string, input string) (string, error) startupReportMu sync.Mutex activeStartupReport *startupReport } @@ -122,3 +123,11 @@ func (o *Orchestrator) SetCommandOverrides(run commandOverrideFunc, runSensitive o.runOverride = run o.runSensitiveOverride = runSensitive } + +// SetSSHInputOverride runs one orchestration or CLI step. +// Signature: (o *Orchestrator) SetSSHInputOverride(run func(context.Context, time.Duration, string, string, string) (string, error)). +// Why: password-backed host-repair tests need to assert stdin behavior without +// exposing secrets through ordinary command-argument stubs. +func (o *Orchestrator) SetSSHInputOverride(run func(context.Context, time.Duration, string, string, string) (string, error)) { + o.sshInputOverride = run +} diff --git a/internal/cluster/orchestrator_critical_endpoints.go b/internal/cluster/orchestrator_critical_endpoints.go index 62b91b8..472f1e2 100644 --- a/internal/cluster/orchestrator_critical_endpoints.go +++ b/internal/cluster/orchestrator_critical_endpoints.go @@ -2,12 +2,22 @@ package cluster import ( "context" + "encoding/json" + "errors" "fmt" "sort" "strings" "time" ) +type endpointResource struct { + Subsets []struct { + Addresses []struct { + IP string `json:"ip"` + } `json:"addresses"` + } `json:"subsets"` +} + // waitForCriticalServiceEndpoints runs one orchestration or CLI step. // Signature: (o *Orchestrator) waitForCriticalServiceEndpoints(ctx context.Context) error. // Why: some externally-healthy services (like Grafana) still require backend @@ -25,8 +35,10 @@ func (o *Orchestrator) waitForCriticalServiceEndpoints(ctx context.Context) erro lastFailure := "unknown" lastLogged := time.Time{} lastHealAttempt := time.Time{} + lastRecycleAttempt := time.Time{} for { + o.maybeAutoRecycleStuckPods(ctx, &lastRecycleAttempt) ready, detail, failedNamespace, failedService, err := o.criticalServiceEndpointsReady(ctx) if err != nil { lastFailure = err.Error() @@ -44,6 +56,16 @@ func (o *Orchestrator) waitForCriticalServiceEndpoints(ctx context.Context) erro lastHealAttempt = now healed, healErr := o.maybeHealCriticalEndpointBackends(ctx, failedNamespace, failedService) if healErr != nil { + probeRepairs, probeErr := o.maybeRepairCriticalBackendStartupProbes(ctx, failedNamespace, failedService) + if probeErr != nil { + o.log.Printf("warning: critical endpoint startup-probe repair failed for %s/%s: %v", failedNamespace, failedService, probeErr) + } + if len(probeRepairs) > 0 { + sort.Strings(probeRepairs) + repairDetail := fmt.Sprintf("repaired critical endpoint startup probes: %s", joinLimited(probeRepairs, 8)) + o.log.Printf("%s", repairDetail) + o.noteStartupAutoHeal(repairDetail) + } o.log.Printf("warning: critical endpoint backend auto-heal failed for %s/%s: %v", failedNamespace, failedService, healErr) } if len(healed) > 0 { @@ -120,7 +142,7 @@ func (o *Orchestrator) maybeHealCriticalEndpointBackends(ctx context.Context, na } return healed, fmt.Errorf("scale %s/%s/%s to 1: %w", namespace, kind, service, err) } - if err := o.waitWorkloadReady(ctx, workload); err != nil { + if err := o.workloadRolloutReadyOnce(ctx, workload); err != nil { if !isNotFoundErr(err) { return healed, err } @@ -131,6 +153,83 @@ func (o *Orchestrator) maybeHealCriticalEndpointBackends(ctx context.Context, na return healed, nil } +// workloadRolloutReadyOnce runs one orchestration or CLI step. +// Signature: (o *Orchestrator) workloadRolloutReadyOnce(ctx context.Context, w startupWorkload) error. +// Why: endpoint backend healing should issue one bounded readiness nudge and +// let the outer endpoint wait enforce the configured convergence deadline. +func (o *Orchestrator) workloadRolloutReadyOnce(ctx context.Context, w startupWorkload) error { + _, err := o.kubectl( + ctx, + 45*time.Second, + "-n", + w.Namespace, + "rollout", + "status", + fmt.Sprintf("%s/%s", w.Kind, w.Name), + "--timeout=30s", + ) + return err +} + +// healUnreadyConfiguredServiceBackends runs one orchestration or CLI step. +// Signature: (o *Orchestrator) healUnreadyConfiguredServiceBackends(ctx context.Context) ([]string, error). +// Why: the daemon's post-start loop should be able to repair configured service +// backends, including Mailu, without rerunning the whole startup workflow. +func (o *Orchestrator) healUnreadyConfiguredServiceBackends(ctx context.Context) ([]string, error) { + healed := []string{} + if len(o.cfg.Startup.CriticalServiceEndpoints) > 0 { + errs := []string{} + for _, entry := range o.cfg.Startup.CriticalServiceEndpoints { + namespace, service, err := parseCriticalServiceEndpoint(entry) + if err != nil { + errs = append(errs, err.Error()) + continue + } + count, err := o.endpointAddressCount(ctx, namespace, service) + if err != nil && !isNotFoundErr(err) { + errs = append(errs, fmt.Sprintf("query endpoints %s/%s: %v", namespace, service, err)) + continue + } + if err == nil && count > 0 { + continue + } + detail := fmt.Sprintf("%s/%s endpoints=0", namespace, service) + if err != nil { + detail = fmt.Sprintf("%s/%s not found", namespace, service) + } + o.log.Printf("warning: configured critical endpoint unhealthy (%s); attempting backend repair", detail) + items, healErr := o.maybeHealCriticalEndpointBackends(ctx, namespace, service) + healed = append(healed, items...) + if healErr != nil { + probeItems, probeErr := o.maybeRepairCriticalBackendStartupProbes(ctx, namespace, service) + healed = append(healed, probeItems...) + if probeErr != nil { + errs = append(errs, fmt.Sprintf("heal critical endpoint %s/%s (%s): %v; startup-probe repair: %v", namespace, service, detail, healErr, probeErr)) + continue + } + if o.cfg.Startup.AutoRecycleStuckPods { + o.bestEffort("recycle stuck pods after configured service backend repair failure", func() error { + return o.recycleStuckControllerPods(ctx) + }) + } + if len(probeItems) == 0 { + errs = append(errs, fmt.Sprintf("heal critical endpoint %s/%s (%s): %v", namespace, service, detail, healErr)) + } + } + } + if len(errs) > 0 { + return healed, errors.New(strings.Join(errs, "; ")) + } + } + + tcpHealed, err := o.healFailedTCPServiceBackends(ctx) + healed = append(healed, tcpHealed...) + if err != nil { + return healed, err + } + return healed, nil +} + // endpointAddressCount runs one orchestration or CLI step. // Signature: (o *Orchestrator) endpointAddressCount(ctx context.Context, namespace string, service string) (int, error). // Why: endpoint address counts provide an objective service-backend readiness @@ -145,12 +244,43 @@ func (o *Orchestrator) endpointAddressCount(ctx context.Context, namespace strin "endpoints", service, "-o", - `jsonpath={range .subsets[*].addresses[*]}{.ip}{"\n"}{end}`, + "json", ) if err != nil { return 0, err } - return len(lines(out)), nil + cleaned := stripKubectlWarnings(out) + if strings.HasPrefix(cleaned, "{") { + var endpoint endpointResource + if err := json.Unmarshal([]byte(cleaned), &endpoint); err != nil { + return 0, fmt.Errorf("decode endpoints %s/%s: %w", namespace, service, err) + } + count := 0 + for _, subset := range endpoint.Subsets { + for _, address := range subset.Addresses { + if strings.TrimSpace(address.IP) != "" { + count++ + } + } + } + return count, nil + } + return len(lines(cleaned)), nil +} + +// stripKubectlWarnings runs one orchestration or CLI step. +// Signature: stripKubectlWarnings(out string) string. +// Why: kubectl writes deprecation warnings to stderr; command execution combines +// streams, so readiness parsers must not mistake warning lines for health data. +func stripKubectlWarnings(out string) string { + kept := []string{} + for _, line := range strings.Split(out, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "Warning:") { + continue + } + kept = append(kept, line) + } + return strings.TrimSpace(strings.Join(kept, "\n")) } // parseCriticalServiceEndpoint runs one orchestration or CLI step. diff --git a/internal/cluster/orchestrator_critical_probe_repair.go b/internal/cluster/orchestrator_critical_probe_repair.go new file mode 100644 index 0000000..f3cb649 --- /dev/null +++ b/internal/cluster/orchestrator_critical_probe_repair.go @@ -0,0 +1,144 @@ +package cluster + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +type criticalBackendWorkload struct { + Spec struct { + Template struct { + Spec struct { + Containers []criticalBackendContainer `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` +} + +type criticalBackendContainer struct { + Name string `json:"name"` + LivenessProbe map[string]any `json:"livenessProbe"` + StartupProbe map[string]any `json:"startupProbe"` +} + +// maybeRepairCriticalBackendStartupProbes runs one orchestration or CLI step. +// Signature: (o *Orchestrator) maybeRepairCriticalBackendStartupProbes(ctx context.Context, namespace string, service string) ([]string, error). +// Why: critical backends that are slow after outage recovery can be killed by +// liveness probes before they ever become ready; adding a startup probe lets +// Kubernetes distinguish slow boot from dead process without app-specific code. +func (o *Orchestrator) maybeRepairCriticalBackendStartupProbes(ctx context.Context, namespace string, service string) ([]string, error) { + if !o.cfg.Startup.CriticalServiceStartupProbeRepair { + return nil, nil + } + namespace = strings.TrimSpace(namespace) + service = strings.TrimSpace(service) + if namespace == "" || service == "" { + return nil, nil + } + repaired := []string{} + errs := []string{} + for _, kind := range []string{"deployment", "statefulset"} { + items, err := o.repairWorkloadStartupProbes(ctx, namespace, kind, service) + if err != nil { + if isNotFoundErr(err) { + continue + } + errs = append(errs, err.Error()) + continue + } + repaired = append(repaired, items...) + } + if len(errs) > 0 { + return repaired, errors.New(strings.Join(errs, "; ")) + } + return repaired, nil +} + +// repairWorkloadStartupProbes runs one orchestration or CLI step. +// Signature: (o *Orchestrator) repairWorkloadStartupProbes(ctx context.Context, namespace string, kind string, name string) ([]string, error). +// Why: the startup-probe patch is deliberately limited to an existing +// deployment/statefulset with a liveness probe and no startup probe. +func (o *Orchestrator) repairWorkloadStartupProbes(ctx context.Context, namespace string, kind string, name string) ([]string, error) { + out, err := o.kubectl(ctx, 20*time.Second, "-n", namespace, "get", kind, name, "-o", "json") + if err != nil { + if strings.TrimSpace(out) != "" { + return nil, fmt.Errorf("%w: %s", err, strings.TrimSpace(out)) + } + return nil, err + } + var workload criticalBackendWorkload + if err := json.Unmarshal([]byte(out), &workload); err != nil { + return nil, fmt.Errorf("decode %s/%s/%s for startup probe repair: %w", namespace, kind, name, err) + } + + containers := []map[string]any{} + threshold := o.cfg.Startup.CriticalServiceStartupProbeThreshold + if threshold <= 0 { + threshold = 30 + } + for _, container := range workload.Spec.Template.Spec.Containers { + containerName := strings.TrimSpace(container.Name) + if containerName == "" || len(container.LivenessProbe) == 0 || len(container.StartupProbe) > 0 { + continue + } + startupProbe := cloneMap(container.LivenessProbe) + startupProbe["failureThreshold"] = threshold + if _, ok := startupProbe["successThreshold"]; !ok { + startupProbe["successThreshold"] = 1 + } + containers = append(containers, map[string]any{ + "name": containerName, + "startupProbe": startupProbe, + }) + } + if len(containers) == 0 { + return nil, nil + } + + now := time.Now().UTC().Format(time.RFC3339) + patch := map[string]any{ + "spec": map[string]any{ + "template": map[string]any{ + "metadata": map[string]any{ + "annotations": map[string]string{ + "ananke.bstein.dev/startup-probe-repair-at": now, + "ananke.bstein.dev/startup-probe-repair-reason": "critical-service-backend-unready", + }, + }, + "spec": map[string]any{ + "containers": containers, + }, + }, + }, + } + rawPatch, err := json.Marshal(patch) + if err != nil { + return nil, fmt.Errorf("encode startup probe repair patch: %w", err) + } + if _, err := o.kubectl(ctx, 25*time.Second, "-n", namespace, "patch", kind, name, "--type=strategic", "-p", string(rawPatch)); err != nil { + return nil, fmt.Errorf("patch startup probes on %s/%s/%s: %w", namespace, kind, name, err) + } + o.log.Printf("repaired startup probes on critical backend %s/%s/%s containers=%d", namespace, kind, name, len(containers)) + o.noteStartupAutoHeal(fmt.Sprintf("repaired startup probes on %s/%s/%s", namespace, kind, name)) + return []string{namespace + "/" + kind + "/" + name}, nil +} + +// cloneMap runs one orchestration or CLI step. +// Signature: cloneMap(in map[string]any) map[string]any. +// Why: Kubernetes probe objects are patch payloads; cloning avoids mutating the +// decoded workload fixture while changing the generated startup probe threshold. +func cloneMap(in map[string]any) map[string]any { + out := map[string]any{} + for key, value := range in { + if nested, ok := value.(map[string]any); ok { + out[key] = cloneMap(nested) + continue + } + out[key] = value + } + return out +} diff --git a/internal/cluster/orchestrator_critical_vault.go b/internal/cluster/orchestrator_critical_vault.go index 33d37cb..011013c 100644 --- a/internal/cluster/orchestrator_critical_vault.go +++ b/internal/cluster/orchestrator_critical_vault.go @@ -178,6 +178,9 @@ func (o *Orchestrator) waitWorkloadReady(ctx context.Context, w startupWorkload) if err == nil { return nil } + if isNotFoundErr(err) { + return err + } lastErr = err select { @@ -318,28 +321,6 @@ func (o *Orchestrator) ensureVaultUnsealed(ctx context.Context) error { return fmt.Errorf("vault remained sealed after 5 auto-unseal attempts") } -// vaultSealed runs one orchestration or CLI step. -// Signature: (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error). -// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve. -func (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error) { - out, err := o.kubectl( - ctx, - 25*time.Second, - "-n", "vault", - "exec", "vault-0", "--", - "sh", "-lc", - "VAULT_ADDR=http://127.0.0.1:8200 vault status -format=json 2>/dev/null || true", - ) - if err != nil { - return false, fmt.Errorf("vault status check failed: %w", err) - } - sealed, err := parseVaultSealed(out) - if err != nil { - return false, fmt.Errorf("parse vault status: %w", err) - } - return sealed, nil -} - // parseVaultSealed runs one orchestration or CLI step. // Signature: parseVaultSealed(raw string) (bool, error). // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve. diff --git a/internal/cluster/orchestrator_hardening_test.go b/internal/cluster/orchestrator_hardening_test.go new file mode 100644 index 0000000..7a913ba --- /dev/null +++ b/internal/cluster/orchestrator_hardening_test.go @@ -0,0 +1,455 @@ +package cluster + +import ( + "bufio" + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "log" + "net" + "strconv" + "strings" + "testing" + "time" + + "scm.bstein.dev/bstein/ananke/internal/config" +) + +// TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret runs one orchestration or CLI step. +// Signature: TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret(t *testing.T). +// Why: host repair must recover from password-required sudo while keeping the +// sudo password out of command arguments and logs. +func TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret(t *testing.T) { + secret := "correct horse battery staple" + var logs bytes.Buffer + orch := buildOrchestratorWithStubs(t, config.Config{ + Startup: config.Startup{ + HostSudoSecretNamespace: "ops", + HostSudoSecretNameTemplate: "sudo-{node}", + HostSudoSecretPasswordKey: "password", + }, + }, nil) + orch.log = log.New(&logs, "", 0) + + orch.SetCommandOverrides(func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) { + joined := strings.Join(args, " ") + switch { + case name == "ssh" && strings.Contains(joined, "sudo -n /usr/bin/systemctl --version"): + return "sudo: a password is required", errors.New("exit status 1") + case name == "kubectl" && strings.Contains(joined, "-n ops get secret sudo-titan-05 -o json"): + return `{"data":{"password":"` + base64.StdEncoding.EncodeToString([]byte(secret)) + `"}}`, nil + default: + return "", nil + } + }, nil) + + inputSeen := "" + commandSeen := "" + orch.SetSSHInputOverride(func(_ context.Context, _ time.Duration, node string, command string, input string) (string, error) { + if node != "titan-05" { + t.Fatalf("unexpected node %q", node) + } + commandSeen = command + inputSeen = input + return "systemd 255", nil + }) + + if err := orch.preflightHostPrivilege(context.Background(), "titan-05"); err != nil { + t.Fatalf("preflightHostPrivilege failed: %v", err) + } + if !strings.Contains(commandSeen, "sudo -S -p '' /usr/bin/systemctl --version") { + t.Fatalf("expected sudo stdin command, got %q", commandSeen) + } + if inputSeen != secret+"\n" { + t.Fatalf("expected sudo password on stdin only") + } + if strings.Contains(logs.String(), secret) || strings.Contains(commandSeen, secret) { + t.Fatalf("sudo password leaked through logs or command") + } +} + +// TestHostPrivilegeMissingSecretReportsUnavailable runs one orchestration or CLI step. +// Signature: TestHostPrivilegeMissingSecretReportsUnavailable(t *testing.T). +// Why: missing Vault/Kubernetes sudo material should be a classified blocker, +// not an opaque loop of failed host commands. +func TestHostPrivilegeMissingSecretReportsUnavailable(t *testing.T) { + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + { + match: matchContains("ssh", "sudo -n /usr/bin/systemctl --version"), + out: "sudo: a password is required", + err: errors.New("exit status 1"), + }, + }) + err := orch.preflightHostPrivilege(context.Background(), "titan-05") + if err == nil || !strings.Contains(err.Error(), "host-privilege-unavailable") { + t.Fatalf("expected host-privilege-unavailable, got %v", err) + } +} + +// TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel runs one orchestration or CLI step. +// Signature: TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel(t *testing.T). +// Why: a Kubernetes Ready node with a Longhorn Node CR should regain missing +// manager selector labels instead of leaving PVC workloads blocked. +func TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel(t *testing.T) { + labeled := false + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + { + match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), + out: `{"items":[{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`, + }, + { + match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager", "-o", "json"), + out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true"}}}}`, + }, + { + match: matchContains("kubectl", "get", "nodes", "-o", "json"), + out: `{"items":[{"metadata":{"name":"titan-22","labels":{"kubernetes.io/hostname":"titan-22"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`, + }, + { + match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json", "-l", "longhorn-host=true"), + out: `{"items":[]}`, + }, + { + match: func(name string, args []string) bool { + if !matchContains("kubectl", "label", "node", "titan-22", "--overwrite", "longhorn-host=true")(name, args) { + return false + } + labeled = true + return true + }, + }, + }) + + repaired, err := orch.reconcileLonghornKubernetesReadiness(context.Background()) + if err != nil { + t.Fatalf("reconcileLonghornKubernetesReadiness failed: %v", err) + } + if repaired != 1 || !labeled { + t.Fatalf("expected one Longhorn label repair, repaired=%d labeled=%v", repaired, labeled) + } +} + +// TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter runs one orchestration or CLI step. +// Signature: TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter(t *testing.T). +// Why: stale RWO cleanup may force-delete sidecar-only owners but must not clear +// an API object while a live application container still mounts the PVC. +func TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter(t *testing.T) { + old := time.Now().Add(-30 * time.Minute).UTC().Format(time.RFC3339) + pvcJSON := `{"items":[{"metadata":{"namespace":"finance","name":"firefly-storage"},"spec":{"accessModes":["ReadWriteOnce"],"volumeName":"pvc-1"}}]}` + eventsJSON := `{"items":[{"involvedObject":{"kind":"Pod","namespace":"finance","name":"firefly-new"},"type":"Warning","reason":"FailedAttachVolume","message":"Multi-Attach error for volume pvc-1: Volume is already used by pod(s) firefly-old"}]}` + + t.Run("sidecar-only force delete", func(t *testing.T) { + podsJSON := `{"items":[` + + staleRWOPodJSON("firefly-old", old, "titan-04", true, false) + `,` + + replacementRWOPodJSON("firefly-new", "titan-06") + + `]}` + orch := staleRWOOrchestrator(t, podsJSON, pvcJSON, eventsJSON) + + var pods podList + if err := jsonUnmarshal(podsJSON, &pods); err != nil { + t.Fatalf("decode pods: %v", err) + } + decisions, err := orch.staleRWOPVCOwnerDecisions(context.Background(), pods, 5*time.Minute) + if err != nil { + t.Fatalf("staleRWOPVCOwnerDecisions failed: %v", err) + } + decision := decisions["finance/firefly-old"] + if !decision.ForceDelete || decision.Unsafe { + t.Fatalf("expected sidecar-only force delete decision, got %#v", decision) + } + }) + + t.Run("live writer unsafe", func(t *testing.T) { + podsJSON := `{"items":[` + + staleRWOPodJSON("firefly-old", old, "titan-04", true, true) + `,` + + replacementRWOPodJSON("firefly-new", "titan-06") + + `]}` + orch := staleRWOOrchestrator(t, podsJSON, pvcJSON, eventsJSON) + + var pods podList + if err := jsonUnmarshal(podsJSON, &pods); err != nil { + t.Fatalf("decode pods: %v", err) + } + decisions, err := orch.staleRWOPVCOwnerDecisions(context.Background(), pods, 5*time.Minute) + if err != nil { + t.Fatalf("staleRWOPVCOwnerDecisions failed: %v", err) + } + decision := decisions["finance/firefly-old"] + if !decision.Unsafe || decision.ForceDelete { + t.Fatalf("expected live writer unsafe decision, got %#v", decision) + } + }) +} + +// TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS runs one orchestration or CLI step. +// Signature: TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS(t *testing.T). +// Why: DNS-caused image pulls should be reported as registry/DNS blockers rather +// than recycled as if deleting the pod could fix name resolution. +func TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS(t *testing.T) { + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + { + match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), + out: `{"items":[{"involvedObject":{"kind":"Pod","namespace":"logging","name":"oauth2"},"type":"Warning","reason":"Failed","message":"Failed to pull image: lookup registry-1.docker.io: Try again"}]}`, + }, + }) + reasons, err := orch.imagePullDNSBlockerReasons(context.Background()) + if err != nil { + t.Fatalf("imagePullDNSBlockerReasons failed: %v", err) + } + if got := reasons["logging/oauth2"]; got != "ImagePullDNSBlocker:registry-1.docker.io" { + t.Fatalf("unexpected image pull DNS reason %q", got) + } +} + +// TestVaultSealedFallsBackToHTTPHealthWhenExecKilled runs one orchestration or CLI step. +// Signature: TestVaultSealedFallsBackToHTTPHealthWhenExecKilled(t *testing.T). +// Why: a transient killed kubectl exec should not block startup when Vault HTTP +// health proves the pod is initialized and unsealed. +func TestVaultSealedFallsBackToHTTPHealthWhenExecKilled(t *testing.T) { + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + { + match: matchContains("kubectl", "-n", "vault", "exec", "vault-0"), + err: errors.New("signal: killed"), + }, + { + match: matchContains("kubectl", "-n", "vault", "get", "--raw", "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health"), + out: `{"initialized":true,"sealed":false,"standby":false}`, + }, + }) + sealed, err := orch.vaultSealed(context.Background()) + if err != nil || sealed { + t.Fatalf("expected HTTP fallback to report unsealed, sealed=%v err=%v", sealed, err) + } +} + +// TestTCPServiceChecklistReadyChecksMailProtocolBanner runs one orchestration or CLI step. +// Signature: TestTCPServiceChecklistReadyChecksMailProtocolBanner(t *testing.T). +// Why: Mailu is not covered by HTTP ingress checks, so Ananke needs direct +// protocol probes that validate a TCP greeting and command response. +func TestTCPServiceChecklistReadyChecksMailProtocolBanner(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + defer listener.Close() + + done := make(chan struct{}) + go func() { + defer close(done) + conn, acceptErr := listener.Accept() + if acceptErr != nil { + return + } + defer conn.Close() + _, _ = conn.Write([]byte("220 mail.bstein.dev ESMTP ready\r\n")) + _, _ = bufio.NewReader(conn).ReadString('\n') + _, _ = conn.Write([]byte("221 2.0.0 Bye\r\n")) + }() + + host, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatalf("split listener address: %v", err) + } + check := config.TCPServiceChecklistCheck{ + Name: "mail-smtp", + Host: host, + Port: atoiForTest(t, port), + Send: "QUIT\r\n", + ExpectContains: "ESMTP", + TimeoutSeconds: 2, + } + orch := buildOrchestratorWithStubs(t, config.Config{}, nil) + ok, detail := orch.tcpServiceCheckReady(context.Background(), check) + if !ok { + t.Fatalf("expected TCP mail check to pass, detail=%s", detail) + } + <-done +} + +// TestServiceChecklistReadyIncludesTCPFailures runs one orchestration or CLI step. +// Signature: TestServiceChecklistReadyIncludesTCPFailures(t *testing.T). +// Why: startup should fail clearly when a configured mail protocol endpoint is +// down even if there are no HTTP checks in that config. +func TestServiceChecklistReadyIncludesTCPFailures(t *testing.T) { + orch := buildOrchestratorWithStubs(t, config.Config{ + Startup: config.Startup{ + TCPServiceChecklist: []config.TCPServiceChecklistCheck{ + { + Name: "mail-smtp", + Host: "127.0.0.1", + Port: 1, + ExpectContains: "ESMTP", + TimeoutSeconds: 1, + }, + }, + }, + }, nil) + ok, detail := orch.serviceChecklistReady(context.Background()) + if ok { + t.Fatalf("expected TCP-only checklist to fail") + } + if !strings.Contains(detail, "tcp mail-smtp") { + t.Fatalf("expected TCP failure detail, got %q", detail) + } +} + +// TestEndpointAddressCountIgnoresKubectlWarnings runs one orchestration or CLI step. +// Signature: TestEndpointAddressCountIgnoresKubectlWarnings(t *testing.T). +// Why: Kubernetes endpoint deprecation warnings must not make an empty service +// look ready by contributing a fake output line. +func TestEndpointAddressCountIgnoresKubectlWarnings(t *testing.T) { + orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + { + match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "endpoints", "mailu-admin", "-o", "json"), + out: "Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\n{\"subsets\":[]}", + }, + }) + count, err := orch.endpointAddressCount(context.Background(), "mailu-mailserver", "mailu-admin") + if err != nil { + t.Fatalf("endpointAddressCount failed: %v", err) + } + if count != 0 { + t.Fatalf("expected empty endpoint count after warning strip, got %d", count) + } +} + +// TestCriticalBackendStartupProbeRepairCopiesLivenessProbe runs one orchestration or CLI step. +// Signature: TestCriticalBackendStartupProbeRepairCopiesLivenessProbe(t *testing.T). +// Why: slow critical backends should get a bounded startup grace without +// changing app-specific liveness semantics or requiring a Mailu special case. +func TestCriticalBackendStartupProbeRepairCopiesLivenessProbe(t *testing.T) { + var patchPayload string + orch := buildOrchestratorWithStubs(t, config.Config{ + Startup: config.Startup{ + CriticalServiceStartupProbeRepair: true, + CriticalServiceStartupProbeThreshold: 24, + }, + }, []commandStub{ + { + match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "deployment", "mailu-admin", "-o", "json"), + out: `{"spec":{"template":{"spec":{"containers":[{"name":"admin","livenessProbe":{"httpGet":{"path":"/ping","port":"http"},"periodSeconds":10,"failureThreshold":3}},` + + `{"name":"sidecar"}]}}}}`, + }, + { + match: func(name string, args []string) bool { + if !matchContains("kubectl", "-n", "mailu-mailserver", "patch", "deployment", "mailu-admin", "--type=strategic", "-p")(name, args) { + return false + } + for i, arg := range args { + if arg == "-p" && i+1 < len(args) { + patchPayload = args[i+1] + } + } + return true + }, + }, + { + match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "statefulset", "mailu-admin", "-o", "json"), + err: errors.New("not found"), + }, + }) + + repaired, err := orch.maybeRepairCriticalBackendStartupProbes(context.Background(), "mailu-mailserver", "mailu-admin") + if err != nil { + t.Fatalf("maybeRepairCriticalBackendStartupProbes failed: %v", err) + } + if len(repaired) != 1 || repaired[0] != "mailu-mailserver/deployment/mailu-admin" { + t.Fatalf("unexpected repaired list: %v", repaired) + } + if patchPayload == "" { + t.Fatalf("expected strategic merge patch payload") + } + if !strings.Contains(patchPayload, `"startupProbe"`) || !strings.Contains(patchPayload, `"failureThreshold":24`) || !strings.Contains(patchPayload, `"path":"/ping"`) { + t.Fatalf("unexpected startup-probe patch payload: %s", patchPayload) + } +} + +// TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet runs one orchestration or CLI step. +// Signature: TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet(t *testing.T). +// Why: Mailu backends are Deployments; a kubectl NotFound message on the +// StatefulSet probe path must not turn a successful/no-op Deployment check into +// an auto-heal failure. +func TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet(t *testing.T) { + orch := buildOrchestratorWithStubs(t, config.Config{ + Startup: config.Startup{ + CriticalServiceStartupProbeRepair: true, + CriticalServiceStartupProbeThreshold: 24, + }, + }, []commandStub{ + { + match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "deployment", "mailu-rspamd", "-o", "json"), + out: `{"spec":{"template":{"spec":{"containers":[{"name":"rspamd","livenessProbe":{"httpGet":{"path":"/","port":"http"}},"startupProbe":{"httpGet":{"path":"/","port":"http"}}}]}}}}`, + }, + { + match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "statefulset", "mailu-rspamd", "-o", "json"), + out: `Error from server (NotFound): statefulsets.apps "mailu-rspamd" not found`, + err: errors.New("exit status 1"), + }, + }) + repaired, err := orch.maybeRepairCriticalBackendStartupProbes(context.Background(), "mailu-mailserver", "mailu-rspamd") + if err != nil { + t.Fatalf("expected missing statefulset to be ignored, got %v", err) + } + if len(repaired) != 0 { + t.Fatalf("expected no repair for existing startup probe, got %v", repaired) + } +} + +// atoiForTest runs one orchestration or CLI step. +// Signature: atoiForTest(t *testing.T, raw string) int. +// Why: TCP listener fixtures need the kernel-selected port as a typed config +// value without obscuring test failures. +func atoiForTest(t *testing.T, raw string) int { + t.Helper() + port, err := strconv.Atoi(raw) + if err != nil { + t.Fatalf("parse port %q: %v", raw, err) + } + return port +} + +// staleRWOOrchestrator runs one orchestration or CLI step. +// Signature: staleRWOOrchestrator(t *testing.T, podsJSON string, pvcJSON string, eventsJSON string) *Orchestrator. +// Why: stale RWO tests need a tiny fake API surface for pods, PVCs, and events +// without depending on a live Kubernetes cluster. +func staleRWOOrchestrator(t *testing.T, podsJSON string, pvcJSON string, eventsJSON string) *Orchestrator { + t.Helper() + return buildOrchestratorWithStubs(t, config.Config{}, []commandStub{ + {match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: eventsJSON}, + {match: matchContains("kubectl", "get", "pvc", "-A", "-o", "json"), out: pvcJSON}, + {match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: podsJSON}, + }) +} + +// staleRWOPodJSON runs one orchestration or CLI step. +// Signature: staleRWOPodJSON(name string, deletedAt string, node string, sidecarRunning bool, appRunning bool) string. +// Why: the sidecar-only and live-writer fixtures differ only in container +// status, so one builder keeps the safety distinction obvious. +func staleRWOPodJSON(name string, deletedAt string, node string, sidecarRunning bool, appRunning bool) string { + statuses := []string{} + if sidecarRunning { + statuses = append(statuses, `{"name":"vault-agent","state":{"running":{"startedAt":"`+deletedAt+`"}}}`) + } + if appRunning { + statuses = append(statuses, `{"name":"firefly","state":{"running":{"startedAt":"`+deletedAt+`"}}}`) + } + return `{"metadata":{"namespace":"finance","name":"` + name + `","creationTimestamp":"` + deletedAt + `","deletionTimestamp":"` + deletedAt + `","ownerReferences":[{"kind":"ReplicaSet","name":"firefly"}]},"spec":{"nodeName":"` + node + `","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"firefly-storage"}},{"name":"vault-secret"}],"containers":[{"name":"firefly","volumeMounts":[{"name":"data","mountPath":"/var/www/html/storage"}]},{"name":"vault-agent","volumeMounts":[{"name":"vault-secret","mountPath":"/vault/secrets"}]}]},"status":{"phase":"Running","containerStatuses":[` + strings.Join(statuses, ",") + `]}}` +} + +// replacementRWOPodJSON runs one orchestration or CLI step. +// Signature: replacementRWOPodJSON(name string, node string) string. +// Why: stale-owner tests need a controller sibling blocked in Pending on a +// different node while sharing the same RWO claim. +func replacementRWOPodJSON(name string, node string) string { + return `{"metadata":{"namespace":"finance","name":"` + name + `","creationTimestamp":"` + time.Now().Add(-10*time.Minute).UTC().Format(time.RFC3339) + `","ownerReferences":[{"kind":"ReplicaSet","name":"firefly"}]},"spec":{"nodeName":"` + node + `","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"firefly-storage"}}],"containers":[{"name":"firefly","volumeMounts":[{"name":"data","mountPath":"/var/www/html/storage"}]}]},"status":{"phase":"Pending","containerStatuses":[{"name":"firefly","state":{"waiting":{"reason":"ContainerCreating"}}}]}}` +} + +// jsonUnmarshal runs one orchestration or CLI step. +// Signature: jsonUnmarshal(raw string, target any) error. +// Why: local JSON fixtures should use the same decoder semantics as Kubernetes +// command output parsing. +func jsonUnmarshal(raw string, target any) error { + return json.Unmarshal([]byte(raw), target) +} diff --git a/internal/cluster/orchestrator_host_privilege.go b/internal/cluster/orchestrator_host_privilege.go new file mode 100644 index 0000000..87bd19f --- /dev/null +++ b/internal/cluster/orchestrator_host_privilege.go @@ -0,0 +1,331 @@ +package cluster + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +type hostPrivilegedAction string + +const ( + hostActionSudoPreflight hostPrivilegedAction = "sudo-preflight" + hostActionK3sAgentShow hostPrivilegedAction = "k3s-agent-show" + hostActionK3sAgentIsActive hostPrivilegedAction = "k3s-agent-is-active" + hostActionK3sAgentRestart hostPrivilegedAction = "k3s-agent-restart-no-block" + hostActionHostReboot hostPrivilegedAction = "host-reboot" + hostActionInstallCryptsetup hostPrivilegedAction = "install-cryptsetup-bin" + hostActionModprobeDMCrypt hostPrivilegedAction = "modprobe-dm-crypt" + hostActionCrictlPods hostPrivilegedAction = "crictl-pods" + hostActionCrictlPs hostPrivilegedAction = "crictl-ps" + hostActionCrictlInspect hostPrivilegedAction = "crictl-inspect" + hostActionCrictlStop hostPrivilegedAction = "crictl-stop" +) + +type hostPrivilegeError struct { + Class string + Node string + Action hostPrivilegedAction + Detail string +} + +// Error runs one orchestration or CLI step. +// Signature: (e hostPrivilegeError) Error() string. +// Why: host repair failures need stable operator-facing classes without +// embedding secret material or raw command input. +func (e hostPrivilegeError) Error() string { + parts := []string{strings.TrimSpace(e.Class)} + if strings.TrimSpace(e.Node) != "" { + parts = append(parts, "node="+strings.TrimSpace(e.Node)) + } + if e.Action != "" { + parts = append(parts, "action="+string(e.Action)) + } + if strings.TrimSpace(e.Detail) != "" { + parts = append(parts, strings.TrimSpace(e.Detail)) + } + return strings.Join(parts, " ") +} + +type hostSudoSecret struct { + Data map[string]string `json:"data"` +} + +// preflightHostPrivilege verifies that Ananke can run harmless sudo on a node. +// Signature: (o *Orchestrator) preflightHostPrivilege(ctx context.Context, node string) error. +// Why: host repair should fail with a clear privilege class before an outage +// path needs a package install, k3s-agent restart, runtime inspection, or reboot. +func (o *Orchestrator) preflightHostPrivilege(ctx context.Context, node string) error { + _, err := o.runHostPrivilegedAction(ctx, node, hostActionSudoPreflight, 12*time.Second) + return err +} + +// runHostPrivilegedAction executes one allowlisted sudo action on a managed host. +// Signature: (o *Orchestrator) runHostPrivilegedAction(ctx context.Context, node string, action hostPrivilegedAction, timeout time.Duration, args ...string) (string, error). +// Why: recovery must not turn Vault-backed sudo into arbitrary remote root +// execution; every privileged operation is classified, bounded, and auditable. +func (o *Orchestrator) runHostPrivilegedAction(ctx context.Context, node string, action hostPrivilegedAction, timeout time.Duration, args ...string) (string, error) { + node = strings.TrimSpace(node) + if node == "" { + return "", hostPrivilegeError{Class: "host-privilege-unavailable", Action: action, Detail: "node is empty"} + } + if !o.sshManaged(node) { + return "", hostPrivilegeError{Class: "host-privilege-unavailable", Node: node, Action: action, Detail: "node is not SSH-managed"} + } + commandArgs, err := hostPrivilegedCommandArgs(action, args...) + if err != nil { + return "", err + } + if timeout <= 0 { + timeout = o.hostPrivilegedCommandTimeout() + } + + passwordless := sudoRemoteCommand(false, commandArgs) + out, err := o.sshWithTimeout(ctx, node, passwordless, timeout) + if err == nil { + o.logHostPrivilegedAction(node, action, "passwordless-sudo") + return out, nil + } + if !sudoMayNeedPassword(out, err) { + return out, classifyHostPrivilegeFailure(node, action, err, out) + } + + password, secretClass, secretErr := o.hostSudoPassword(ctx, node) + if secretErr != nil { + return "", hostPrivilegeError{ + Class: "host-privilege-unavailable", + Node: node, + Action: action, + Detail: fmt.Sprintf("%s: %v", secretClass, secretErr), + } + } + passwordBacked := sudoRemoteCommand(true, commandArgs) + out, err = o.sshWithInput(ctx, node, passwordBacked, password+"\n", timeout) + if err != nil { + return out, classifyHostPrivilegeFailure(node, action, err, out) + } + o.logHostPrivilegedAction(node, action, "vault-backed-sudo") + return out, nil +} + +// hostPrivilegedCommandTimeout runs one orchestration or CLI step. +// Signature: (o *Orchestrator) hostPrivilegedCommandTimeout() time.Duration. +// Why: every privileged host command must have a bounded default timeout even +// when a focused unit test builds a partial config. +func (o *Orchestrator) hostPrivilegedCommandTimeout() time.Duration { + seconds := o.cfg.Startup.HostPrivilegedCommandTimeoutSec + if seconds <= 0 { + seconds = 90 + } + return time.Duration(seconds) * time.Second +} + +// logHostPrivilegedAction runs one orchestration or CLI step. +// Signature: (o *Orchestrator) logHostPrivilegedAction(node string, action hostPrivilegedAction, mode string). +// Why: audit logs should record the action class and sudo mode while omitting +// command stdin, passwords, and secret contents. +func (o *Orchestrator) logHostPrivilegedAction(node string, action hostPrivilegedAction, mode string) { + o.log.Printf("host privileged action completed node=%s action=%s mode=%s", node, action, mode) +} + +// hostPrivilegedCommandArgs runs one orchestration or CLI step. +// Signature: hostPrivilegedCommandArgs(action hostPrivilegedAction, args ...string) ([]string, error). +// Why: the Vault-backed sudo path must stay on a strict allowlist instead of +// accepting arbitrary remote root commands. +func hostPrivilegedCommandArgs(action hostPrivilegedAction, args ...string) ([]string, error) { + switch action { + case hostActionSudoPreflight: + return []string{"/usr/bin/systemctl", "--version"}, nil + case hostActionK3sAgentShow: + return []string{"systemctl", "show", "k3s-agent", "--property=ActiveState,SubState,Result,MainPID", "--no-pager"}, nil + case hostActionK3sAgentIsActive: + return []string{"systemctl", "is-active", "k3s-agent"}, nil + case hostActionK3sAgentRestart: + return []string{"systemctl", "--no-block", "restart", "k3s-agent"}, nil + case hostActionHostReboot: + return []string{"systemctl", "reboot"}, nil + case hostActionInstallCryptsetup: + return []string{ + "env", + "DEBIAN_FRONTEND=noninteractive", + "sh", + "-lc", + "apt-get update && apt-get install -y --no-install-recommends cryptsetup-bin && (command -v cryptsetup >/dev/null 2>&1 || test -x /usr/sbin/cryptsetup || test -x /usr/bin/cryptsetup) && echo __ANANKE_CRYPTSETUP_INSTALLED__", + }, nil + case hostActionModprobeDMCrypt: + return []string{"modprobe", "dm_crypt"}, nil + case hostActionCrictlPods: + return []string{"crictl", "pods", "-o", "json"}, nil + case hostActionCrictlPs: + return []string{"crictl", "ps", "-a", "-o", "json"}, nil + case hostActionCrictlInspect: + if len(args) != 1 || strings.TrimSpace(args[0]) == "" || strings.ContainsAny(args[0], " \t\r\n;&|`$<>") { + return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "crictl inspect requires one safe container id"} + } + return []string{"crictl", "inspect", strings.TrimSpace(args[0])}, nil + case hostActionCrictlStop: + if len(args) != 1 || strings.TrimSpace(args[0]) == "" || strings.ContainsAny(args[0], " \t\r\n;&|`$<>") { + return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "crictl stop requires one safe container id"} + } + return []string{"crictl", "stop", strings.TrimSpace(args[0])}, nil + default: + return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "action is not allowlisted"} + } +} + +// sudoRemoteCommand runs one orchestration or CLI step. +// Signature: sudoRemoteCommand(passwordBacked bool, commandArgs []string) string. +// Why: passwordless and password-backed sudo share one command builder so +// quoting and prompt suppression stay consistent. +func sudoRemoteCommand(passwordBacked bool, commandArgs []string) string { + parts := []string{"sudo"} + if passwordBacked { + parts = append(parts, "-S", "-p", "") + } else { + parts = append(parts, "-n") + } + parts = append(parts, commandArgs...) + return shellJoin(parts) +} + +// shellJoin runs one orchestration or CLI step. +// Signature: shellJoin(parts []string) string. +// Why: remote SSH commands need readable simple words and safe quoting for +// shell-sensitive fragments such as package-manager scripts. +func shellJoin(parts []string) string { + out := make([]string, 0, len(parts)) + for _, part := range parts { + out = append(out, shellQuoteIfNeeded(part)) + } + return strings.Join(out, " ") +} + +// shellQuoteIfNeeded runs one orchestration or CLI step. +// Signature: shellQuoteIfNeeded(part string) string. +// Why: audit-friendly commands should avoid unnecessary quotes while still +// protecting whitespace, quotes, and metacharacters. +func shellQuoteIfNeeded(part string) string { + if part == "" { + return shellQuote(part) + } + for _, r := range part { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { + continue + } + switch r { + case '_', '-', '.', '/', '=', ':': + continue + default: + return shellQuote(part) + } + } + return part +} + +// sudoMayNeedPassword runs one orchestration or CLI step. +// Signature: sudoMayNeedPassword(out string, err error) bool. +// Why: Ananke should try Vault-backed sudo only for sudo privilege failures, not +// for unrelated SSH or command errors. +func sudoMayNeedPassword(out string, err error) bool { + full := strings.ToLower(strings.TrimSpace(out + " " + fmt.Sprint(err))) + needles := []string{ + "a password is required", + "password is required", + "sudo: a terminal is required", + "no tty present", + "may not run sudo", + "sudo denied", + "is not in the sudoers file", + } + for _, needle := range needles { + if strings.Contains(full, needle) { + return true + } + } + return false +} + +// classifyHostPrivilegeFailure runs one orchestration or CLI step. +// Signature: classifyHostPrivilegeFailure(node string, action hostPrivilegedAction, err error, out string) error. +// Why: host repair needs distinct timeout, auth, SSH, and command-failure +// classes so status output tells the operator what kind of help is needed. +func classifyHostPrivilegeFailure(node string, action hostPrivilegedAction, err error, out string) error { + full := strings.ToLower(strings.TrimSpace(out + " " + fmt.Sprint(err))) + class := "host-command-failed" + switch { + case strings.Contains(full, "context deadline exceeded"), strings.Contains(full, "timed out"), strings.Contains(full, "timeout"): + class = "host-command-timeout" + case strings.Contains(full, "authentication failure"), strings.Contains(full, "incorrect password"), strings.Contains(full, "try again"): + class = "host-privilege-auth-failed" + case strings.Contains(full, "permission denied"), strings.Contains(full, "publickey"): + class = "host-ssh-unavailable" + } + return hostPrivilegeError{Class: class, Node: node, Action: action, Detail: scrubHostPrivilegeDetail(out, err)} +} + +// scrubHostPrivilegeDetail runs one orchestration or CLI step. +// Signature: scrubHostPrivilegeDetail(out string, err error) string. +// Why: host command details should be compact enough for annotations and avoid +// accidental multi-line or control-character log noise. +func scrubHostPrivilegeDetail(out string, err error) string { + detail := strings.TrimSpace(fmt.Sprint(err)) + if trimmed := strings.TrimSpace(out); trimmed != "" { + if detail != "" { + detail += ": " + } + detail += trimmed + } + return sanitizeCordonAnnotationValue(detail) +} + +// hostSudoPassword runs one orchestration or CLI step. +// Signature: (o *Orchestrator) hostSudoPassword(ctx context.Context, node string) (string, string, error). +// Why: sudo material should come from a configured Vault-synced Kubernetes +// Secret and never be logged or passed through ordinary command arguments. +func (o *Orchestrator) hostSudoPassword(ctx context.Context, node string) (string, string, error) { + namespace := strings.TrimSpace(o.cfg.Startup.HostSudoSecretNamespace) + template := strings.TrimSpace(o.cfg.Startup.HostSudoSecretNameTemplate) + key := strings.TrimSpace(o.cfg.Startup.HostSudoSecretPasswordKey) + if key == "" { + key = "password" + } + if namespace == "" || template == "" { + return "", "secret-lookup-unconfigured", fmt.Errorf("host sudo secret namespace/template is not configured") + } + secretName := strings.ReplaceAll(template, "{node}", node) + secretName = strings.TrimSpace(secretName) + if secretName == "" { + return "", "secret-lookup-invalid", fmt.Errorf("host sudo secret name resolved empty") + } + + out, err := o.kubectl(ctx, 15*time.Second, "-n", namespace, "get", "secret", secretName, "-o", "json") + if err != nil { + return "", "secret-lookup-failed", err + } + var secret hostSudoSecret + if err := json.Unmarshal([]byte(out), &secret); err != nil { + return "", "secret-decode-failed", err + } + encoded := strings.TrimSpace(secret.Data[key]) + if encoded == "" { + keys := make([]string, 0, len(secret.Data)) + for k := range secret.Data { + keys = append(keys, k) + } + sort.Strings(keys) + return "", "secret-key-missing", fmt.Errorf("password key %q missing (available=%s)", key, joinLimited(keys, 4)) + } + decoded, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return "", "secret-decode-failed", err + } + password := strings.TrimRight(string(decoded), "\r\n") + if password == "" { + return "", "secret-empty", fmt.Errorf("password key %q is empty", key) + } + return password, "secret-lookup-ok", nil +} diff --git a/internal/cluster/orchestrator_image_pull_dns.go b/internal/cluster/orchestrator_image_pull_dns.go new file mode 100644 index 0000000..aca15a2 --- /dev/null +++ b/internal/cluster/orchestrator_image_pull_dns.go @@ -0,0 +1,88 @@ +package cluster + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" +) + +// imagePullDNSBlockerReasons classifies image pull failures caused by DNS. +// Signature: (o *Orchestrator) imagePullDNSBlockerReasons(ctx context.Context) (map[string]string, error). +// Why: deleting an unchanged ImagePullBackOff pod does not repair registry DNS; +// Ananke should report the blocker and wait for DNS/registry health to change. +func (o *Orchestrator) imagePullDNSBlockerReasons(ctx context.Context) (map[string]string, error) { + eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json") + if err != nil { + return nil, fmt.Errorf("query events for image-pull DNS scan: %w", err) + } + reasons := map[string]string{} + if strings.TrimSpace(eventsOut) == "" { + return reasons, nil + } + var events eventList + if err := json.Unmarshal([]byte(eventsOut), &events); err != nil { + return nil, fmt.Errorf("decode events for image-pull DNS scan: %w", err) + } + for _, event := range events.Items { + if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") { + continue + } + if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") { + continue + } + reason := strings.TrimSpace(event.Reason) + if reason != "Failed" && reason != "FailedPull" && reason != "ErrImagePull" && reason != "ImagePullBackOff" { + continue + } + message := strings.TrimSpace(event.Message) + if !imagePullMessageHasDNSFailure(message) { + continue + } + key := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name) + if key == "/" { + continue + } + reasons[key] = "ImagePullDNSBlocker:" + imagePullRegistryHost(message) + } + return reasons, nil +} + +// imagePullMessageHasDNSFailure runs one orchestration or CLI step. +// Signature: imagePullMessageHasDNSFailure(message string) bool. +// Why: image pull recycling should be suppressed only for resolver failures, +// leaving ordinary pull/auth errors on their existing paths. +func imagePullMessageHasDNSFailure(message string) bool { + lower := strings.ToLower(message) + if !strings.Contains(lower, "lookup ") { + return false + } + return strings.Contains(lower, "try again") || + strings.Contains(lower, "no such host") || + strings.Contains(lower, "server misbehaving") || + strings.Contains(lower, "temporary failure") || + strings.Contains(lower, "i/o timeout") +} + +// imagePullRegistryHost runs one orchestration or CLI step. +// Signature: imagePullRegistryHost(message string) string. +// Why: operator status should group DNS blockers by registry host instead of +// repeating full kubelet event text for every pod. +func imagePullRegistryHost(message string) string { + lower := strings.ToLower(message) + idx := strings.Index(lower, "lookup ") + if idx < 0 { + return "unknown-registry" + } + rest := strings.TrimSpace(message[idx+len("lookup "):]) + fields := strings.Fields(rest) + if len(fields) == 0 { + return "unknown-registry" + } + host := strings.Trim(fields[0], `"'[]():,`) + if host == "" { + return "unknown-registry" + } + return host +} diff --git a/internal/cluster/orchestrator_longhorn_reconcile.go b/internal/cluster/orchestrator_longhorn_reconcile.go new file mode 100644 index 0000000..7ebfcd3 --- /dev/null +++ b/internal/cluster/orchestrator_longhorn_reconcile.go @@ -0,0 +1,265 @@ +package cluster + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" +) + +type longhornNodeList struct { + Items []longhornNode `json:"items"` +} + +type longhornNode struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + AllowScheduling *bool `json:"allowScheduling"` + } `json:"spec"` + Status struct { + Conditions []longhornCondition `json:"conditions"` + } `json:"status"` +} + +type longhornCondition struct { + Type string `json:"type"` + Status string `json:"status"` + Reason string `json:"reason"` + Message string `json:"message"` +} + +type daemonSetResource struct { + Metadata struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Selector struct { + MatchLabels map[string]string `json:"matchLabels"` + } `json:"selector"` + } `json:"spec"` +} + +// reconcileLonghornKubernetesReadiness repairs safe Kubernetes/Longhorn drift. +// Signature: (o *Orchestrator) reconcileLonghornKubernetesReadiness(ctx context.Context) (int, error). +// Why: Kubernetes Ready=True is not enough for PVC scheduling when Longhorn's +// manager, labels, and node conditions disagree with the Kubernetes node model. +func (o *Orchestrator) reconcileLonghornKubernetesReadiness(ctx context.Context) (int, error) { + longhornNodes, err := o.queryLonghornNodes(ctx) + if err != nil { + return 0, err + } + if len(longhornNodes.Items) == 0 { + return 0, nil + } + managerDS, err := o.queryLonghornManagerDaemonSet(ctx) + if err != nil { + return 0, err + } + selector := normalizeSelectorLabels(managerDS.Spec.Selector.MatchLabels) + if len(selector) == 0 { + return 0, fmt.Errorf("longhorn manager daemonset has no matchLabels selector") + } + nodes, err := o.queryReadyNodes(ctx) + if err != nil { + return 0, err + } + kubeNodes := map[string]nodeReadyItem{} + for _, node := range nodes.Items { + name := strings.TrimSpace(node.Metadata.Name) + if name != "" { + kubeNodes[name] = node + } + } + managerPods, err := o.longhornManagerPodsByNode(ctx, selector) + if err != nil { + return 0, err + } + + repaired := 0 + errs := []string{} + for _, lhNode := range longhornNodes.Items { + name := strings.TrimSpace(lhNode.Metadata.Name) + if name == "" { + continue + } + kubeNode, ok := kubeNodes[name] + if !ok { + continue + } + if _, hasManager := managerPods[name]; hasManager { + continue + } + readyCond := longhornConditionByType(lhNode, "Ready") + if readyCond == nil || strings.EqualFold(strings.TrimSpace(readyCond.Status), "True") { + continue + } + if !strings.EqualFold(strings.TrimSpace(readyCond.Reason), "ManagerPodMissing") { + continue + } + missing := missingSelectorLabels(kubeNode.Metadata.Labels, selector) + if len(missing) == 0 { + o.log.Printf("warning: Longhorn manager pod missing on node=%s but node already matches selector; leaving for DaemonSet scheduler reconciliation", name) + continue + } + + labels := make([]string, 0, len(missing)) + for key, value := range missing { + labels = append(labels, key+"="+value) + } + sort.Strings(labels) + args := append([]string{"label", "node", name, "--overwrite"}, labels...) + o.log.Printf("warning: restoring Longhorn manager selector labels on node=%s labels=%s reason=LonghornNodeExistsAndManagerPodMissing", name, strings.Join(labels, ",")) + if _, err := o.kubectl(ctx, 25*time.Second, args...); err != nil { + errs = append(errs, fmt.Sprintf("%s restore Longhorn selector labels: %v", name, err)) + continue + } + repaired++ + o.noteStartupAutoHeal(fmt.Sprintf("restored Longhorn manager selector labels on %s: %s", name, strings.Join(labels, ","))) + } + if len(errs) > 0 { + return repaired, errors.New(strings.Join(errs, "; ")) + } + return repaired, nil +} + +// queryLonghornNodes runs one orchestration or CLI step. +// Signature: (o *Orchestrator) queryLonghornNodes(ctx context.Context) (longhornNodeList, error). +// Why: Longhorn/Kubernetes drift reconciliation starts from Longhorn's own node +// model rather than assuming Kubernetes Ready is sufficient for storage. +func (o *Orchestrator) queryLonghornNodes(ctx context.Context) (longhornNodeList, error) { + out, err := o.kubectl(ctx, 30*time.Second, "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json") + if err != nil { + if isNotFoundErr(err) { + return longhornNodeList{}, nil + } + return longhornNodeList{}, fmt.Errorf("query longhorn nodes: %w", err) + } + if strings.TrimSpace(out) == "" { + return longhornNodeList{}, nil + } + var nodes longhornNodeList + if err := json.Unmarshal([]byte(out), &nodes); err != nil { + return longhornNodeList{}, fmt.Errorf("decode longhorn nodes: %w", err) + } + return nodes, nil +} + +// queryLonghornManagerDaemonSet runs one orchestration or CLI step. +// Signature: (o *Orchestrator) queryLonghornManagerDaemonSet(ctx context.Context) (daemonSetResource, error). +// Why: manager selector labels are installation policy, so repair must read +// the live DaemonSet selector instead of hard-coding a Titan label. +func (o *Orchestrator) queryLonghornManagerDaemonSet(ctx context.Context) (daemonSetResource, error) { + out, err := o.kubectl(ctx, 20*time.Second, "-n", "longhorn-system", "get", "daemonset", "longhorn-manager", "-o", "json") + if err != nil { + return daemonSetResource{}, fmt.Errorf("query longhorn manager daemonset: %w", err) + } + var ds daemonSetResource + if err := json.Unmarshal([]byte(out), &ds); err != nil { + return daemonSetResource{}, fmt.Errorf("decode longhorn manager daemonset: %w", err) + } + return ds, nil +} + +// longhornManagerPodsByNode runs one orchestration or CLI step. +// Signature: (o *Orchestrator) longhornManagerPodsByNode(ctx context.Context, selector map[string]string) (map[string]struct{}, error). +// Why: manager-pod presence is the postcondition for selector repair and +// distinguishes label drift from a healthy Longhorn node. +func (o *Orchestrator) longhornManagerPodsByNode(ctx context.Context, selector map[string]string) (map[string]struct{}, error) { + args := []string{"-n", "longhorn-system", "get", "pods", "-o", "json"} + if len(selector) > 0 { + args = append(args, "-l", selectorLabelString(selector)) + } + out, err := o.kubectl(ctx, 20*time.Second, args...) + if err != nil { + return nil, fmt.Errorf("query longhorn manager pods: %w", err) + } + var pods podList + if err := json.Unmarshal([]byte(out), &pods); err != nil { + return nil, fmt.Errorf("decode longhorn manager pods: %w", err) + } + byNode := map[string]struct{}{} + for _, pod := range pods.Items { + node := strings.TrimSpace(pod.Spec.NodeName) + if node == "" { + continue + } + if podMatchesLabels(pod.Metadata.Labels, selector) { + byNode[node] = struct{}{} + } + } + return byNode, nil +} + +// normalizeSelectorLabels runs one orchestration or CLI step. +// Signature: normalizeSelectorLabels(labels map[string]string) map[string]string. +// Why: empty selector fragments should not produce accidental node label writes. +func normalizeSelectorLabels(labels map[string]string) map[string]string { + out := map[string]string{} + for key, value := range labels { + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + if key != "" && value != "" { + out[key] = value + } + } + return out +} + +// selectorLabelString runs one orchestration or CLI step. +// Signature: selectorLabelString(labels map[string]string) string. +// Why: kubectl label selectors need stable ordering for deterministic tests and +// readable logs. +func selectorLabelString(labels map[string]string) string { + parts := make([]string, 0, len(labels)) + for key, value := range labels { + parts = append(parts, key+"="+value) + } + sort.Strings(parts) + return strings.Join(parts, ",") +} + +// podMatchesLabels runs one orchestration or CLI step. +// Signature: podMatchesLabels(labels map[string]string, selector map[string]string) bool. +// Why: Longhorn manager pod checks should use the DaemonSet selector rather +// than assuming a specific app label. +func podMatchesLabels(labels map[string]string, selector map[string]string) bool { + for key, value := range selector { + if labels == nil || strings.TrimSpace(labels[key]) != value { + return false + } + } + return true +} + +// missingSelectorLabels runs one orchestration or CLI step. +// Signature: missingSelectorLabels(labels map[string]string, selector map[string]string) map[string]string. +// Why: label-drift repair should write exactly the manager selector labels that +// are absent or wrong on the node. +func missingSelectorLabels(labels map[string]string, selector map[string]string) map[string]string { + missing := map[string]string{} + for key, value := range selector { + if labels == nil || strings.TrimSpace(labels[key]) != value { + missing[key] = value + } + } + return missing +} + +// longhornConditionByType runs one orchestration or CLI step. +// Signature: longhornConditionByType(node longhornNode, condType string) *longhornCondition. +// Why: Longhorn node condition parsing is shared by readiness and schedulability +// checks and should remain case-insensitive. +func longhornConditionByType(node longhornNode, condType string) *longhornCondition { + for i := range node.Status.Conditions { + if strings.EqualFold(strings.TrimSpace(node.Status.Conditions[i].Type), condType) { + return &node.Status.Conditions[i] + } + } + return nil +} diff --git a/internal/cluster/orchestrator_longhorn_recovery.go b/internal/cluster/orchestrator_longhorn_recovery.go index 2639581..051bc2c 100644 --- a/internal/cluster/orchestrator_longhorn_recovery.go +++ b/internal/cluster/orchestrator_longhorn_recovery.go @@ -104,25 +104,37 @@ func (o *Orchestrator) repairEncryptedVolumeMountPrereqs(ctx context.Context, po // Why: kubelet's encrypted Longhorn mount helper shells into the host namespace, // so the package must exist on the node host, not merely inside a workload pod. func (o *Orchestrator) ensureHostCryptsetup(ctx context.Context, node string) error { - command := strings.Join([]string{ + checkCommand := strings.Join([]string{ "set -eu", "if command -v cryptsetup >/dev/null 2>&1 || [ -x /usr/sbin/cryptsetup ] || [ -x /usr/bin/cryptsetup ]; then echo __ANANKE_CRYPTSETUP_PRESENT__; exit 0; fi", "if ! command -v apt-get >/dev/null 2>&1; then echo __ANANKE_CRYPTSETUP_NO_APT__; exit 42; fi", - "sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update", - "sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cryptsetup-bin", - "if command -v cryptsetup >/dev/null 2>&1 || [ -x /usr/sbin/cryptsetup ] || [ -x /usr/bin/cryptsetup ]; then echo __ANANKE_CRYPTSETUP_INSTALLED__; exit 0; fi", - "echo __ANANKE_CRYPTSETUP_INSTALL_FAILED__", - "exit 43", + "echo __ANANKE_CRYPTSETUP_MISSING__", + "exit 41", }, "; ") - out, err := o.sshWithTimeout(ctx, node, command, 5*time.Minute) + out, err := o.sshWithTimeout(ctx, node, checkCommand, 25*time.Second) + if err == nil && strings.Contains(out, "__ANANKE_CRYPTSETUP_PRESENT__") { + o.log.Printf("ensured cryptsetup prerequisite on %s: __ANANKE_CRYPTSETUP_PRESENT__", node) + return nil + } + if strings.Contains(out, "__ANANKE_CRYPTSETUP_NO_APT__") { + return fmt.Errorf("install cryptsetup-bin: apt-get is unavailable on host") + } + + out, err = o.runHostPrivilegedAction(ctx, node, hostActionInstallCryptsetup, 5*time.Minute) if err != nil { return fmt.Errorf("install cryptsetup-bin: %w (output=%s)", err, strings.TrimSpace(out)) } - trimmed := strings.TrimSpace(out) - o.log.Printf("ensured cryptsetup prerequisite on %s: %s", node, trimmed) - if strings.Contains(trimmed, "__ANANKE_CRYPTSETUP_INSTALLED__") { + if strings.Contains(out, "__ANANKE_CRYPTSETUP_INSTALLED__") { + o.log.Printf("ensured cryptsetup prerequisite on %s: __ANANKE_CRYPTSETUP_INSTALLED__", node) o.noteStartupAutoHeal(fmt.Sprintf("installed cryptsetup on %s", node)) + return nil } + verifyOut, verifyErr := o.sshWithTimeout(ctx, node, checkCommand, 25*time.Second) + if verifyErr != nil || !strings.Contains(verifyOut, "__ANANKE_CRYPTSETUP_PRESENT__") { + return fmt.Errorf("cryptsetup-bin install completed but verification failed: %w (output=%s)", verifyErr, strings.TrimSpace(verifyOut)) + } + o.log.Printf("ensured cryptsetup prerequisite on %s: __ANANKE_CRYPTSETUP_INSTALLED__", node) + o.noteStartupAutoHeal(fmt.Sprintf("installed cryptsetup on %s", node)) return nil } diff --git a/internal/cluster/orchestrator_node_runtime_recovery.go b/internal/cluster/orchestrator_node_runtime_recovery.go new file mode 100644 index 0000000..99c5ee9 --- /dev/null +++ b/internal/cluster/orchestrator_node_runtime_recovery.go @@ -0,0 +1,162 @@ +package cluster + +import ( + "context" + "fmt" + "strings" + "time" +) + +// recoverManagedNodeRuntime runs one orchestration or CLI step. +// Signature: (o *Orchestrator) recoverManagedNodeRuntime(ctx context.Context, node string, alreadyUnschedulable bool, reason string) error. +// Why: kubelet/containerd wedges need one bounded escalation ladder: cordon, +// non-blocking k3s-agent restart, health proof, and optional controlled reboot. +func (o *Orchestrator) recoverManagedNodeRuntime(ctx context.Context, node string, alreadyUnschedulable bool, reason string) error { + node = strings.TrimSpace(node) + if node == "" { + return fmt.Errorf("node is empty") + } + if !alreadyUnschedulable { + if err := o.cordonNodeWithLease(ctx, node, cordonReasonKubeletProxy, reason); err != nil { + return fmt.Errorf("cordon before kubelet restart: %w", err) + } + } + + o.log.Printf("warning: detected node runtime repair signal on %s; issuing non-blocking k3s-agent restart reason=%s", node, sanitizeCordonAnnotationValue(reason)) + if _, err := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentRestart, 30*time.Second); err != nil { + if !alreadyUnschedulable { + o.bestEffort("uncordon node after failed kubelet proxy repair", func() error { + return o.uncordonAndClearCordonLease(ctx, node, cordonReasonKubeletProxy) + }) + } + return fmt.Errorf("restart k3s-agent: %w", err) + } + + readyErr := o.waitForManagedNodeRuntimeReady(ctx, node, o.nodeRuntimeRestartWait()) + if readyErr == nil { + return o.releaseRuntimeRecoveryCordon(ctx, node, alreadyUnschedulable) + } + + showOut, showErr := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentShow, 20*time.Second) + if showErr != nil { + return fmt.Errorf("%v; k3s-agent state check failed: %w", readyErr, showErr) + } + if !k3sAgentStateRequiresReboot(showOut) { + return readyErr + } + if !o.cfg.Startup.HostRepairAllowReboot { + return fmt.Errorf("node runtime requires controlled reboot but host_repair_allow_reboot is false; k3s-agent state=%s", summarizeSystemctlShow(showOut)) + } + + o.log.Printf("warning: escalating node runtime recovery to controlled reboot node=%s state=%s", node, summarizeSystemctlShow(showOut)) + if _, err := o.runHostPrivilegedAction(ctx, node, hostActionHostReboot, 30*time.Second); err != nil { + return fmt.Errorf("controlled reboot command failed: %w", err) + } + if err := o.waitForManagedNodeRuntimeReady(ctx, node, o.nodeRuntimeRebootWait()); err != nil { + return fmt.Errorf("node did not recover after controlled reboot: %w", err) + } + return o.releaseRuntimeRecoveryCordon(ctx, node, alreadyUnschedulable) +} + +// releaseRuntimeRecoveryCordon runs one orchestration or CLI step. +// Signature: (o *Orchestrator) releaseRuntimeRecoveryCordon(ctx context.Context, node string, alreadyUnschedulable bool) error. +// Why: nodes Ananke cordoned for repair should return to service after proof of +// recovery, while pre-existing cordons remain under their original owner. +func (o *Orchestrator) releaseRuntimeRecoveryCordon(ctx context.Context, node string, alreadyUnschedulable bool) error { + if alreadyUnschedulable { + return nil + } + if err := o.uncordonAndClearCordonLease(ctx, node, cordonReasonKubeletProxy); err != nil { + return fmt.Errorf("uncordon after kubelet proxy repair: %w", err) + } + return nil +} + +// nodeRuntimeRestartWait runs one orchestration or CLI step. +// Signature: (o *Orchestrator) nodeRuntimeRestartWait() time.Duration. +// Why: k3s-agent restart recovery needs a bounded wait even when tests build a +// partial startup config. +func (o *Orchestrator) nodeRuntimeRestartWait() time.Duration { + seconds := o.cfg.Startup.NodeRuntimeRestartWaitSeconds + if seconds <= 0 { + seconds = 140 + } + return time.Duration(seconds) * time.Second +} + +// nodeRuntimeRebootWait runs one orchestration or CLI step. +// Signature: (o *Orchestrator) nodeRuntimeRebootWait() time.Duration. +// Why: controlled reboot recovery has a longer but still finite postcondition +// window for SSH, systemd, kubelet, and Kubernetes readiness. +func (o *Orchestrator) nodeRuntimeRebootWait() time.Duration { + seconds := o.cfg.Startup.NodeRuntimeRebootWaitSeconds + if seconds <= 0 { + seconds = 420 + } + return time.Duration(seconds) * time.Second +} + +// waitForManagedNodeRuntimeReady runs one orchestration or CLI step. +// Signature: (o *Orchestrator) waitForManagedNodeRuntimeReady(ctx context.Context, node string, wait time.Duration) error. +// Why: runtime repair is not complete until Kubernetes Ready, kubelet proxy, and +// k3s-agent service state all agree the node is usable. +func (o *Orchestrator) waitForManagedNodeRuntimeReady(ctx context.Context, node string, wait time.Duration) error { + if wait <= 0 { + wait = 140 * time.Second + } + kubeWait := wait - 20*time.Second + if kubeWait < 10*time.Second { + kubeWait = 10 * time.Second + } + if _, err := o.kubectl(ctx, wait, "wait", "node/"+node, "--for=condition=Ready", "--timeout="+fmt.Sprintf("%.0fs", kubeWait.Seconds())); err != nil { + return fmt.Errorf("wait Ready after k3s-agent restart: %w", err) + } + healthy, err := o.kubeletProxyHealthy(ctx, node) + if !healthy { + return fmt.Errorf("proxy still broken after k3s-agent restart: %v", err) + } + out, err := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentIsActive, 12*time.Second) + if err != nil { + return fmt.Errorf("k3s-agent active check: %w", err) + } + if strings.TrimSpace(out) != "active" { + return fmt.Errorf("k3s-agent is-active returned %q", strings.TrimSpace(out)) + } + return nil +} + +// k3sAgentStateRequiresReboot runs one orchestration or CLI step. +// Signature: k3sAgentStateRequiresReboot(out string) bool. +// Why: only the stuck systemd states observed during hard runtime wedges should +// escalate from non-blocking restart to controlled reboot. +func k3sAgentStateRequiresReboot(out string) bool { + lower := strings.ToLower(out) + needles := []string{ + "activestate=deactivating", + "activestate=activating", + "substate=stop-sigterm", + "substate=stop-sigkill", + "substate=final-sigterm", + "substate=final-sigkill", + "substate=start", + "result=timeout", + } + for _, needle := range needles { + if strings.Contains(lower, needle) { + return true + } + } + return false +} + +// summarizeSystemctlShow runs one orchestration or CLI step. +// Signature: summarizeSystemctlShow(out string) string. +// Why: operator-facing runtime incidents need compact systemd state summaries +// rather than full multi-line command output. +func summarizeSystemctlShow(out string) string { + lines := lines(out) + if len(lines) == 0 { + return "unknown" + } + return sanitizeCordonAnnotationValue(joinLimited(lines, 4)) +} diff --git a/internal/cluster/orchestrator_poststart_exec.go b/internal/cluster/orchestrator_poststart_exec.go index 8822192..692db36 100644 --- a/internal/cluster/orchestrator_poststart_exec.go +++ b/internal/cluster/orchestrator_poststart_exec.go @@ -187,6 +187,81 @@ func (o *Orchestrator) ssh(ctx context.Context, node string, command string) (st // Signature: (o *Orchestrator) sshWithTimeout(ctx context.Context, node string, command string, timeout time.Duration) (string, error). // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve. func (o *Orchestrator) sshWithTimeout(ctx context.Context, node string, command string, timeout time.Duration) (string, error) { + attempts, attemptNames, knownHostsFiles, repairHosts := o.sshCommandAttempts(node, command) + + var lastOut string + var lastErr error + for i, args := range attempts { + out, err := o.run(ctx, timeout, "ssh", args...) + if err == nil { + if i > 0 { + o.log.Printf("warning: ssh %s path failed for %s, using %s path", attemptNames[i-1], node, attemptNames[i]) + } + return out, nil + } + if sshutil.ShouldAttemptKnownHostsRepair(out, err) { + o.log.Printf("warning: ssh failure on %s via %s path may be host-key related; repairing known_hosts and retrying once", node, attemptNames[i]) + sshutil.RepairKnownHosts(ctx, o.log, knownHostsFiles, repairHosts, o.cfg.SSHPort) + retryOut, retryErr := o.run(ctx, timeout, "ssh", args...) + if retryErr == nil { + return retryOut, nil + } + out = retryOut + err = retryErr + } + lastOut = out + lastErr = err + if i < len(attempts)-1 { + o.log.Printf("warning: ssh %s path failed for %s: %v; trying %s path", attemptNames[i], node, err, attemptNames[i+1]) + } + } + return lastOut, lastErr +} + +// sshWithInput runs one orchestration or CLI step. +// Signature: (o *Orchestrator) sshWithInput(ctx context.Context, node string, command string, input string, timeout time.Duration) (string, error). +// Why: Vault-backed sudo must send passwords over stdin, not in the SSH command +// line or logs, while still using Ananke's configured node route. +func (o *Orchestrator) sshWithInput(ctx context.Context, node string, command string, input string, timeout time.Duration) (string, error) { + if o.sshInputOverride != nil { + return o.sshInputOverride(ctx, timeout, node, command, input) + } + attempts, attemptNames, knownHostsFiles, repairHosts := o.sshCommandAttempts(node, command) + + var lastOut string + var lastErr error + for i, args := range attempts { + out, err := o.runSensitiveWithInput(ctx, timeout, input, "ssh", args...) + if err == nil { + if i > 0 { + o.log.Printf("warning: ssh %s path failed for %s, using %s path", attemptNames[i-1], node, attemptNames[i]) + } + return out, nil + } + if sshutil.ShouldAttemptKnownHostsRepair(out, err) { + o.log.Printf("warning: ssh failure on %s via %s path may be host-key related; repairing known_hosts and retrying once", node, attemptNames[i]) + sshutil.RepairKnownHosts(ctx, o.log, knownHostsFiles, repairHosts, o.cfg.SSHPort) + retryOut, retryErr := o.runSensitiveWithInput(ctx, timeout, input, "ssh", args...) + if retryErr == nil { + return retryOut, nil + } + out = retryOut + err = retryErr + } + lastOut = out + lastErr = err + if i < len(attempts)-1 { + o.log.Printf("warning: ssh %s path failed for %s: %v; trying %s path", attemptNames[i], node, err, attemptNames[i+1]) + } + } + return lastOut, lastErr +} + +// sshCommandAttempts runs one orchestration or CLI step. +// Signature: (o *Orchestrator) sshCommandAttempts(node string, command string) ([][]string, []string, []string, []string). +// Why: ordinary SSH and stdin-backed SSH must use identical host, identity, +// port, jump-host, and known-host repair behavior. +func (o *Orchestrator) sshCommandAttempts(node string, command string) ([][]string, []string, []string, []string) { host := node if mapped, ok := o.cfg.SSHNodeHosts[node]; ok && strings.TrimSpace(mapped) != "" { host = strings.TrimSpace(mapped) @@ -240,34 +315,7 @@ func (o *Orchestrator) sshWithTimeout(ctx context.Context, node string, command direct = append(direct, target, command) attempts = append(attempts, direct) attemptNames = append(attemptNames, "direct") - - var lastOut string - var lastErr error - for i, args := range attempts { - out, err := o.run(ctx, timeout, "ssh", args...) - if err == nil { - if i > 0 { - o.log.Printf("warning: ssh %s path failed for %s, using %s path", attemptNames[i-1], node, attemptNames[i]) - } - return out, nil - } - if sshutil.ShouldAttemptKnownHostsRepair(out, err) { - o.log.Printf("warning: ssh failure on %s via %s path may be host-key related; repairing known_hosts and retrying once", node, attemptNames[i]) - sshutil.RepairKnownHosts(ctx, o.log, knownHostsFiles, repairHosts, o.cfg.SSHPort) - retryOut, retryErr := o.run(ctx, timeout, "ssh", args...) - if retryErr == nil { - return retryOut, nil - } - out = retryOut - err = retryErr - } - lastOut = out - lastErr = err - if i < len(attempts)-1 { - o.log.Printf("warning: ssh %s path failed for %s: %v; trying %s path", attemptNames[i], node, err, attemptNames[i+1]) - } - } - return lastOut, lastErr + return attempts, attemptNames, knownHostsFiles, repairHosts } // run runs one orchestration or CLI step. @@ -308,6 +356,31 @@ func (o *Orchestrator) runSensitive(ctx context.Context, timeout time.Duration, return trimmed, nil } +// runSensitiveWithInput runs one orchestration or CLI step. +// Signature: (o *Orchestrator) runSensitiveWithInput(ctx context.Context, timeout time.Duration, input string, name string, args ...string) (string, error). +// Why: sudo passwords and similar secret stdin must not be passed through +// ordinary command arguments or dry-run logging. +func (o *Orchestrator) runSensitiveWithInput(ctx context.Context, timeout time.Duration, input string, name string, args ...string) (string, error) { + runCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + cmd := exec.CommandContext(runCtx, name, args...) + cmd.Env = os.Environ() + if o.runner.Kubeconfig != "" { + cmd.Env = append(cmd.Env, "KUBECONFIG="+o.runner.Kubeconfig) + } + cmd.Stdin = strings.NewReader(input) + out, err := cmd.CombinedOutput() + trimmed := strings.TrimSpace(string(out)) + if err != nil { + if trimmed == "" { + return "", fmt.Errorf("%s failed: %w", name, err) + } + return trimmed, fmt.Errorf("%s failed: %w", name, err) + } + return trimmed, nil +} + // lines runs one orchestration or CLI step. // Signature: lines(in string) []string. // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve. diff --git a/internal/cluster/orchestrator_service_stability.go b/internal/cluster/orchestrator_service_stability.go index 9102d9d..599403a 100644 --- a/internal/cluster/orchestrator_service_stability.go +++ b/internal/cluster/orchestrator_service_stability.go @@ -99,6 +99,7 @@ func (o *Orchestrator) waitForServiceChecklist(ctx context.Context) error { lastRecycleAttempt := time.Time{} lastReplicaHeal := time.Time{} lastIngressHeal := time.Time{} + lastTCPHeal := time.Time{} for { o.maybeAutoRecycleStuckPods(ctx, &lastRecycleAttempt) o.maybeAutoHealCriticalWorkloadReplicas(ctx, &lastReplicaHeal) @@ -110,6 +111,7 @@ func (o *Orchestrator) waitForServiceChecklist(ctx context.Context) error { return nil } o.maybeAutoHealIngressHostBackends(ctx, &lastIngressHeal, lastFailure) + o.maybeAutoHealTCPServiceBackends(ctx, &lastTCPHeal) if lastFailure != prevFailure || time.Since(lastLogged) >= 30*time.Second { remaining := time.Until(deadline).Round(time.Second) if remaining < 0 { @@ -134,7 +136,7 @@ func (o *Orchestrator) waitForServiceChecklist(ctx context.Context) error { // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve. func (o *Orchestrator) serviceChecklistReady(ctx context.Context) (bool, string) { checks := o.cfg.Startup.ServiceChecklist - if len(checks) == 0 { + if len(checks) == 0 && len(o.cfg.Startup.TCPServiceChecklist) == 0 { return true, "no checklist items configured" } for _, check := range checks { @@ -147,7 +149,10 @@ func (o *Orchestrator) serviceChecklistReady(ctx context.Context) (bool, string) return false, fmt.Sprintf("%s: %s", name, detail) } } - return true, fmt.Sprintf("checks=%d", len(checks)) + if ok, detail := o.tcpServiceChecklistReady(ctx); !ok { + return false, "tcp " + detail + } + return true, fmt.Sprintf("checks=%d tcp-checks=%d", len(checks), len(o.cfg.Startup.TCPServiceChecklist)) } // serviceCheckReady runs one orchestration or CLI step. diff --git a/internal/cluster/orchestrator_stale_rwo_owner.go b/internal/cluster/orchestrator_stale_rwo_owner.go new file mode 100644 index 0000000..c82d26c --- /dev/null +++ b/internal/cluster/orchestrator_stale_rwo_owner.go @@ -0,0 +1,319 @@ +package cluster + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +type staleRWOOwnerDecision struct { + Reason string + ForceDelete bool + Unsafe bool +} + +type persistentVolumeClaimList struct { + Items []persistentVolumeClaimResource `json:"items"` +} + +type persistentVolumeClaimResource struct { + Metadata struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + AccessModes []string `json:"accessModes"` + VolumeName string `json:"volumeName"` + } `json:"spec"` +} + +// staleRWOPVCOwnerDecisions finds stale terminating single-writer pods safely. +// Signature: (o *Orchestrator) staleRWOPVCOwnerDecisions(ctx context.Context, pods podList, grace time.Duration) (map[string]staleRWOOwnerDecision, error). +// Why: RWO volume handoff needs container/mount-level safety, not blind repeated +// pod deletion, so sidecar-only stale owners can clear while live writers block. +func (o *Orchestrator) staleRWOPVCOwnerDecisions(ctx context.Context, pods podList, grace time.Duration) (map[string]staleRWOOwnerDecision, error) { + eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json") + if err != nil { + return nil, fmt.Errorf("query events for stale RWO owner scan: %w", err) + } + var events eventList + if strings.TrimSpace(eventsOut) != "" { + if err := json.Unmarshal([]byte(eventsOut), &events); err != nil { + return nil, fmt.Errorf("decode events for stale RWO owner scan: %w", err) + } + } + pvcs, err := o.queryPVCs(ctx) + if err != nil { + return nil, err + } + + now := time.Now() + podsByController := map[string][]podResource{} + for _, pod := range pods.Items { + controllerKey := podControllerKey(pod) + if controllerKey == "" { + continue + } + podsByController[controllerKey] = append(podsByController[controllerKey], pod) + } + + decisions := map[string]staleRWOOwnerDecision{} + for _, oldPod := range pods.Items { + oldKey := podKey(oldPod) + if oldKey == "" || oldPod.Metadata.DeletionTimestamp == nil || !podControllerOwned(oldPod) { + continue + } + if now.Sub(*oldPod.Metadata.DeletionTimestamp) < grace { + continue + } + oldNode := strings.TrimSpace(oldPod.Spec.NodeName) + if oldNode == "" { + continue + } + blockedClaims := podRWOPVCVolumeNames(oldPod, pvcs) + if len(blockedClaims) == 0 { + continue + } + controllerKey := podControllerKey(oldPod) + replacements := podsByController[controllerKey] + replacementKey := "" + for _, replacement := range replacements { + if podKey(replacement) == oldKey { + continue + } + if strings.TrimSpace(replacement.Spec.NodeName) == "" || strings.TrimSpace(replacement.Spec.NodeName) == oldNode { + continue + } + if !replacementPodPendingForAttach(replacement) { + continue + } + if !podHasBlockedAttachEvent(replacement, events) { + continue + } + if !podsShareAnyClaim(oldPod, replacement, blockedClaims) { + continue + } + replacementKey = podKey(replacement) + break + } + if replacementKey == "" { + continue + } + + unsafe := runningContainersMountAnyVolume(oldPod, blockedClaims) + claims := make([]string, 0, len(blockedClaims)) + for _, claim := range blockedClaims { + claims = append(claims, claim) + } + sort.Strings(claims) + if unsafe { + decisions[oldKey] = staleRWOOwnerDecision{ + Reason: fmt.Sprintf("UnsafeStaleRWOPVCOwner:%s:%s", oldNode, strings.Join(claims, ",")), + Unsafe: true, + } + continue + } + decisions[oldKey] = staleRWOOwnerDecision{ + Reason: fmt.Sprintf("SidecarOnlyStaleRWOPVCOwner:%s:%s->%s", oldNode, strings.Join(claims, ","), replacementKey), + ForceDelete: true, + } + } + return decisions, nil +} + +// queryPVCs runs one orchestration or CLI step. +// Signature: (o *Orchestrator) queryPVCs(ctx context.Context) (map[string]persistentVolumeClaimResource, error). +// Why: stale-owner recovery must confirm single-writer PVC semantics before it +// considers force-deleting a terminating pod object. +func (o *Orchestrator) queryPVCs(ctx context.Context) (map[string]persistentVolumeClaimResource, error) { + out, err := o.kubectl(ctx, 30*time.Second, "get", "pvc", "-A", "-o", "json") + if err != nil { + return nil, fmt.Errorf("query pvcs for stale RWO owner scan: %w", err) + } + pvcs := map[string]persistentVolumeClaimResource{} + if strings.TrimSpace(out) == "" { + return pvcs, nil + } + var list persistentVolumeClaimList + if err := json.Unmarshal([]byte(out), &list); err != nil { + return nil, fmt.Errorf("decode pvcs for stale RWO owner scan: %w", err) + } + for _, pvc := range list.Items { + key := strings.TrimSpace(pvc.Metadata.Namespace) + "/" + strings.TrimSpace(pvc.Metadata.Name) + if strings.TrimSpace(pvc.Metadata.Namespace) != "" && strings.TrimSpace(pvc.Metadata.Name) != "" { + pvcs[key] = pvc + } + } + return pvcs, nil +} + +// podRWOPVCVolumeNames runs one orchestration or CLI step. +// Signature: podRWOPVCVolumeNames(pod podResource, pvcs map[string]persistentVolumeClaimResource) map[string]string. +// Why: container mount safety checks need the pod volume names that correspond +// to ReadWriteOnce PVC claims. +func podRWOPVCVolumeNames(pod podResource, pvcs map[string]persistentVolumeClaimResource) map[string]string { + volumes := map[string]string{} + for _, volume := range pod.Spec.Volumes { + if volume.PersistentVolumeClaim == nil { + continue + } + claim := strings.TrimSpace(volume.PersistentVolumeClaim.ClaimName) + if claim == "" { + continue + } + pvc, ok := pvcs[strings.TrimSpace(pod.Metadata.Namespace)+"/"+claim] + if !ok || !pvcSingleWriter(pvc) { + continue + } + volumes[strings.TrimSpace(volume.Name)] = claim + } + return volumes +} + +// pvcSingleWriter runs one orchestration or CLI step. +// Signature: pvcSingleWriter(pvc persistentVolumeClaimResource) bool. +// Why: the stale-owner path is limited to exclusive-writer PVC modes and should +// leave shared volumes to normal Kubernetes cleanup. +func pvcSingleWriter(pvc persistentVolumeClaimResource) bool { + for _, mode := range pvc.Spec.AccessModes { + normalized := strings.ToLower(strings.TrimSpace(mode)) + if normalized == "readwriteonce" || normalized == "readwriteoncepod" { + return true + } + } + return false +} + +// podControllerKey runs one orchestration or CLI step. +// Signature: podControllerKey(pod podResource) string. +// Why: stale owner and replacement pods should be grouped by controller without +// hard-coding ReplicaSet, StatefulSet, or application names. +func podControllerKey(pod podResource) string { + ns := strings.TrimSpace(pod.Metadata.Namespace) + for _, owner := range pod.Metadata.OwnerReferences { + kind := strings.TrimSpace(owner.Kind) + name := strings.TrimSpace(owner.Name) + if ns != "" && kind != "" && name != "" { + return ns + "/" + kind + "/" + name + } + } + return "" +} + +// podKey runs one orchestration or CLI step. +// Signature: podKey(pod podResource) string. +// Why: recovery incidents need stable namespace/name keys for pod maps and +// operator summaries. +func podKey(pod podResource) string { + ns := strings.TrimSpace(pod.Metadata.Namespace) + name := strings.TrimSpace(pod.Metadata.Name) + if ns == "" || name == "" { + return "" + } + return ns + "/" + name +} + +// replacementPodPendingForAttach runs one orchestration or CLI step. +// Signature: replacementPodPendingForAttach(pod podResource) bool. +// Why: stale RWO recovery should only act when a replacement is actually blocked +// during scheduling or initialization. +func replacementPodPendingForAttach(pod podResource) bool { + phase := strings.TrimSpace(pod.Status.Phase) + if strings.EqualFold(phase, "Pending") { + return true + } + for _, st := range append(append([]podContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) { + if st.State.Waiting == nil { + continue + } + reason := strings.TrimSpace(st.State.Waiting.Reason) + if strings.EqualFold(reason, "ContainerCreating") || strings.EqualFold(reason, "PodInitializing") { + return true + } + } + return false +} + +// podHasBlockedAttachEvent runs one orchestration or CLI step. +// Signature: podHasBlockedAttachEvent(pod podResource, events eventList) bool. +// Why: force deletion should be tied to concrete Multi-Attach or exclusive-use +// evidence rather than any old terminating PVC pod. +func podHasBlockedAttachEvent(pod podResource, events eventList) bool { + key := podKey(pod) + for _, event := range events.Items { + if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") { + continue + } + eventKey := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name) + if eventKey != key { + continue + } + if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") { + continue + } + message := strings.ToLower(strings.TrimSpace(event.Message)) + reason := strings.ToLower(strings.TrimSpace(event.Reason)) + if reason == "failedattachvolume" || + strings.Contains(message, "multi-attach") || + strings.Contains(message, "volume is already used by pod") || + strings.Contains(message, "already exclusively attached") { + return true + } + } + return false +} + +// podsShareAnyClaim runs one orchestration or CLI step. +// Signature: podsShareAnyClaim(oldPod podResource, replacement podResource, oldClaims map[string]string) bool. +// Why: the old stale pod and replacement must be competing for the same PVC +// before Ananke treats them as one storage handoff incident. +func podsShareAnyClaim(oldPod podResource, replacement podResource, oldClaims map[string]string) bool { + replacementClaims := map[string]struct{}{} + for _, volume := range replacement.Spec.Volumes { + if volume.PersistentVolumeClaim == nil { + continue + } + claim := strings.TrimSpace(volume.PersistentVolumeClaim.ClaimName) + if claim != "" { + replacementClaims[claim] = struct{}{} + } + } + for _, claim := range oldClaims { + if _, ok := replacementClaims[claim]; ok { + return true + } + } + return false +} + +// runningContainersMountAnyVolume runs one orchestration or CLI step. +// Signature: runningContainersMountAnyVolume(pod podResource, volumeNames map[string]string) bool. +// Why: a live app container mounting the blocked PVC is unsafe to clear, while +// sidecars without that mount can be handled as stale API ownership. +func runningContainersMountAnyVolume(pod podResource, volumeNames map[string]string) bool { + if len(volumeNames) == 0 { + return false + } + specByName := map[string]podContainer{} + for _, c := range pod.Spec.Containers { + specByName[strings.TrimSpace(c.Name)] = c + } + for _, status := range pod.Status.ContainerStatuses { + if status.State.Running == nil { + continue + } + spec, ok := specByName[strings.TrimSpace(status.Name)] + if !ok { + return true + } + for _, mount := range spec.VolumeMounts { + if _, ok := volumeNames[strings.TrimSpace(mount.Name)]; ok { + return true + } + } + } + return false +} diff --git a/internal/cluster/orchestrator_storage_types.go b/internal/cluster/orchestrator_storage_types.go index a08b78d..842be3c 100644 --- a/internal/cluster/orchestrator_storage_types.go +++ b/internal/cluster/orchestrator_storage_types.go @@ -244,6 +244,7 @@ type podResource struct { Metadata struct { Namespace string `json:"namespace"` Name string `json:"name"` + Labels map[string]string `json:"labels"` Annotations map[string]string `json:"annotations"` CreationTimestamp time.Time `json:"creationTimestamp"` DeletionTimestamp *time.Time `json:"deletionTimestamp"` @@ -284,9 +285,11 @@ type podContainerRunningState struct { } type podSpec struct { - NodeSelector map[string]string `json:"nodeSelector"` - Affinity *podAffinity `json:"affinity"` - Volumes []podVolume `json:"volumes"` + NodeSelector map[string]string `json:"nodeSelector"` + Affinity *podAffinity `json:"affinity"` + Volumes []podVolume `json:"volumes"` + Containers []podContainer `json:"containers"` + InitContainers []podContainer `json:"initContainers"` } type podVolume struct { @@ -294,6 +297,17 @@ type podVolume struct { PersistentVolumeClaim *podPersistentVolumeClaim `json:"persistentVolumeClaim"` } +type podContainer struct { + Name string `json:"name"` + Image string `json:"image"` + VolumeMounts []podVolumeMount `json:"volumeMounts"` +} + +type podVolumeMount struct { + Name string `json:"name"` + MountPath string `json:"mountPath"` +} + type podPersistentVolumeClaim struct { ClaimName string `json:"claimName"` } diff --git a/internal/cluster/orchestrator_tcp_service_check.go b/internal/cluster/orchestrator_tcp_service_check.go new file mode 100644 index 0000000..dd9062b --- /dev/null +++ b/internal/cluster/orchestrator_tcp_service_check.go @@ -0,0 +1,164 @@ +package cluster + +import ( + "bufio" + "context" + "crypto/tls" + "fmt" + "net" + "sort" + "strings" + "time" + + "scm.bstein.dev/bstein/ananke/internal/config" +) + +// tcpServiceChecklistReady runs one orchestration or CLI step. +// Signature: (o *Orchestrator) tcpServiceChecklistReady(ctx context.Context) (bool, string). +// Why: SMTP, IMAP, and other non-HTTP services need protocol checks alongside +// HTTP ingress checks before startup can call the cluster healthy. +func (o *Orchestrator) tcpServiceChecklistReady(ctx context.Context) (bool, string) { + checks := o.cfg.Startup.TCPServiceChecklist + if len(checks) == 0 { + return true, "no tcp checklist items configured" + } + for _, check := range checks { + ok, detail := o.tcpServiceCheckReady(ctx, check) + if !ok { + name := strings.TrimSpace(check.Name) + if name == "" { + name = fmt.Sprintf("%s:%d", strings.TrimSpace(check.Host), check.Port) + } + return false, fmt.Sprintf("%s: %s", name, detail) + } + } + return true, fmt.Sprintf("tcp-checks=%d", len(checks)) +} + +// tcpServiceCheckReady runs one orchestration or CLI step. +// Signature: (o *Orchestrator) tcpServiceCheckReady(ctx context.Context, check config.TCPServiceChecklistCheck) (bool, string). +// Why: a single mail protocol endpoint should report whether connect, TLS, and +// banner expectations passed without exposing any configured secret material. +func (o *Orchestrator) tcpServiceCheckReady(ctx context.Context, check config.TCPServiceChecklistCheck) (bool, string) { + banner, err := tcpServiceProbe(ctx, check) + if err != nil { + return false, err.Error() + } + expected := strings.TrimSpace(check.ExpectContains) + if expected != "" && !checklistContains(banner, expected) { + return false, fmt.Sprintf("banner missing expected marker %q", expected) + } + return true, "connected" +} + +// tcpServiceProbe runs one orchestration or CLI step. +// Signature: tcpServiceProbe(ctx context.Context, check config.TCPServiceChecklistCheck) (string, error). +// Why: protocol probes need a small direct TCP/TLS implementation so Ananke can +// validate Mailu without shelling out to nc or openssl. +func tcpServiceProbe(ctx context.Context, check config.TCPServiceChecklistCheck) (string, error) { + timeout := time.Duration(check.TimeoutSeconds) * time.Second + if timeout <= 0 { + timeout = 8 * time.Second + } + address := net.JoinHostPort(strings.TrimSpace(check.Host), fmt.Sprintf("%d", check.Port)) + dialer := net.Dialer{Timeout: timeout} + conn, err := dialer.DialContext(ctx, "tcp", address) + if err != nil { + return "", fmt.Errorf("connect %s failed: %w", address, err) + } + defer conn.Close() + deadline := time.Now().Add(timeout) + _ = conn.SetDeadline(deadline) + + if check.TLS { + tlsConn := tls.Client(conn, &tls.Config{ + ServerName: strings.TrimSpace(check.Host), + InsecureSkipVerify: check.InsecureSkipTLS, + }) + if err := tlsConn.HandshakeContext(ctx); err != nil { + return "", fmt.Errorf("tls handshake %s failed: %w", address, err) + } + conn = tlsConn + _ = conn.SetDeadline(deadline) + } + + reader := bufio.NewReader(conn) + bannerParts := []string{} + line, _ := reader.ReadString('\n') + if strings.TrimSpace(line) != "" { + bannerParts = append(bannerParts, strings.TrimSpace(line)) + } + if send := check.Send; send != "" { + if _, err := conn.Write([]byte(send)); err != nil { + return strings.Join(bannerParts, "\n"), fmt.Errorf("write probe command failed: %w", err) + } + line, _ = reader.ReadString('\n') + if strings.TrimSpace(line) != "" { + bannerParts = append(bannerParts, strings.TrimSpace(line)) + } + } + return strings.Join(bannerParts, "\n"), nil +} + +// maybeAutoHealTCPServiceBackends runs one orchestration or CLI step. +// Signature: (o *Orchestrator) maybeAutoHealTCPServiceBackends(ctx context.Context, lastAttempt *time.Time). +// Why: failed Mailu protocol probes should drive the same backend repair loop as +// HTTP services when a Kubernetes service hint is configured. +func (o *Orchestrator) maybeAutoHealTCPServiceBackends(ctx context.Context, lastAttempt *time.Time) { + if o.runner.DryRun || len(o.cfg.Startup.TCPServiceChecklist) == 0 { + return + } + now := time.Now() + if lastAttempt != nil && !lastAttempt.IsZero() && now.Sub(*lastAttempt) < 45*time.Second { + return + } + if lastAttempt != nil { + *lastAttempt = now + } + healed, err := o.healFailedTCPServiceBackends(ctx) + if err != nil { + o.log.Printf("warning: tcp service backend auto-heal failed: %v", err) + return + } + if len(healed) == 0 { + return + } + sort.Strings(healed) + detail := fmt.Sprintf("restored tcp service backends: %s", joinLimited(healed, 8)) + o.log.Printf("%s", detail) + o.noteStartupAutoHeal(detail) +} + +// healFailedTCPServiceBackends runs one orchestration or CLI step. +// Signature: (o *Orchestrator) healFailedTCPServiceBackends(ctx context.Context) ([]string, error). +// Why: failed TCP protocol checks need a direct backend-heal primitive that can +// be reused by startup waits and post-start daemon repair. +func (o *Orchestrator) healFailedTCPServiceBackends(ctx context.Context) ([]string, error) { + if o.runner.DryRun || len(o.cfg.Startup.TCPServiceChecklist) == 0 { + return nil, nil + } + healed := []string{} + attempted := map[string]struct{}{} + for _, check := range o.cfg.Startup.TCPServiceChecklist { + namespace := strings.TrimSpace(check.Namespace) + service := strings.TrimSpace(check.Service) + if namespace == "" || service == "" { + continue + } + key := namespace + "/" + service + if _, ok := attempted[key]; ok { + continue + } + ok, _ := o.tcpServiceCheckReady(ctx, check) + if ok { + continue + } + attempted[key] = struct{}{} + items, err := o.maybeHealCriticalEndpointBackends(ctx, namespace, service) + if err != nil { + return healed, fmt.Errorf("%s/%s: %w", namespace, service, err) + } + healed = append(healed, items...) + } + return healed, nil +} diff --git a/internal/cluster/orchestrator_vault_health.go b/internal/cluster/orchestrator_vault_health.go new file mode 100644 index 0000000..1130461 --- /dev/null +++ b/internal/cluster/orchestrator_vault_health.go @@ -0,0 +1,74 @@ +package cluster + +import ( + "context" + "fmt" + "strings" + "time" +) + +// vaultSealed runs one orchestration or CLI step. +// Signature: (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error). +// Why: treats exec and HTTP health as independent Vault signals so one flaky +// kubectl exec does not block startup when HTTP health proves Vault is usable. +func (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error) { + sealed, err := o.vaultSealedViaExec(ctx) + if err == nil { + return sealed, nil + } + httpSealed, httpErr := o.vaultSealedViaHTTP(ctx) + if httpErr == nil { + o.log.Printf("warning: vault exec status probe failed but HTTP health succeeded; continuing with HTTP health result: %v", err) + return httpSealed, nil + } + return false, fmt.Errorf("vault status check failed: exec=%v http=%v", err, httpErr) +} + +// vaultSealedViaExec runs one orchestration or CLI step. +// Signature: (o *Orchestrator) vaultSealedViaExec(ctx context.Context) (bool, error). +// Why: preserves the existing direct Vault CLI probe as the primary status +// signal when kubectl exec is healthy. +func (o *Orchestrator) vaultSealedViaExec(ctx context.Context) (bool, error) { + out, err := o.kubectl( + ctx, + 25*time.Second, + "-n", "vault", + "exec", "vault-0", "--", + "sh", "-lc", + "VAULT_ADDR=http://127.0.0.1:8200 vault status -format=json 2>/dev/null || true", + ) + if err != nil { + return false, fmt.Errorf("vault status check failed: %w", err) + } + sealed, err := parseVaultSealed(out) + if err != nil { + return false, fmt.Errorf("parse vault status: %w", err) + } + return sealed, nil +} + +// vaultSealedViaHTTP runs one orchestration or CLI step. +// Signature: (o *Orchestrator) vaultSealedViaHTTP(ctx context.Context) (bool, error). +// Why: gives startup a second authoritative Vault health path when exec is +// transiently killed or the apiserver exec tunnel is degraded. +func (o *Orchestrator) vaultSealedViaHTTP(ctx context.Context) (bool, error) { + out, err := o.kubectl( + ctx, + 15*time.Second, + "-n", "vault", + "get", + "--raw", + "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health", + ) + if err != nil && strings.TrimSpace(out) == "" { + return false, fmt.Errorf("vault HTTP health check failed: %w", err) + } + sealed, parseErr := parseVaultSealed(out) + if parseErr != nil { + if err != nil { + return false, fmt.Errorf("vault HTTP health check failed: %w; parse: %v", err, parseErr) + } + return false, fmt.Errorf("parse vault HTTP health: %w", parseErr) + } + return sealed, nil +} diff --git a/internal/cluster/orchestrator_workload_convergence.go b/internal/cluster/orchestrator_workload_convergence.go index 8281a77..f450226 100644 --- a/internal/cluster/orchestrator_workload_convergence.go +++ b/internal/cluster/orchestrator_workload_convergence.go @@ -188,6 +188,18 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error { } else { stalePhaseReasons = reasons } + staleRWODecisions := map[string]staleRWOOwnerDecision{} + if decisions, scanErr := o.staleRWOPVCOwnerDecisions(ctx, list, grace); scanErr != nil { + o.log.Printf("warning: stale RWO PVC owner scan failed: %v", scanErr) + } else { + staleRWODecisions = decisions + } + imagePullDNSReasons := map[string]string{} + if reasons, scanErr := o.imagePullDNSBlockerReasons(ctx); scanErr != nil { + o.log.Printf("warning: image-pull DNS blocker scan failed: %v", scanErr) + } else { + imagePullDNSReasons = reasons + } containerRuntimeWedgeReasons := map[string]string{} if reasons, scanErr := o.containerRuntimeWedgePodReasons(ctx, list, grace); scanErr != nil { o.log.Printf("warning: container runtime wedge scan failed: %v", scanErr) @@ -219,6 +231,10 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error { continue } reason := stuckContainerReason(pod, stuckReasons) + if (reason == "ImagePullBackOff" || reason == "ErrImagePull") && imagePullDNSReasons[ns+"/"+name] != "" { + o.log.Printf("warning: not recycling pod %s/%s because image pull is blocked by DNS/registry lookup: %s", ns, name, imagePullDNSReasons[ns+"/"+name]) + continue + } if reason == "" { reason = stuckVaultInitReason(pod, grace) } @@ -231,6 +247,15 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error { if reason == "" { reason = stalePhaseReasons[ns+"/"+name] } + if decision, ok := staleRWODecisions[ns+"/"+name]; ok && decision.Unsafe { + o.log.Printf("warning: stale RWO PVC owner is unsafe to force delete pod=%s/%s reason=%s", ns, name, decision.Reason) + continue + } + if reason == "" { + if decision, ok := staleRWODecisions[ns+"/"+name]; ok && decision.ForceDelete { + reason = decision.Reason + } + } if runtimeReason := containerRuntimeWedgeReasons[ns+"/"+name]; runtimeReason != "" { reason = runtimeReason } @@ -242,6 +267,9 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error { } deleteArgs := []string{"-n", ns, "delete", "pod", name, "--wait=false"} forceDelete := staleControllerPodForceDeleteSafe(pod, grace) + if decision, ok := staleRWODecisions[ns+"/"+name]; ok && decision.ForceDelete { + forceDelete = true + } if forceDelete { deleteArgs = append(deleteArgs, "--grace-period=0", "--force") } diff --git a/internal/config/apply_defaults.go b/internal/config/apply_defaults.go index 267b208..55c5b9d 100644 --- a/internal/config/apply_defaults.go +++ b/internal/config/apply_defaults.go @@ -129,11 +129,21 @@ func (c *Config) applyDefaults() { } else { c.Startup.ServiceChecklist = mergeServiceChecklistDefaults(c.Startup.ServiceChecklist, defaultServiceChecklist()) } + if c.Startup.TCPServiceChecklistExplicitOnly { + c.Startup.TCPServiceChecklist = mergeTCPServiceChecklistDefaults(c.Startup.TCPServiceChecklist, []TCPServiceChecklistCheck{}) + } else { + c.Startup.TCPServiceChecklist = mergeTCPServiceChecklistDefaults(c.Startup.TCPServiceChecklist, defaultTCPServiceChecklist()) + } for i := range c.Startup.ServiceChecklist { if c.Startup.ServiceChecklist[i].TimeoutSeconds <= 0 { c.Startup.ServiceChecklist[i].TimeoutSeconds = 12 } } + for i := range c.Startup.TCPServiceChecklist { + if c.Startup.TCPServiceChecklist[i].TimeoutSeconds <= 0 { + c.Startup.TCPServiceChecklist[i].TimeoutSeconds = 8 + } + } if c.Startup.CriticalServiceEndpointWaitSec <= 0 { c.Startup.CriticalServiceEndpointWaitSec = 420 } @@ -141,6 +151,9 @@ func (c *Config) applyDefaults() { c.Startup.CriticalServiceEndpointPollSec = 5 } c.Startup.CriticalServiceEndpoints = mergeStringDefaults(c.Startup.CriticalServiceEndpoints, defaultCriticalServiceEndpoints()) + if c.Startup.CriticalServiceStartupProbeThreshold <= 0 { + c.Startup.CriticalServiceStartupProbeThreshold = 30 + } if c.Startup.IngressChecklistWaitSeconds <= 0 { c.Startup.IngressChecklistWaitSeconds = 420 } @@ -207,6 +220,18 @@ func (c *Config) applyDefaults() { if c.Startup.DeadNodeCleanupGraceSeconds <= 0 { c.Startup.DeadNodeCleanupGraceSeconds = 300 } + if strings.TrimSpace(c.Startup.HostSudoSecretPasswordKey) == "" { + c.Startup.HostSudoSecretPasswordKey = "password" + } + if c.Startup.HostPrivilegedCommandTimeoutSec <= 0 { + c.Startup.HostPrivilegedCommandTimeoutSec = 90 + } + if c.Startup.NodeRuntimeRestartWaitSeconds <= 0 { + c.Startup.NodeRuntimeRestartWaitSeconds = 140 + } + if c.Startup.NodeRuntimeRebootWaitSeconds <= 0 { + c.Startup.NodeRuntimeRebootWaitSeconds = 420 + } if strings.TrimSpace(c.Startup.VaultUnsealKeyFile) == "" { c.Startup.VaultUnsealKeyFile = "/var/lib/ananke/vault-unseal.key" } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 48935ec..dee5c3b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -132,6 +132,71 @@ state: } } +// TestLoadDefaultsCriticalStartupProbeRepairWhenOmitted runs one orchestration or CLI step. +// Signature: TestLoadDefaultsCriticalStartupProbeRepairWhenOmitted(t *testing.T). +// Why: existing configs with a startup block should inherit the Mailu hardening +// repair instead of silently decoding the new boolean as false. +func TestLoadDefaultsCriticalStartupProbeRepairWhenOmitted(t *testing.T) { + tmp := t.TempDir() + cfgPath := filepath.Join(tmp, "ananke.yaml") + raw := ` +control_planes: [titan-0a, titan-0b, titan-0c] +expected_flux_branch: main +iac_repo_path: /opt/titan-iac +startup: + api_wait_seconds: 900 +ups: + enabled: false +state: + run_history_path: /tmp/runs.json + lock_path: /tmp/ananke.lock +` + if err := os.WriteFile(cfgPath, []byte(strings.TrimSpace(raw)), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("load config: %v", err) + } + if !cfg.Startup.CriticalServiceStartupProbeRepair { + t.Fatalf("expected omitted startup-probe repair setting to default true") + } + if cfg.Startup.CriticalServiceStartupProbeThreshold != 30 { + t.Fatalf("expected startup-probe threshold default 30, got %d", cfg.Startup.CriticalServiceStartupProbeThreshold) + } +} + +// TestLoadHonorsExplicitCriticalStartupProbeRepairFalse runs one orchestration or CLI step. +// Signature: TestLoadHonorsExplicitCriticalStartupProbeRepairFalse(t *testing.T). +// Why: operators should still be able to disable the repair explicitly even +// though omitted values inherit the hardening default. +func TestLoadHonorsExplicitCriticalStartupProbeRepairFalse(t *testing.T) { + tmp := t.TempDir() + cfgPath := filepath.Join(tmp, "ananke.yaml") + raw := ` +control_planes: [titan-0a, titan-0b, titan-0c] +expected_flux_branch: main +iac_repo_path: /opt/titan-iac +startup: + critical_service_startup_probe_repair: false +ups: + enabled: false +state: + run_history_path: /tmp/runs.json + lock_path: /tmp/ananke.lock +` + if err := os.WriteFile(cfgPath, []byte(strings.TrimSpace(raw)), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + cfg, err := Load(cfgPath) + if err != nil { + t.Fatalf("load config: %v", err) + } + if cfg.Startup.CriticalServiceStartupProbeRepair { + t.Fatalf("expected explicit startup-probe repair false to be honored") + } +} + // TestValidateRejectsInvalidStartupShutdownCooldown runs one orchestration or CLI step. // Signature: TestValidateRejectsInvalidStartupShutdownCooldown(t *testing.T). // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve. diff --git a/internal/config/defaults.go b/internal/config/defaults.go index c8547ab..a4e75ca 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -94,10 +94,13 @@ func defaults() Config { AdminSecretPasswordKey: "password", }, ServiceChecklist: defaultServiceChecklist(), + TCPServiceChecklist: defaultTCPServiceChecklist(), RequireCriticalServiceEndpoints: true, CriticalServiceEndpointWaitSec: 420, CriticalServiceEndpointPollSec: 5, CriticalServiceEndpoints: defaultCriticalServiceEndpoints(), + CriticalServiceStartupProbeRepair: true, + CriticalServiceStartupProbeThreshold: 30, RequireIngressChecklist: true, IngressChecklistWaitSeconds: 420, IngressChecklistPollSeconds: 5, @@ -122,6 +125,10 @@ func defaults() Config { AutoRecycleStuckPods: true, StuckPodGraceSeconds: 180, RecoveryCordonMaxSeconds: 3600, + HostSudoSecretPasswordKey: "password", + HostPrivilegedCommandTimeoutSec: 90, + NodeRuntimeRestartWaitSeconds: 140, + NodeRuntimeRebootWaitSeconds: 420, VaultUnsealKeyFile: "/var/lib/ananke/vault-unseal.key", VaultUnsealBreakglassTimeout: 15, }, diff --git a/internal/config/load.go b/internal/config/load.go index d9f0407..a960a85 100644 --- a/internal/config/load.go +++ b/internal/config/load.go @@ -17,13 +17,52 @@ func Load(path string) (Config, error) { if err != nil { return Config{}, fmt.Errorf("read config %s: %w", path, err) } + startupProbeRepairConfigured, err := yamlPathExists(b, "startup", "critical_service_startup_probe_repair") + if err != nil { + return Config{}, fmt.Errorf("decode config %s: %w", path, err) + } if err := yaml.Unmarshal(b, &cfg); err != nil { return Config{}, fmt.Errorf("decode config %s: %w", path, err) } cfg.applyDefaults() + if !startupProbeRepairConfigured { + cfg.Startup.CriticalServiceStartupProbeRepair = defaults().Startup.CriticalServiceStartupProbeRepair + } if err := cfg.Validate(); err != nil { return Config{}, err } return cfg, nil } + +// yamlPathExists runs one orchestration or CLI step. +// Signature: yamlPathExists(raw []byte, path ...string) (bool, error). +// Why: selected boolean defaults need to distinguish an omitted YAML key from an +// explicit false value after nested startup mappings are decoded. +func yamlPathExists(raw []byte, path ...string) (bool, error) { + var root yaml.Node + if err := yaml.Unmarshal(raw, &root); err != nil { + return false, err + } + node := &root + if node.Kind == yaml.DocumentNode && len(node.Content) > 0 { + node = node.Content[0] + } + for _, key := range path { + if node.Kind != yaml.MappingNode { + return false, nil + } + found := false + for i := 0; i+1 < len(node.Content); i += 2 { + if node.Content[i].Value == key { + node = node.Content[i+1] + found = true + break + } + } + if !found { + return false, nil + } + } + return true, nil +} diff --git a/internal/config/startup_service_catalog.go b/internal/config/startup_service_catalog.go index fb5715a..25d0606 100644 --- a/internal/config/startup_service_catalog.go +++ b/internal/config/startup_service_catalog.go @@ -251,6 +251,57 @@ func defaultServiceChecklist() []ServiceChecklistCheck { } } +// defaultTCPServiceChecklist runs one orchestration or CLI step. +// Signature: defaultTCPServiceChecklist() []TCPServiceChecklistCheck. +// Why: mail and other non-HTTP edge services need protocol-level startup checks +// so Ananke does not declare success while SMTP/IMAP ports are broken. +func defaultTCPServiceChecklist() []TCPServiceChecklistCheck { + return []TCPServiceChecklistCheck{ + { + Name: "mail-smtp", + Host: "mail.bstein.dev", + Port: 25, + Send: "QUIT\r\n", + ExpectContains: "ESMTP", + TimeoutSeconds: 8, + Namespace: "mailu-mailserver", + Service: "mailu-front", + }, + { + Name: "mail-smtps", + Host: "mail.bstein.dev", + Port: 465, + TLS: true, + Send: "QUIT\r\n", + ExpectContains: "ready", + TimeoutSeconds: 8, + Namespace: "mailu-mailserver", + Service: "mailu-front", + }, + { + Name: "mail-submission", + Host: "mail.bstein.dev", + Port: 587, + Send: "QUIT\r\n", + ExpectContains: "ready", + TimeoutSeconds: 8, + Namespace: "mailu-mailserver", + Service: "mailu-front", + }, + { + Name: "mail-imaps", + Host: "mail.bstein.dev", + Port: 993, + TLS: true, + Send: "a001 LOGOUT\r\n", + ExpectContains: "OK", + TimeoutSeconds: 8, + Namespace: "mailu-mailserver", + Service: "mailu-front", + }, + } +} + // defaultCriticalServiceEndpoints runs one orchestration or CLI step. // Signature: defaultCriticalServiceEndpoints() []string. // Why: service edge checks are insufficient for protected stacks; endpoint @@ -263,6 +314,11 @@ func defaultCriticalServiceEndpoints() []string { "logging/oauth2-proxy-logs", "logging/opensearch-dashboards", "logging/opensearch-master", + "mailu-mailserver/mailu-admin", + "mailu-mailserver/mailu-dovecot", + "mailu-mailserver/mailu-front", + "mailu-mailserver/mailu-postfix", + "mailu-mailserver/mailu-rspamd", } } @@ -301,6 +357,41 @@ func mergeServiceChecklistDefaults(existing, defaults []ServiceChecklistCheck) [ return out } +// mergeTCPServiceChecklistDefaults runs one orchestration or CLI step. +// Signature: mergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck. +// Why: host configs should inherit new mail/TCP health checks while preserving +// site-specific protocol probes. +func mergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck { + if len(existing) == 0 { + out := make([]TCPServiceChecklistCheck, 0, len(defaults)) + out = append(out, defaults...) + return out + } + + defaultByName := map[string]struct{}{} + for _, check := range defaults { + name := strings.TrimSpace(check.Name) + if name == "" { + continue + } + defaultByName[name] = struct{}{} + } + + out := make([]TCPServiceChecklistCheck, 0, len(defaults)+len(existing)) + out = append(out, defaults...) + for _, check := range existing { + name := strings.TrimSpace(check.Name) + if name == "" { + continue + } + if _, exists := defaultByName[name]; exists { + continue + } + out = append(out, check) + } + return out +} + // mergeStringDefaults runs one orchestration or CLI step. // Signature: mergeStringDefaults(existing, defaults []string) []string. // Why: keeps baseline startup guards applied while preserving site-specific diff --git a/internal/config/testing_hooks.go b/internal/config/testing_hooks.go index 36d469f..29b34c6 100644 --- a/internal/config/testing_hooks.go +++ b/internal/config/testing_hooks.go @@ -18,6 +18,15 @@ func TestHookDefaultCriticalServiceEndpoints() []string { return out } +// TestHookDefaultTCPServiceChecklist runs one orchestration or CLI step. +// Signature: TestHookDefaultTCPServiceChecklist() []TCPServiceChecklistCheck. +// Why: exposes default TCP/mail service checks to top-level tests. +func TestHookDefaultTCPServiceChecklist() []TCPServiceChecklistCheck { + out := make([]TCPServiceChecklistCheck, 0, len(defaultTCPServiceChecklist())) + out = append(out, defaultTCPServiceChecklist()...) + return out +} + // TestHookMergeServiceChecklistDefaults runs one orchestration or CLI step. // Signature: TestHookMergeServiceChecklistDefaults(existing, defaults []ServiceChecklistCheck) []ServiceChecklistCheck. // Why: exposes checklist merge helper to top-level tests. @@ -31,3 +40,10 @@ func TestHookMergeServiceChecklistDefaults(existing, defaults []ServiceChecklist func TestHookMergeStringDefaults(existing, defaults []string) []string { return mergeStringDefaults(existing, defaults) } + +// TestHookMergeTCPServiceChecklistDefaults runs one orchestration or CLI step. +// Signature: TestHookMergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck. +// Why: exposes TCP checklist merge helper to top-level tests. +func TestHookMergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck { + return mergeTCPServiceChecklistDefaults(existing, defaults) +} diff --git a/internal/config/types.go b/internal/config/types.go index b5478f3..bd2b502 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -62,10 +62,14 @@ type Startup struct { ServiceChecklistAuth ServiceChecklistAuthSettings `yaml:"service_checklist_auth"` ServiceChecklistExplicitOnly bool `yaml:"service_checklist_explicit_only"` ServiceChecklist []ServiceChecklistCheck `yaml:"service_checklist"` + TCPServiceChecklistExplicitOnly bool `yaml:"tcp_service_checklist_explicit_only"` + TCPServiceChecklist []TCPServiceChecklistCheck `yaml:"tcp_service_checklist"` RequireCriticalServiceEndpoints bool `yaml:"require_critical_service_endpoints"` CriticalServiceEndpointWaitSec int `yaml:"critical_service_endpoint_wait_seconds"` CriticalServiceEndpointPollSec int `yaml:"critical_service_endpoint_poll_seconds"` CriticalServiceEndpoints []string `yaml:"critical_service_endpoints"` + CriticalServiceStartupProbeRepair bool `yaml:"critical_service_startup_probe_repair"` + CriticalServiceStartupProbeThreshold int `yaml:"critical_service_startup_probe_failure_threshold"` RequireIngressChecklist bool `yaml:"require_ingress_checklist"` IngressChecklistWaitSeconds int `yaml:"ingress_checklist_wait_seconds"` IngressChecklistPollSeconds int `yaml:"ingress_checklist_poll_seconds"` @@ -96,6 +100,13 @@ type Startup struct { PostStartAutoHealSeconds int `yaml:"post_start_auto_heal_seconds"` RecoveryCordonMaxSeconds int `yaml:"recovery_cordon_max_seconds"` DeadNodeCleanupGraceSeconds int `yaml:"dead_node_cleanup_grace_seconds"` + HostSudoSecretNamespace string `yaml:"host_sudo_secret_namespace"` + HostSudoSecretNameTemplate string `yaml:"host_sudo_secret_name_template"` + HostSudoSecretPasswordKey string `yaml:"host_sudo_secret_password_key"` + HostRepairAllowReboot bool `yaml:"host_repair_allow_reboot"` + HostPrivilegedCommandTimeoutSec int `yaml:"host_privileged_command_timeout_seconds"` + NodeRuntimeRestartWaitSeconds int `yaml:"node_runtime_restart_wait_seconds"` + NodeRuntimeRebootWaitSeconds int `yaml:"node_runtime_reboot_wait_seconds"` VaultUnsealKeyFile string `yaml:"vault_unseal_key_file"` VaultUnsealBreakglassCommand string `yaml:"vault_unseal_breakglass_command"` VaultUnsealBreakglassTimeout int `yaml:"vault_unseal_breakglass_timeout_seconds"` @@ -117,6 +128,19 @@ type ServiceChecklistCheck struct { InsecureSkipTLS bool `yaml:"insecure_skip_tls"` } +type TCPServiceChecklistCheck struct { + Name string `yaml:"name"` + Host string `yaml:"host"` + Port int `yaml:"port"` + TLS bool `yaml:"tls"` + Send string `yaml:"send"` + ExpectContains string `yaml:"expect_contains"` + TimeoutSeconds int `yaml:"timeout_seconds"` + InsecureSkipTLS bool `yaml:"insecure_skip_tls"` + Namespace string `yaml:"namespace"` + Service string `yaml:"service"` +} + type ServiceChecklistAuthSettings struct { Mode string `yaml:"mode"` KeycloakBaseURL string `yaml:"keycloak_base_url"` diff --git a/internal/config/validate.go b/internal/config/validate.go index 7f6b819..4711c70 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -203,6 +203,28 @@ func (c Config) Validate() error { } } } + for i, check := range c.Startup.TCPServiceChecklist { + if strings.TrimSpace(check.Name) == "" { + return fmt.Errorf("config.startup.tcp_service_checklist[%d].name must not be empty", i) + } + if strings.TrimSpace(check.Host) == "" { + return fmt.Errorf("config.startup.tcp_service_checklist[%d].host must not be empty", i) + } + if strings.ContainsAny(check.Host, " \t\r\n/") { + return fmt.Errorf("config.startup.tcp_service_checklist[%d].host is invalid: %q", i, check.Host) + } + if check.Port <= 0 || check.Port > 65535 { + return fmt.Errorf("config.startup.tcp_service_checklist[%d].port must be in range 1-65535", i) + } + if check.TimeoutSeconds <= 0 { + return fmt.Errorf("config.startup.tcp_service_checklist[%d].timeout_seconds must be > 0", i) + } + if strings.TrimSpace(check.Namespace) != "" || strings.TrimSpace(check.Service) != "" { + if strings.TrimSpace(check.Namespace) == "" || strings.TrimSpace(check.Service) == "" { + return fmt.Errorf("config.startup.tcp_service_checklist[%d].namespace and service must be set together", i) + } + } + } if c.Startup.CriticalServiceEndpointWaitSec <= 0 { return fmt.Errorf("config.startup.critical_service_endpoint_wait_seconds must be > 0") } @@ -221,6 +243,9 @@ func (c Config) Validate() error { return fmt.Errorf("config.startup.critical_service_endpoints entries must be namespace/service, got %q", entry) } } + if c.Startup.CriticalServiceStartupProbeRepair && c.Startup.CriticalServiceStartupProbeThreshold <= 0 { + return fmt.Errorf("config.startup.critical_service_startup_probe_failure_threshold must be > 0 when critical_service_startup_probe_repair is true") + } if c.Startup.IngressChecklistWaitSeconds <= 0 { return fmt.Errorf("config.startup.ingress_checklist_wait_seconds must be > 0") } @@ -286,6 +311,26 @@ func (c Config) Validate() error { if c.Startup.DeadNodeCleanupGraceSeconds <= 0 { return fmt.Errorf("config.startup.dead_node_cleanup_grace_seconds must be > 0") } + if strings.TrimSpace(c.Startup.HostSudoSecretNamespace) != "" || strings.TrimSpace(c.Startup.HostSudoSecretNameTemplate) != "" { + if strings.TrimSpace(c.Startup.HostSudoSecretNamespace) == "" { + return fmt.Errorf("config.startup.host_sudo_secret_namespace must not be empty when host sudo secret lookup is configured") + } + if strings.TrimSpace(c.Startup.HostSudoSecretNameTemplate) == "" { + return fmt.Errorf("config.startup.host_sudo_secret_name_template must not be empty when host sudo secret lookup is configured") + } + if strings.TrimSpace(c.Startup.HostSudoSecretPasswordKey) == "" { + return fmt.Errorf("config.startup.host_sudo_secret_password_key must not be empty when host sudo secret lookup is configured") + } + } + if c.Startup.HostPrivilegedCommandTimeoutSec <= 0 { + return fmt.Errorf("config.startup.host_privileged_command_timeout_seconds must be > 0") + } + if c.Startup.NodeRuntimeRestartWaitSeconds <= 0 { + return fmt.Errorf("config.startup.node_runtime_restart_wait_seconds must be > 0") + } + if c.Startup.NodeRuntimeRebootWaitSeconds <= 0 { + return fmt.Errorf("config.startup.node_runtime_reboot_wait_seconds must be > 0") + } for _, probe := range c.Startup.PostStartProbes { if strings.TrimSpace(probe) == "" { return fmt.Errorf("config.startup.post_start_probes entries must not be empty") diff --git a/internal/config/validate_matrix_test.go b/internal/config/validate_matrix_test.go index 58b95cb..beb5a64 100644 --- a/internal/config/validate_matrix_test.go +++ b/internal/config/validate_matrix_test.go @@ -52,6 +52,21 @@ func TestValidateRejectsInvalidFieldsMatrix(t *testing.T) { {"bad_service_checklist_http_code", func(c *Config) { c.Startup.ServiceChecklist = []ServiceChecklistCheck{{Name: "x", URL: "https://ok", TimeoutSeconds: 5, AcceptedStatuses: []int{999}}} }}, + {"bad_tcp_service_checklist_name", func(c *Config) { + c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Host: "mail.bstein.dev", Port: 25, TimeoutSeconds: 5}} + }}, + {"bad_tcp_service_checklist_host", func(c *Config) { + c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail/bad", Port: 25, TimeoutSeconds: 5}} + }}, + {"bad_tcp_service_checklist_port", func(c *Config) { + c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail.bstein.dev", Port: 70000, TimeoutSeconds: 5}} + }}, + {"bad_tcp_service_checklist_timeout", func(c *Config) { + c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail.bstein.dev", Port: 25}} + }}, + {"bad_tcp_service_checklist_partial_backend", func(c *Config) { + c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail.bstein.dev", Port: 25, TimeoutSeconds: 5, Namespace: "mailu-mailserver"}} + }}, {"bad_critical_endpoint_wait", func(c *Config) { c.Startup.CriticalServiceEndpointWaitSec = 0 }}, {"bad_critical_endpoint_poll", func(c *Config) { c.Startup.CriticalServiceEndpointPollSec = 0 }}, {"bad_empty_critical_endpoints_when_required", func(c *Config) { @@ -159,6 +174,9 @@ func TestApplyDefaultsPopulatesZeroConfig(t *testing.T) { if len(cfg.Startup.CriticalServiceEndpoints) == 0 { t.Fatalf("expected critical service endpoint defaults to be set") } + if len(cfg.Startup.TCPServiceChecklist) == 0 { + t.Fatalf("expected TCP service checklist defaults to be set") + } if cfg.Shutdown.SSHParallelism <= 0 || cfg.Shutdown.ScaleParallelism <= 0 || cfg.Shutdown.DrainParallelism <= 0 { t.Fatalf("expected shutdown parallelism defaults to be set") } diff --git a/testing/config/config_quality_matrix_test.go b/testing/config/config_quality_matrix_test.go index 77026bf..228a8fe 100644 --- a/testing/config/config_quality_matrix_test.go +++ b/testing/config/config_quality_matrix_test.go @@ -51,15 +51,33 @@ func TestHookServiceCatalogAndMergeContracts(t *testing.T) { t.Fatalf("expected critical endpoint defaults") } foundMonitoring := false + foundMailAdmin := false for _, entry := range critical { if entry == "monitoring/grafana" { foundMonitoring = true - break + } + if entry == "mailu-mailserver/mailu-admin" { + foundMailAdmin = true } } if !foundMonitoring { t.Fatalf("expected monitoring/grafana critical endpoint default") } + if !foundMailAdmin { + t.Fatalf("expected mailu-mailserver/mailu-admin critical endpoint default") + } + + tcpChecks := icfg.TestHookDefaultTCPServiceChecklist() + seenTCP := map[string]icfg.TCPServiceChecklistCheck{} + for _, check := range tcpChecks { + seenTCP[strings.TrimSpace(check.Name)] = check + } + if seenTCP["mail-smtp"].Port != 25 || seenTCP["mail-smtp"].Service != "mailu-front" { + t.Fatalf("expected mail-smtp TCP default to target Mailu front") + } + if !seenTCP["mail-imaps"].TLS || seenTCP["mail-imaps"].Port != 993 { + t.Fatalf("expected mail-imaps TCP default with TLS on 993") + } mergedChecks := icfg.TestHookMergeServiceChecklistDefaults( []icfg.ServiceChecklistCheck{ @@ -75,6 +93,20 @@ func TestHookServiceCatalogAndMergeContracts(t *testing.T) { t.Fatalf("expected 3 merged checks with dedupe, got %d", len(mergedChecks)) } + mergedTCP := icfg.TestHookMergeTCPServiceChecklistDefaults( + []icfg.TCPServiceChecklistCheck{ + {Name: "custom-tcp", Host: "custom.bstein.dev", Port: 1234, TimeoutSeconds: 5}, + {Name: "mail-smtp", Host: "override.invalid", Port: 25, TimeoutSeconds: 5}, + }, + []icfg.TCPServiceChecklistCheck{ + {Name: "mail-smtp", Host: "mail.bstein.dev", Port: 25, TimeoutSeconds: 5}, + {Name: "mail-imaps", Host: "mail.bstein.dev", Port: 993, TimeoutSeconds: 5}, + }, + ) + if len(mergedTCP) != 3 { + t.Fatalf("expected 3 merged TCP checks with dedupe, got %d", len(mergedTCP)) + } + mergedStrings := icfg.TestHookMergeStringDefaults( []string{" one ", "one", "", "two"}, []string{"two", "three", " "}, @@ -84,6 +116,33 @@ func TestHookServiceCatalogAndMergeContracts(t *testing.T) { } } +// TestTitanCoordinatorConfigInheritsMailHardeningDefaults runs one orchestration or CLI step. +// Signature: TestTitanCoordinatorConfigInheritsMailHardeningDefaults(t *testing.T). +// Why: the deployed coordinator config has a startup block, so new Mailu recovery +// defaults must still be present after YAML decoding and default merging. +func TestTitanCoordinatorConfigInheritsMailHardeningDefaults(t *testing.T) { + cfg, err := icfg.Load(filepath.Join("..", "..", "configs", "ananke.titan-db.yaml")) + if err != nil { + t.Fatalf("load titan coordinator config: %v", err) + } + if !cfg.Startup.CriticalServiceStartupProbeRepair { + t.Fatalf("expected titan config to inherit critical startup-probe repair") + } + seenMailAdmin := false + seenMailRspamd := false + for _, entry := range cfg.Startup.CriticalServiceEndpoints { + if entry == "mailu-mailserver/mailu-admin" { + seenMailAdmin = true + } + if entry == "mailu-mailserver/mailu-rspamd" { + seenMailRspamd = true + } + } + if !seenMailAdmin || !seenMailRspamd { + t.Fatalf("expected titan config to include Mailu critical endpoints, admin=%v rspamd=%v", seenMailAdmin, seenMailRspamd) + } +} + // TestValidateServiceChecklistAuthContracts runs one orchestration or CLI step. // Signature: TestValidateServiceChecklistAuthContracts(t *testing.T). // Why: covers service-checklist auth and final-url validation branches that are diff --git a/testing/hygiene/in_tree_test_allowlist.txt b/testing/hygiene/in_tree_test_allowlist.txt index 1c98892..ffaf5be 100644 --- a/testing/hygiene/in_tree_test_allowlist.txt +++ b/testing/hygiene/in_tree_test_allowlist.txt @@ -4,6 +4,7 @@ cmd/ananke/bootstrap_handoff_additional_test.go cmd/ananke/bootstrap_handoff_test.go cmd/ananke/builder_test.go cmd/ananke/command_additional_test.go +cmd/ananke/command_handlers_autoheal_test.go cmd/ananke/command_handlers_injection_test.go cmd/ananke/command_handlers_status_error_test.go cmd/ananke/command_handlers_test.go @@ -16,9 +17,11 @@ internal/cluster/orchestrator_report_test.go internal/cluster/orchestrator_autorepair_test.go internal/cluster/orchestrator_autorepair_cleanup_test.go internal/cluster/orchestrator_autorepair_proxy_test.go +internal/cluster/orchestrator_autorepair_service_test.go internal/cluster/orchestrator_critical_endpoint_additional_test.go internal/cluster/orchestrator_cordon_lease_test.go internal/cluster/orchestrator_test.go +internal/cluster/orchestrator_hardening_test.go internal/cluster/orchestrator_unit_additional_test.go internal/cluster/orchestrator_workload_recovery_test.go internal/cluster/orchestrator_vault_test.go diff --git a/testing/orchestrator/drill_lifecycle_test.go b/testing/orchestrator/drill_lifecycle_test.go index a0f4269..110fba7 100644 --- a/testing/orchestrator/drill_lifecycle_test.go +++ b/testing/orchestrator/drill_lifecycle_test.go @@ -147,9 +147,9 @@ func lifecycleDispatcher(recorder *commandRecorder) func(context.Context, time.D case strings.Contains(command, "jsonpath={.spec.ref.branch}"): return "main", nil case strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"): - return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false,"timeout":"30s"},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}}]}`, nil + return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false,"timeout":"30s"},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false,"timeout":"30s"},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}}]}`, nil case strings.Contains(command, "-n flux-system get kustomizations.kustomize.toolkit.fluxcd.io -o jsonpath="): - return "services\n", nil + return "flux-system\nservices\n", nil case strings.Contains(command, "get helmreleases.helm.toolkit.fluxcd.io -A -o jsonpath="): return "monitoring/grafana\n", nil case strings.Contains(command, "annotate kustomizations.kustomize.toolkit.fluxcd.io"): diff --git a/testing/orchestrator/drill_recovery_test.go b/testing/orchestrator/drill_recovery_test.go index 3f7c8b0..bdd21ac 100644 --- a/testing/orchestrator/drill_recovery_test.go +++ b/testing/orchestrator/drill_recovery_test.go @@ -155,6 +155,7 @@ func TestLifecycleStartupFluxImmutableJobSelfHeal(t *testing.T) { cfg := lifecycleConfig(t) cfg.Startup.RequireCriticalServiceEndpoints = false cfg.Startup.RequireWorkloadConvergence = false + cfg.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/services"} cfg.Startup.FluxHealthWaitSeconds = 4 cfg.Startup.FluxHealthPollSeconds = 1 @@ -186,15 +187,15 @@ func TestLifecycleStartupFluxImmutableJobSelfHeal(t *testing.T) { case strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"): fluxCalls++ if fluxCalls <= 2 { - return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","message":"Job update failed: field is immutable"}]}}]}`, nil + return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","message":"Job update failed: field is immutable"}]}}]}`, nil } - return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil + return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil case strings.Contains(command, "get jobs -A -o json"): return `{"items":[{"metadata":{"namespace":"flux-system","name":"reconcile-services","labels":{"kustomize.toolkit.fluxcd.io/name":"services"}},"status":{"failed":1,"conditions":[{"type":"Failed","status":"True"}]}}]}`, nil case strings.Contains(command, "delete job reconcile-services"): return "", nil case strings.Contains(command, "-n flux-system get kustomizations.kustomize.toolkit.fluxcd.io -o jsonpath="): - return "services\n", nil + return "flux-system\nservices\n", nil case strings.Contains(command, "get helmreleases.helm.toolkit.fluxcd.io -A -o jsonpath="): return "", nil case strings.Contains(command, "patch "), strings.Contains(command, "annotate "): diff --git a/testing/orchestrator/hooks_access_deep_matrix_test.go b/testing/orchestrator/hooks_access_deep_matrix_test.go index e77cefb..d86e50d 100644 --- a/testing/orchestrator/hooks_access_deep_matrix_test.go +++ b/testing/orchestrator/hooks_access_deep_matrix_test.go @@ -32,7 +32,7 @@ func TestHookAccessDeepMatrixReconcileAndSSHAuth(t *testing.T) { } orch, _ := newHookOrchestrator(t, cfg, run, run) err := orch.TestHookReconcileNodeAccess(context.Background(), []string{"titan-db", "titan-23"}) - if err == nil || !strings.Contains(err.Error(), "access validation had") || !strings.Contains(err.Error(), "missing sudo access") { + if err == nil || !strings.Contains(err.Error(), "access validation had 2 errors") || !strings.Contains(err.Error(), "host-privilege-unavailable") { t.Fatalf("expected aggregated reconcile error, got %v", err) } }) diff --git a/testing/orchestrator/hooks_access_vault_lifecycle_matrix_test.go b/testing/orchestrator/hooks_access_vault_lifecycle_matrix_test.go index 28b7f62..a5e131d 100644 --- a/testing/orchestrator/hooks_access_vault_lifecycle_matrix_test.go +++ b/testing/orchestrator/hooks_access_vault_lifecycle_matrix_test.go @@ -241,7 +241,7 @@ func TestHookAccessVaultLifecycleMatrix(t *testing.T) { case name == "kubectl" && strings.Contains(command, "get ingress -A -o json"): return `{"items":[]}`, nil case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"): - return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil + return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"): return `{"items":[{"kind":"Deployment","metadata":{"namespace":"monitoring","name":"grafana"},"spec":{"replicas":1,"template":{"spec":{}}},"status":{"readyReplicas":1}}]}`, nil case name == "kubectl" && strings.Contains(command, "get endpoints victoria-metrics-single-server"): diff --git a/testing/orchestrator/hooks_branch_saturation_test.go b/testing/orchestrator/hooks_branch_saturation_test.go index f807616..070f59a 100644 --- a/testing/orchestrator/hooks_branch_saturation_test.go +++ b/testing/orchestrator/hooks_branch_saturation_test.go @@ -323,7 +323,7 @@ func TestHookStartupConvergenceAndStabilitySaturation(t *testing.T) { case name == "kubectl" && strings.Contains(command, "get ingress -A -o json"): return `{"items":[]}`, nil case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"): - return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"Ready","message":"ok"}]}}]}`, nil + return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"Ready","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"Ready","message":"ok"}]}}]}`, nil case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"): return `{"items":[]}`, nil case name == "kubectl" && strings.Contains(command, "get pods -A -o json"): diff --git a/testing/orchestrator/hooks_health_failure_matrix_test.go b/testing/orchestrator/hooks_health_failure_matrix_test.go index 23c4f5b..94b3e57 100644 --- a/testing/orchestrator/hooks_health_failure_matrix_test.go +++ b/testing/orchestrator/hooks_health_failure_matrix_test.go @@ -253,9 +253,17 @@ func TestHookFluxHealthFailureBranches(t *testing.T) { t.Run("flux-health-ready-and-immutable-job-heal-branches", func(t *testing.T) { cfg := lifecycleConfig(t) cfg.Startup.IgnoreFluxKustomizations = []string{"infra/ignored"} + cfg.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/services"} fluxItems := map[string]any{ "items": []map[string]any{ + { + "metadata": map[string]any{"namespace": "flux-system", "name": "flux-system"}, + "spec": map[string]any{"suspend": false, "timeout": "30s"}, + "status": map[string]any{"conditions": []map[string]any{ + {"type": "Ready", "status": "True", "reason": "ReconciliationSucceeded", "message": "ok"}, + }}, + }, { "metadata": map[string]any{"namespace": "infra", "name": "ignored"}, "spec": map[string]any{"suspend": false, "timeout": "30s"}, diff --git a/testing/orchestrator/hooks_health_storage_test.go b/testing/orchestrator/hooks_health_storage_test.go index aea1498..a96195d 100644 --- a/testing/orchestrator/hooks_health_storage_test.go +++ b/testing/orchestrator/hooks_health_storage_test.go @@ -23,6 +23,7 @@ func TestHookFluxHealthAndStorageBranches(t *testing.T) { cfg.Startup.FluxHealthWaitSeconds = 2 cfg.Startup.FluxHealthPollSeconds = 1 cfg.Startup.IgnoreFluxKustomizations = []string{"flux-system/skip-me"} + cfg.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/services"} cfg.Startup.StorageReadyWaitSeconds = 2 cfg.Startup.StorageReadyPollSeconds = 1 cfg.Startup.StorageMinReadyNodes = 1 @@ -38,9 +39,9 @@ func TestHookFluxHealthAndStorageBranches(t *testing.T) { recorder.record(name, args) fluxCalls++ if fluxCalls <= 1 { - return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","reason":"Unknown","message":"waiting"}]}}]}`, nil + return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","reason":"Unknown","message":"waiting"}]}}]}`, nil } - return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}}]}`, nil + return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}}]}`, nil case name == "kubectl" && strings.Contains(command, "get jobs -A -o json"): recorder.record(name, args) return `{"items":[]}`, nil diff --git a/testing/orchestrator/hooks_lowfile_coverage_test.go b/testing/orchestrator/hooks_lowfile_coverage_test.go index 9477b23..2ed9eeb 100644 --- a/testing/orchestrator/hooks_lowfile_coverage_test.go +++ b/testing/orchestrator/hooks_lowfile_coverage_test.go @@ -54,8 +54,8 @@ func TestHookLowFileCoverageBoost(t *testing.T) { if !cluster.TestHookPodControllerOwned([]string{"DaemonSet"}) { t.Fatalf("expected DaemonSet owner to be controller-owned") } - if cluster.TestHookPodControllerOwned([]string{"Job"}) { - t.Fatalf("expected Job owner to be non controller-owned") + if !cluster.TestHookPodControllerOwned([]string{"Job"}) { + t.Fatalf("expected Job owner to be controller-owned") } if got := cluster.TestHookStuckContainerReason([]string{"ImagePullBackOff"}, nil, []string{"ImagePullBackOff"}); got != "ImagePullBackOff" { @@ -370,7 +370,7 @@ func TestHookLowFileCoverageBoost(t *testing.T) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"): - return `{"items":[{"metadata":{"namespace":"flux-system","name":"ready"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil + return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"ready"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil default: return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } @@ -382,7 +382,7 @@ func TestHookLowFileCoverageBoost(t *testing.T) { if ready, detail, err := orchFlux.TestHookFluxHealthReady(context.Background()); err != nil || ready || !strings.Contains(detail, "not ready") { t.Fatalf("expected flux health not-ready result, ready=%v detail=%q err=%v", ready, detail, err) } - if ready, detail, err := orchNoTimeout.TestHookFluxHealthReady(context.Background()); err != nil || !ready || !strings.Contains(detail, "all kustomizations ready=") { + if ready, detail, err := orchNoTimeout.TestHookFluxHealthReady(context.Background()); err != nil || !ready || !strings.Contains(detail, "ready=") { t.Fatalf("expected flux health ready result, ready=%v detail=%q err=%v", ready, detail, err) } if !cluster.TestHookLooksLikeImmutableJobError("Job update failed: field is immutable") { diff --git a/testing/orchestrator/hooks_lowfunc_matrix_test.go b/testing/orchestrator/hooks_lowfunc_matrix_test.go index 89aae56..9b7e550 100644 --- a/testing/orchestrator/hooks_lowfunc_matrix_test.go +++ b/testing/orchestrator/hooks_lowfunc_matrix_test.go @@ -124,7 +124,7 @@ func TestHookServiceStabilityLowFunctionMatrix(t *testing.T) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"): - return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil + return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"): return `{"items":[]}`, nil case name == "kubectl" && strings.Contains(command, "get pods -A -o json"): diff --git a/testing/orchestrator/hooks_vault_lifecycle_branch_matrix_test.go b/testing/orchestrator/hooks_vault_lifecycle_branch_matrix_test.go index bf1b730..40191a4 100644 --- a/testing/orchestrator/hooks_vault_lifecycle_branch_matrix_test.go +++ b/testing/orchestrator/hooks_vault_lifecycle_branch_matrix_test.go @@ -283,11 +283,12 @@ func TestHookVaultLifecycleBranchMatrix(t *testing.T) { t.Run("flux-ingress-service-and-inventory-low-branches", func(t *testing.T) { t.Run("flux-health-ready-reason-fallback-and-heal-query-error", func(t *testing.T) { cfg := lifecycleConfig(t) + cfg.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/apps"} runReady := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") switch { case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"): - return `{"items":[{"metadata":{"namespace":"","name":"skip"},"spec":{"suspend":false},"status":{"conditions":[]}},{"metadata":{"namespace":"flux-system","name":"apps"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","reason":"","message":""}]}}]}`, nil + return `{"items":[{"metadata":{"namespace":"","name":"skip"},"spec":{"suspend":false},"status":{"conditions":[]}},{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"Ready","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"apps"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","reason":"","message":""}]}}]}`, nil case name == "kubectl" && strings.Contains(command, "get jobs -A -o json"): return "", errors.New("jobs query failed") default: diff --git a/testing/orchestrator/hooks_vault_poststart_branch_matrix_test.go b/testing/orchestrator/hooks_vault_poststart_branch_matrix_test.go index ad41173..d1fc437 100644 --- a/testing/orchestrator/hooks_vault_poststart_branch_matrix_test.go +++ b/testing/orchestrator/hooks_vault_poststart_branch_matrix_test.go @@ -226,10 +226,11 @@ func TestHookVaultPostStartBranchMatrix(t *testing.T) { runMissingReady := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) { command := name + " " + strings.Join(args, " ") if name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json") { - return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Reconciling","status":"True","reason":"Progressing"}]}}]}`, nil + return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"Ready","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Reconciling","status":"True","reason":"Progressing"}]}}]}`, nil } return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...) } + cfg.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/services"} orchMissingReady, _ := newHookOrchestrator(t, cfg, runMissingReady, runMissingReady) ready, detail, err := orchMissingReady.TestHookFluxHealthReady(context.Background()) if err != nil || ready || !strings.Contains(detail, "ready condition missing") {