recovery: harden post-start auto-heal

cmd/ananke/command_handlers.go

@@ -56,6 +56,9 @@ var (
 	etcdRestoreOrchestratorCommand = func(ctx context.Context, orch *cluster.Orchestrator, opts cluster.EtcdRestoreOptions) error {
 		return orch.EtcdRestore(ctx, opts)
 	}
+	autoHealOrchestratorCommand = func(ctx context.Context, orch *cluster.Orchestrator) error {
+		return orch.RunPostStartAutoHeal(ctx)
+	}
 	daemonRunCommand   = func(ctx context.Context, daemon *service.Daemon) error { return daemon.Run(ctx) }
 	readIntentCommand  = state.ReadIntent
 	writeIntentCommand = state.MustWriteIntent
@@ -132,6 +135,25 @@ func runStartup(logger *log.Logger, args []string) error {
 	})
 }
 
+// runAutoHeal runs one orchestration or CLI step.
+// Signature: runAutoHeal(logger *log.Logger, args []string) error.
+// Why: operators need a narrow way to run Ananke's daemon repair pass once
+// without invoking a full startup or depending on the UPS daemon loop.
+func runAutoHeal(logger *log.Logger, args []string) error {
+	fs := flag.NewFlagSet("auto-heal", flag.ExitOnError)
+	configPath := fs.String("config", "/etc/ananke/ananke.yaml", "Path to config file")
+	execute := fs.Bool("execute", false, "Actually execute changes (default dry-run)")
+	_ = fs.Parse(args)
+
+	_, orch, err := buildOrchestratorCommand(logger, *configPath, !*execute)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	return autoHealOrchestratorCommand(ctx, orch)
+}
+
 // runShutdown runs one orchestration or CLI step.
 // Signature: runShutdown(logger *log.Logger, args []string) error.
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

cmd/ananke/command_handlers_autoheal_test.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"context"
+	"io"
+	"log"
+	"testing"
+
+	"scm.bstein.dev/bstein/ananke/internal/cluster"
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestRunAutoHealInvokesPostStartRepair runs one orchestration or CLI step.
+// Signature: TestRunAutoHealInvokesPostStartRepair(t *testing.T).
+// Why: the one-shot auto-heal command must call the same repair entrypoint as
+// the daemon loop so operators can run Ananke's post-start repairs directly.
+func TestRunAutoHealInvokesPostStartRepair(t *testing.T) {
+	restore := stubCommandHandlerHooks()
+	defer restore()
+
+	called := false
+	buildOrchestratorCommand = func(_ *log.Logger, _ string, dryRun bool) (config.Config, *cluster.Orchestrator, error) {
+		cfg := minimalHandlerConfig(t)
+		return cfg, newTestOrchestrator(cfg, dryRun), nil
+	}
+	autoHealOrchestratorCommand = func(_ context.Context, orch *cluster.Orchestrator) error {
+		called = true
+		if orch == nil {
+			t.Fatalf("expected orchestrator")
+		}
+		return nil
+	}
+	logger := log.New(io.Discard, "", 0)
+	if err := runAutoHeal(logger, []string{"--config", "/ignored.yaml", "--execute"}); err != nil {
+		t.Fatalf("expected auto-heal success, got %v", err)
+	}
+	if !called {
+		t.Fatalf("expected auto-heal orchestrator hook to be called")
+	}
+}

cmd/ananke/command_handlers_injection_test.go

@@ -403,6 +403,7 @@ func stubCommandHandlerHooks() func() {
 	prevStartup := startupOrchestratorCommand
 	prevShutdown := shutdownOrchestratorCommand
 	prevRestore := etcdRestoreOrchestratorCommand
+	prevAutoHeal := autoHealOrchestratorCommand
 	prevDaemon := daemonRunCommand
 	prevReadIntent := readIntentCommand
 	prevWriteIntent := writeIntentCommand
@@ -416,6 +417,7 @@ func stubCommandHandlerHooks() func() {
 		startupOrchestratorCommand = prevStartup
 		shutdownOrchestratorCommand = prevShutdown
 		etcdRestoreOrchestratorCommand = prevRestore
+		autoHealOrchestratorCommand = prevAutoHeal
 		daemonRunCommand = prevDaemon
 		readIntentCommand = prevReadIntent
 		writeIntentCommand = prevWriteIntent

cmd/ananke/main_dispatch.go

@@ -11,6 +11,7 @@ var (
 	shutdownCommand    = runShutdown
 	etcdRestoreCommand = runEtcdRestore
 	daemonCommand      = runDaemon
+	autoHealCommand    = runAutoHeal
 	statusCommand      = runStatus
 	intentCommand      = runIntent
 )
@@ -54,6 +55,11 @@ func runMain(logger *log.Logger, args []string) int {
 			logger.Printf("daemon failed: %v", err)
 			return 1
 		}
+	case "auto-heal":
+		if err := autoHealCommand(logger, args[2:]); err != nil {
+			logger.Printf("auto-heal failed: %v", err)
+			return 1
+		}
 	case "status":
 		if err := statusCommand(logger, args[2:]); err != nil {
 			logger.Printf("status failed: %v", err)
@@ -88,6 +94,7 @@ Commands:
   shutdown  Perform graceful cluster shutdown
   etcd-restore  Restore etcd from snapshot on a control plane
   daemon    Monitor UPS and auto-trigger shutdown
+  auto-heal Run one post-start auto-heal pass
   status    Print current ananke status and estimates
   intent    Read or manually set intent state
 
@@ -96,6 +103,7 @@ Examples:
   ananke shutdown --config /etc/ananke/ananke.yaml --execute --mode cluster-only --reason "manual-maintenance"
   ananke etcd-restore --config /etc/ananke/ananke.yaml --execute
   ananke daemon --config /etc/ananke/ananke.yaml
+  ananke auto-heal --config /etc/ananke/ananke.yaml --execute
   ananke status --config /etc/ananke/ananke.yaml
   ananke intent --config /etc/ananke/ananke.yaml --set normal --reason "manual-clear" --execute
 

cmd/ananke/main_dispatch_additional_test.go

@@ -71,10 +71,11 @@ func TestRunMainDispatchSuccessPaths(t *testing.T) {
 		func(_ *log.Logger, _ []string) error { return nil },
 		func(_ *log.Logger, _ []string) error { return nil },
 		func(_ *log.Logger, _ []string) error { return nil },
+		func(_ *log.Logger, _ []string) error { return nil },
 	)
 	defer restore()
 
-	commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "status", "intent", "help", "-h", "--help"}
+	commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "auto-heal", "status", "intent", "help", "-h", "--help"}
 	for _, command := range commands {
 		code := runMain(logger, []string{"ananke", command})
 		if code != 0 {
@@ -96,10 +97,11 @@ func TestRunMainDispatchErrorPaths(t *testing.T) {
 		func(_ *log.Logger, _ []string) error { return fail },
 		func(_ *log.Logger, _ []string) error { return fail },
 		func(_ *log.Logger, _ []string) error { return fail },
+		func(_ *log.Logger, _ []string) error { return fail },
 	)
 	defer restore()
 
-	commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "status", "intent"}
+	commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "auto-heal", "status", "intent"}
 	for _, command := range commands {
 		code := runMain(logger, []string{"ananke", command})
 		if code != 1 {
@@ -122,13 +124,14 @@ func TestRunMainUsageAndUnknownPaths(t *testing.T) {
 }
 
 // stubDispatchCommands runs one orchestration or CLI step.
-// Signature: stubDispatchCommands(startup, shutdown, etcdRestore, daemon, status, intent func(*log.Logger, []string) error) func().
+// Signature: stubDispatchCommands(startup, shutdown, etcdRestore, daemon, autoHeal, status, intent func(*log.Logger, []string) error) func().
 // Why: keeps command-var overrides scoped and safely restorable across tests.
 func stubDispatchCommands(
 	startup,
 	shutdown,
 	etcdRestore,
 	daemon,
+	autoHeal,
 	status,
 	intent func(*log.Logger, []string) error,
 ) func() {
@@ -136,6 +139,7 @@ func stubDispatchCommands(
 	prevShutdown := shutdownCommand
 	prevEtcdRestore := etcdRestoreCommand
 	prevDaemon := daemonCommand
+	prevAutoHeal := autoHealCommand
 	prevStatus := statusCommand
 	prevIntent := intentCommand
 
@@ -143,6 +147,7 @@ func stubDispatchCommands(
 	shutdownCommand = shutdown
 	etcdRestoreCommand = etcdRestore
 	daemonCommand = daemon
+	autoHealCommand = autoHeal
 	statusCommand = status
 	intentCommand = intent
 
@@ -151,6 +156,7 @@ func stubDispatchCommands(
 		shutdownCommand = prevShutdown
 		etcdRestoreCommand = prevEtcdRestore
 		daemonCommand = prevDaemon
+		autoHealCommand = prevAutoHeal
 		statusCommand = prevStatus
 		intentCommand = prevIntent
 	}

docs/ananke-hardening-prompt-2026-07-07.md

@@ -0,0 +1,465 @@
+# Prompt For Ananke Recovery Hardening
+
+You are a fresh, stateless Codex session working in the Ananke repository. Your goal is to implement, test, and validate generalized recovery improvements in Ananke based on a real Titan cluster power-loss incident on 2026-07-07. Treat this as production reliability work: read the repo first, follow existing code style and architecture, keep changes cohesive, and solve failure classes categorically rather than hard-coding node names, namespaces, or one-off incidents.
+
+## Mission
+
+Make Ananke resilient and automatic during post-power-loss cluster recovery. After your changes, one Ananke startup/recovery process should be able to:
+
+- Bring Vault and critical dependencies back safely.
+- Repair or escalate wedged k3s/containerd nodes using existing Ananke SSH/host access.
+- Use sudo credentials from Vault, securely and noninteractively, for host repair and diagnostics.
+- Avoid leaving Longhorn storage nodes cordoned indefinitely.
+- Detect Kubernetes/Longhorn readiness mismatches and repair label/manager/VolumeAttachment drift.
+- Resolve stale pod/PVC ownership safely, with different behavior for sidecar-only stale owners versus still-running application containers that mount the PVC.
+- Continue post-success convergence monitoring long enough to catch late node/runtime/storage wedges.
+- Report concise, actionable blockers without noisy repeated pod recycling.
+
+Do not implement special cases for `titan-04`, `titan-05`, `titan-15`, `titan-17`, `titan-22`, Firefly, Jellyfin, or any specific app. Use those names only as incident fixtures/test names. The code should operate on generic managed nodes, generic Longhorn nodes, generic Kubernetes workloads, and generic RWO PVCs.
+
+## Incident Context
+
+The Titan cluster lost power for a few minutes. Ariadne may have started shutdown behavior. The operator asked for a single Ananke process to bring the cluster back up and learn from any manual help required.
+
+Ananke did a lot correctly:
+
+- It recovered Vault and unsealed it.
+- It recovered critical workloads, Vault injector, Postgres, Keycloak, OpenSearch, oauth2 logs, CoreDNS, metrics-server, and many app pods.
+- It ran a single startup workflow and eventually completed successfully.
+- It recycled many stuck pods and ran several checks.
+
+But Ananke needed manual assistance in multiple reusable failure classes:
+
+- Node runtime wedges where Kubernetes readiness did not fully reflect kubelet/containerd health.
+- Host repairs blocked by sudo privilege gaps.
+- Longhorn nodes cordoned by Kubernetes remained unavailable for storage scheduling.
+- Longhorn readiness diverged from Kubernetes readiness.
+- Missing node labels caused Longhorn manager pods to disappear.
+- Stale terminating pods held RWO volumes and blocked replacement pods.
+- Repeated pod deletes did not resolve storage ownership or runtime reservation issues.
+- Ananke declared startup success before all late runtime/storage fallout was resolved.
+
+## Live Incident Facts To Encode As Tests
+
+Use these as fixtures for tests and documentation, not as hard-coded behavior:
+
+- `titan-05`: k3s-agent restart hung. Host SSH worked. `k3s-agent.service` entered `deactivating`, `final-sigkill`, `activating/start`, and `Result=timeout`. Kubelet port `10250` stayed closed. Kubernetes heartbeat was stale with `node.kubernetes.io/unreachable` taints. In-cluster helper pod could not start. Manual controlled reboot recovered it.
+- `titan-04`: after Ananke reported startup success, the node was Kubernetes `Ready` and kubelet `10250` initially open, but new pods failed with `FailedCreatePodSandBox`, `failed to reserve sandbox name`, and `context deadline exceeded`. A non-blocking `k3s-agent` restart then stuck in `deactivating/stop-sigkill`, kubelet `10250` closed, Kubernetes moved to `Ready=Unknown`, and a controlled reboot recovered it.
+- `titan-07`: a pod looked running in the API, but `kubectl exec` failed with `cannot exec in a deleted state`; restarting k3s-agent and recreating the pod fixed it.
+- `titan-14`: a replacement pod hit `failed to reserve container name` / `CreateContainerError`; k3s-agent restart and rerunning the in-cluster helper fixed it. Later another pod on the same node showed the same sandbox reservation pattern, but then progressed after enough time. Tests should distinguish transient progress from a stall.
+- `titan-15` and `titan-17`: Ananke cordoned them after encrypted Longhorn mounts exposed missing `cryptsetup`. Longhorn itself had `allowScheduling=true` and ready disks, but Longhorn `Schedulable=False` because Kubernetes had cordoned the nodes. Manual `kubectl uncordon titan-15 titan-17` restored Kubernetes scheduling and Longhorn schedulability. Ananke had attempted `cryptsetup-bin` repair but failed because `atlas` sudo required a password.
+- `titan-22`: Kubernetes `Ready=True`, CSI plugin and engine images were running, but Longhorn node `Ready=False` because `longhorn-manager` pod was missing. The root cause was missing node label `longhorn-host=true`, while the `longhorn-manager` DaemonSet selected `longhorn-host=true`. Manual `kubectl label node titan-22 longhorn-host=true --overwrite` started `longhorn-manager` and Longhorn node Ready became true. A pending wallet pod still carried an old failed VolumeAttachment until the pod was deleted/recreated.
+- Firefly stale owner, safe-to-force variant: `finance/firefly-...-q8rtq` was terminating for more than 25 minutes, had no finalizers, and only a Vault sidecar remained running. Replacement pod was blocked by `Multi-Attach` for an RWO PVC. Force-deleting the stale pod cleared ownership and allowed Longhorn attach to the replacement.
+- Firefly stale owner, unsafe-to-force variant: a later terminating Firefly pod still had the main application container running and mounting the RWO PVC. It was not safe to blindly force-delete. Waiting allowed kubelet to finish termination, then Longhorn attached the volume to the replacement.
+- Manual runtime inspection attempt on `titan-06`: `sudo -n crictl ...` failed because `atlas` required a sudo password. Ananke is expected to have access to sudo passwords in Vault, so this must be designed as a first-class Ananke capability rather than manual operator shell access.
+- Vault health check robustness: at one point `kubectl exec vault-0 -- vault status` was killed, but HTTP health via Kubernetes pod proxy showed `initialized=true`, `sealed=false`, `standby=false`. Startup should not fail solely on transient `kubectl exec` failure when endpoint and HTTP health prove Vault is available.
+- Image/DNS issues: several image pulls failed with `lookup registry-1.docker.io`, `lookup auth.docker.io`, or `lookup production.cloudfront.docker.com: Try again`, then later succeeded. Ananke repeatedly recycling unchanged ImagePullBackOff pods created noise.
+
+## Required Implementation Areas
+
+### 1. Secure Sudo Capability From Vault
+
+Ananke should be able to run bounded host diagnostics and repairs that require sudo.
+
+Requirements:
+
+- Discover and use the existing Vault integration and secret naming conventions already present in the repo/config. Do not invent a new hard-coded secret path if the repo has one.
+- Support noninteractive sudo for configured managed nodes.
+- Never log sudo passwords, Vault tokens, rendered secrets, command input containing secrets, or full environment dumps.
+- Provide a preflight check: for each managed node, can Ananke retrieve host sudo material and can it run a harmless command such as `sudo -S -p '' true` with a timeout?
+- If credentials are missing, wrong, or Vault is unavailable, Ananke should report `host-privilege-unavailable` with node, intended action, and secret lookup class, not spin.
+- Use strict command allowlists for privileged commands. The recovery path should not become arbitrary remote root execution.
+- Use bounded timeouts for all host commands and classify timeout versus auth failure versus command failure.
+- Prefer safe wrappers for:
+  - `systemctl show/is-active/restart --no-block k3s-agent`
+  - `systemctl reboot` when configured and escalated
+  - `crictl ps/pods/inspect/stop` for recovery diagnostics and controlled container stop
+  - package checks/install for `cryptsetup-bin`, `open-iscsi`, `nfs-common`, `dmsetup`
+  - `modprobe dm_crypt` or equivalent kernel-module validation where appropriate
+
+Tests:
+
+- Vault lookup succeeds and sudo command succeeds.
+- Vault lookup succeeds but sudo password is wrong.
+- Vault lookup missing.
+- Vault unavailable while Vault HTTP health is also degraded.
+- Vault unavailable but action does not require sudo.
+- Command times out.
+- Command returns nonzero.
+- Logs are scanned to ensure no password appears.
+
+### 2. Managed Node Runtime Recovery Primitive
+
+Implement a generic state machine for node runtime recovery.
+
+Detection signals:
+
+- Kubernetes node `Ready=Unknown` or stale heartbeat.
+- Kubernetes node `Ready=True` but pods on that node show repeated:
+  - `FailedCreatePodSandBox`
+  - `failed to reserve sandbox name`
+  - `failed to reserve container name`
+  - `CreateContainerError`
+  - `FailedKillPod`
+  - long-lived `ContainerCreating` or `PodInitializing` with no progress
+  - `kubectl exec`/logs failures indicating deleted runtime state
+- Host SSH works but kubelet port `127.0.0.1:10250` is closed or slow.
+- `systemctl show k3s-agent` reports `deactivating`, `stop-sigterm`, `stop-sigkill`, `final-sigterm`, `final-sigkill`, `activating/start`, or `Result=timeout`.
+- Existing in-cluster maintenance helper pod for the node cannot start, stays `ContainerCreating`, or cannot be scheduled because the node runtime is already unhealthy.
+
+Escalation ladder:
+
+- Open a single node recovery incident and suppress noisy repeated pod recycling for pods on that node while host repair is in progress.
+- Cordon the node before host-level intervention unless it is already unreachable/tainted.
+- Try in-cluster maintenance helper if kubelet/runtime can create pods.
+- If helper cannot run, use Ananke SSH route and sudo from Vault.
+- Prefer `systemctl --no-block restart k3s-agent`; never use a blocking restart over SSH as the primary recovery action.
+- Wait a bounded interval and require both:
+  - Kubernetes Ready=True with fresh heartbeat
+  - local kubelet `10250` open
+- If k3s-agent remains stuck in stop/kill/activating states or heartbeats remain stale, escalate to configured controlled reboot.
+- After reboot, wait for SSH, k3s-agent active/running, kubelet port open, Kubernetes Ready=True, and unreachable taints cleared.
+- Uncordon only after the node is healthy and after storage-specific checks pass.
+- Continue this monitoring after startup success for a configurable post-success convergence window.
+
+Tests:
+
+- Ready node with repeated sandbox reservation events triggers recovery.
+- Ready node with one transient sandbox event and then progress does not trigger restart.
+- Unknown node with SSH unavailable reports external blocker without repeated action.
+- Unknown node with SSH available, kubelet closed, k3s-agent stuck, helper unavailable escalates to reboot.
+- Non-blocking restart recovers node.
+- Non-blocking restart moves service to `deactivating/stop-sigkill`; reboot escalation occurs.
+- Reboot command accepted but SSH does not return within timeout.
+- Reboot returns and all readiness conditions clear.
+- False positive prevention: node Ready=True, kubelet open, events old/stale only, no action.
+- Idempotency: repeated Ananke runs do not issue repeated restarts/reboots while an incident is active.
+
+### 3. Longhorn And Kubernetes Readiness Reconciliation
+
+Ananke must not treat Kubernetes node Ready as sufficient for storage workloads.
+
+Implement a Longhorn readiness reconciler:
+
+- Read Kubernetes Nodes, Longhorn Node CRs, Longhorn manager DaemonSet, Longhorn manager pods, CSI pods, engine-image pods, Volume CRs, VolumeAttachments, and relevant PVC-bound pods.
+- Compare:
+  - Kubernetes Ready/SchedulingDisabled/taints
+  - Node labels required by Longhorn manager DaemonSet, especially `longhorn-host=true`
+  - Longhorn Node `Ready` and `Schedulable` conditions
+  - Longhorn Node `spec.allowScheduling`
+  - Longhorn manager pod presence on nodes where Longhorn Node exists or where PVC workloads are allowed
+  - CSI plugin readiness
+  - VolumeAttachment errors that reference stale Longhorn node readiness
+- Detect label drift: Longhorn node exists and CSI/engine pods exist, but manager pod is missing because the node no longer matches the DaemonSet selector.
+- Repair label drift only if it is safe:
+  - There is a known desired label source in repo/config/state, or
+  - Longhorn Node CR exists for that Kubernetes node and existing Longhorn metadata indicates the node is intended to be a Longhorn node.
+  - Emit exactly what label will be restored and why.
+- If safe repair cannot be proven, cordon or avoid scheduling PVC workloads to the node and report a blocker.
+- After restoring labels or manager pods, wait for Longhorn Node Ready=True.
+- For VolumeAttachments that failed because Longhorn thought a now-ready node was not ready, trigger a bounded retry:
+  - Prefer deleting/recreating the controller-owned Pending pod if that is the established pattern.
+  - Delete stale failed VolumeAttachment only if Kubernetes controller semantics and repo policy allow it.
+  - Never delete PV/PVC/Longhorn Volume data.
+
+Specific expectations from incident:
+
+- `titan-22` class: Kubernetes Ready, missing `longhorn-host=true`, no manager pod, Longhorn Node Ready=False ManagerPodMissing. Ananke should restore label or report exact unsafe reason, then wait for manager pod and Longhorn Ready.
+- `titan-15`/`titan-17` class: Kubernetes cordon makes Longhorn Schedulable=False despite Longhorn `allowScheduling=true` and ready disks. Ananke should not leave them cordoned after the blocking repair condition is resolved or after operator-approved override.
+
+Tests:
+
+- Longhorn manager selector changes are handled generically.
+- Missing Longhorn label restored from desired state.
+- Missing Longhorn label not restored if node lacks Longhorn Node CR and no desired-state evidence exists.
+- Longhorn Node Ready=False ManagerPodMissing becomes Ready after label restoration and manager pod startup.
+- VolumeAttachment error due stale Longhorn node readiness is retried after Longhorn Ready.
+- Kubernetes cordoned node causes Longhorn Schedulable=False and is uncordoned after repair.
+- Kubernetes cordoned node remains cordoned if required package/kernel preflight still fails and no override is configured.
+- `allowScheduling=false` is respected; Ananke should not enable Longhorn disk scheduling unless explicitly configured, but it may still ensure manager/CSI readiness for attachment if workloads are allowed there.
+
+### 4. Cordon/Uncordon And Host Package Repair Policy
+
+Ananke cordoned `titan-15` and `titan-17` because encrypted Longhorn mounts exposed missing `cryptsetup`; it failed to install because sudo required a password. This must become a closed-loop repair.
+
+Requirements:
+
+- Distinguish:
+  - Kubernetes scheduling cordon
+  - Longhorn disk scheduling (`allowScheduling`)
+  - Longhorn node readiness/schedulability
+  - ordinary `PreferNoSchedule` taints used as soft placement hints
+- If Ananke cordons a node, it must record why, when, what remediation is required, and what condition will uncordon it.
+- If the remediation is package/module installation, use Vault-backed sudo and verify the result.
+- If package repair succeeds, uncordon automatically and verify Longhorn Schedulable=True.
+- If package repair fails due missing sudo secret, wrong sudo password, package manager lock, apt/dpkg failure, or network outage, report the exact class and do not leave the node silently disabled.
+- Support an operator override policy for "uncordon for Longhorn availability even if encrypted workload preflight is not perfect", but this must be explicit/configured and logged.
+- Do not remove existing soft taints such as `atlas.bstein.dev/spillover=true:PreferNoSchedule` or `longhorn=true:PreferNoSchedule` unless the repo’s desired-state policy says to.
+
+Tests:
+
+- Missing cryptsetup with valid sudo installs package, loads module if needed, uncordons node.
+- Missing cryptsetup with missing sudo secret reports privilege blocker and leaves cordon with reason.
+- Node already has package but missing kernel module: module load path tested.
+- Node has Longhorn ready disks but is Kubernetes cordoned: Longhorn schedulability blocker is reported.
+- Uncordon preserves unrelated soft taints.
+- Re-running recovery is idempotent.
+
+### 5. Stale RWO PVC Owner Recovery
+
+Implement a generic stale-owner recovery flow for controller-owned pods and RWO PVCs.
+
+Detection:
+
+- Old pod has `metadata.deletionTimestamp` older than threshold.
+- Replacement pod from same controller exists and is pending/initializing.
+- Replacement events include `Multi-Attach`, `FailedAttachVolume`, `Volume is already used by pod(s)`, or `Volume is already exclusively attached to one node`.
+- PVC is `ReadWriteOnce` or equivalent single-writer mode.
+- Old and new pods are on different nodes.
+- Longhorn Volume/VolumeAttachment shows old node still owns attachment or new attach blocked.
+- Ordinary pod deletes have been attempted repeatedly without changing ownership.
+
+Recovery grouping:
+
+- Group old pod, replacement pod, controller, PVC, PV, VolumeAttachment(s), Longhorn Volume, old node, new node, and relevant events into one incident.
+- Emit one concise operator record.
+- Stop repeated blind pod deletes while the incident is active.
+
+Safety decisions:
+
+- Inspect old pod container statuses and volume mounts.
+- Classify containers:
+  - PVC-writing application containers: mount the blocked PVC.
+  - Sidecars that do not mount the blocked PVC: Vault agent, projected service account, emptyDir-only, etc.
+  - Unknown containers: treat as unsafe.
+- If only sidecars remain and no container mounts the PVC, force-delete the stale pod after a bounded wait.
+- If an application container still runs and mounts the PVC, do not blindly force-delete. Instead:
+  - Wait for kubelet termination if it is making progress.
+  - If stalled and host runtime inspection is available through Vault-backed sudo, inspect and optionally stop the specific container through a controlled allowlisted path.
+  - If Ananke cannot prove the PVC-writing container is stopped, report `unsafe-stale-owner` and do not clear the API object.
+- If old node kubelet/runtime is unhealthy, route through managed node recovery before stale-owner cleanup.
+
+Tests:
+
+- Sidecar-only stale pod force-clears and replacement attaches.
+- Live app container with blocked PVC waits and does not force-delete.
+- Live app container eventually exits; replacement attaches without force-delete.
+- Host runtime inspection unavailable due sudo privilege reports unsafe blocker.
+- Host runtime inspection available and controlled container stop succeeds.
+- Longhorn detach/attach follows cleanup.
+- Replacement readiness is rechecked after attach.
+- Multiple PVCs in one pod are handled.
+- Non-RWO PVC is not processed by this recovery path.
+
+### 6. Vault Health Robustness
+
+Startup should not fail because one Vault probe path is flaky if other authoritative health signals prove Vault is usable.
+
+Requirements:
+
+- Prefer multiple health signals:
+  - pod Running/Ready
+  - service endpoints populated
+  - HTTP `/v1/sys/health` through service or pod proxy
+  - `vault status` through exec as one signal, not the only signal
+- If `kubectl exec vault status` is killed/transiently fails but HTTP health reports initialized/unsealed/active or standby-acceptable per config, classify as transient exec failure and continue with warning.
+- If HTTP health and exec disagree, report clearly and retry with bounded backoff.
+- Never log tokens.
+
+Tests:
+
+- exec killed but HTTP health unsealed => startup continues with warning.
+- exec says sealed and HTTP says sealed => startup blocks/unseals.
+- endpoints missing => startup waits.
+- HTTP unavailable but exec succeeds => startup may proceed if policy allows.
+
+### 7. Image Pull And DNS Failure Classification
+
+Ananke should not repeatedly recycle pods whose only issue is unchanged image pull or DNS failure.
+
+Requirements:
+
+- Detect ImagePullBackOff/ErrImagePull events caused by DNS/registry lookup.
+- Run separate DNS checks:
+  - CoreDNS pods ready
+  - in-cluster DNS query from a known diagnostic pod if available
+  - node host DNS or kubelet image pull DNS failures grouped by node
+  - registry reachability by hostname class, not one app
+- If the image pull failure is unchanged and recent, do not repeatedly delete the pod.
+- Emit a compact blocker: image, registry host, node, last error, and whether DNS is cluster-wide or node-specific.
+- Retry only after DNS/registry health changes or after bounded interval.
+
+Tests:
+
+- Docker Hub auth DNS failure is classified.
+- Private registry DNS failure is classified.
+- One-off pull failure that later succeeds is not over-escalated.
+- Repeated pod recycle is suppressed.
+
+### 8. Startup Completion And Post-Success Monitoring
+
+Ananke declared `startup_status=success` while late runtime/storage issues still surfaced. Make completion smarter.
+
+Requirements:
+
+- Startup can reach "cluster core recovered" while still monitoring app convergence.
+- Do not mark total recovery complete until:
+  - critical services pass
+  - all managed nodes are either healthy, intentionally excluded, or explicitly blocked
+  - Longhorn node readiness/schedulability matches desired policy
+  - no active node-runtime incidents exist
+  - no unsafe stale-owner incidents exist
+- Add a post-success watch window for late events: sandbox reservation, FailedKillPod, kubelet metrics scrape failures, Longhorn manager missing, VolumeAttachment errors.
+- Distinguish statuses:
+  - `core_recovered`
+  - `converging`
+  - `success`
+  - `success_with_noncritical_blockers`
+  - `blocked_requires_operator`
+- Report app-level blockers separately from cluster-level blockers.
+
+Tests:
+
+- Critical services pass but node runtime wedge appears during post-success window => not final success until resolved.
+- Noncritical app image pull blocker can be `success_with_noncritical_blockers`.
+- Unsafe stale RWO owner blocks full success.
+- All clear => success.
+
+## Status And Operator Reporting
+
+Improve operator output. The user should not need to read journals to know what is happening.
+
+Status should include:
+
+- Whether a startup/recovery workflow is already running.
+- Current phase and elapsed time.
+- Current incidents grouped by type:
+  - node runtime recovery
+  - Longhorn/Kubernetes drift
+  - stale RWO owner
+  - Vault health
+  - image/DNS pull
+  - host privilege/preflight
+- For each incident: resource, trigger signals, action taken, next action, elapsed time, and whether operator input is needed.
+- Counts should be accurate and stable.
+- Repeated noisy details should be summarized with "same blocker still present for X minutes" instead of repeated lines.
+
+## Security Constraints
+
+- Do not log sudo passwords, Vault tokens, unsealed keys, or rendered secret contents.
+- Redact secrets in command arguments and environment.
+- Use command allowlists for privileged host actions.
+- Make destructive operations explicit and guarded:
+  - reboot only after escalation criteria
+  - force-delete pod only after stale-owner safety checks
+  - deleting VolumeAttachments only if repo policy allows and after safeguards
+- Keep audit logs of actions: node/resource, reason, command class, result, elapsed time, but not secrets.
+
+## Implementation Guidance
+
+Before editing:
+
+- Read Ananke’s existing startup/recovery code, Kubernetes client abstractions, Vault access layer, SSH/host command layer, Longhorn helpers if any, status/progress writer, and tests.
+- Reuse existing abstractions. Do not invent a parallel framework unless necessary.
+- Use fake clients and table-driven tests where the repo already uses them.
+- Keep changes modular:
+  - host privilege provider
+  - node runtime incident detector/reconciler
+  - Longhorn readiness reconciler
+  - stale RWO owner reconciler
+  - image/DNS classifier
+  - status/progress summarizer
+- Ensure all reconcilers are idempotent and bounded.
+
+Suggested model:
+
+- Define incident structs with stable keys, timestamps, observed signals, action history, next action, and terminal status.
+- Every recovery action should require:
+  - detection evidence
+  - policy permission
+  - safety gate
+  - timeout
+  - postcondition check
+- Avoid "delete pod until healthy" loops. Replace with evidence-driven remediation.
+
+## Validation Plan
+
+Run the repository’s standard checks first, then add targeted tests.
+
+Required validation:
+
+- Unit tests for each new classifier/reconciler.
+- Tests for redaction/no secret leakage.
+- Tests for idempotency on repeated reconcile calls.
+- Tests for progress/status output.
+- Integration-style tests with fake Kubernetes/Longhorn objects:
+  - nodes, pods, events, PVCs, PVs, VolumeAttachments, Longhorn Node CRs, Longhorn Volume CRs, DaemonSets, and pods.
+- Host command fake tests:
+  - SSH unavailable
+  - sudo unavailable
+  - sudo succeeds
+  - command timeout
+  - k3s-agent service states
+  - reboot accepted/not accepted
+- Vault fake tests:
+  - secret found
+  - secret missing
+  - Vault sealed/unavailable
+  - HTTP health fallback
+- End-to-end scenario tests composed from the incident:
+  - `titan-05` hard runtime wedge -> reboot -> Ready
+  - `titan-04` post-success sandbox wedge -> restart fails -> reboot -> Ready
+  - `titan-15`/`titan-17` cryptsetup/cordon flow -> sudo repair or explicit blocker -> uncordon when safe
+  - `titan-22` missing `longhorn-host` label -> restore label -> manager pod appears -> Longhorn Ready
+  - Firefly sidecar-only stale owner -> force delete -> attach replacement
+  - Firefly live app container stale owner -> wait/report unsafe, no force delete
+  - Vault exec killed but HTTP healthy -> continue
+  - DNS image pull errors -> classify without repeated recycle
+
+Run commands appropriate for the repo, likely including:
+
+```bash
+go test ./...
+go test -race ./...
+go vet ./...
+```
+
+If the repo uses other tooling, discover and run it. If any expensive/integration tests require cluster credentials, document how to run them and ensure fake-client tests cover the logic by default.
+
+## Acceptance Criteria
+
+The work is complete only when:
+
+- Ananke can retrieve and use Vault-backed sudo credentials securely or clearly reports why it cannot.
+- Node runtime wedges are detected and recovered by a bounded escalation ladder.
+- Ananke can handle Ready nodes that cannot create sandboxes.
+- Ananke can recover from k3s-agent stop-sigkill/timeout by controlled reboot when configured.
+- Ananke does not leave Longhorn nodes cordoned indefinitely after repair/override conditions are satisfied.
+- Ananke detects and fixes or reports Longhorn/Kubernetes drift, including missing manager-pod labels.
+- Ananke verifies Longhorn node readiness before trusting Kubernetes node readiness for PVC workloads.
+- Ananke handles stale RWO ownership safely, distinguishing sidecar-only stale pods from live PVC-writing app containers.
+- Ananke avoids noisy repeated deletion of unchanged image-pull failures.
+- Vault health checks tolerate transient exec failures when HTTP health is authoritative.
+- Startup success reflects real cluster recovery, with explicit `success_with_noncritical_blockers` or `blocked_requires_operator` states where appropriate.
+- All changes are covered by rigorous tests and pass the repo’s standard test suite.
+
+## What Not To Do
+
+- Do not hard-code Titan node names, app names, PVC names, or namespaces.
+- Do not log secrets.
+- Do not delete PVs, PVCs, or Longhorn Volume data.
+- Do not force-delete pods with live app containers mounting RWO PVCs unless a controlled, audited stop path has proven storage safety.
+- Do not repeatedly restart/reboot nodes without incident state and cooldowns.
+- Do not treat Kubernetes Ready as equivalent to Longhorn Ready.
+- Do not declare full success while node-runtime or unsafe stale-owner incidents are active.
+
+## Useful Terminology From The Incident
+
+- "Node runtime wedge": Kubernetes or host state where kubelet/containerd cannot create/kill sandboxes even if the node appears partially Ready.
+- "Stale RWO owner": a terminating pod or runtime survivor that still holds an exclusive PVC and blocks replacement attach.
+- "Sidecar-only stale owner": old pod still exists, but only non-PVC sidecars remain; force deletion may be safe after checks.
+- "Unsafe stale owner": app container still running and mounting the PVC; wait or controlled stop required.
+- "Longhorn/Kubernetes drift": Kubernetes node readiness/labels and Longhorn node readiness/manager state disagree.
+- "Host privilege unavailable": Ananke cannot run necessary sudo actions despite needing host-level repair.
+
+Use this prompt to implement production-grade, categorical recovery logic in Ananke, not one-off incident handling.

docs/ananke-recovery-2026-07-07.md

@@ -0,0 +1,207 @@
+# Ananke Recovery Notes - 2026-07-07
+
+## Context
+
+- Titan cluster lost power briefly; Ariadne may have started shutdown behavior.
+- User goal: let one Ananke recovery process bring the cluster back to health, monitor it, and record any manual help or automation gaps.
+- Operator view is from `/home/brad/Development/bstein_dev_home`; Kubernetes API is reachable through `/home/brad/titan/kubeconfig`.
+
+## Current Ananke Run
+
+- Coordinator host: `titan-db`.
+- `ananke.service` daemon is active.
+- `ananke-bootstrap.service` is active and running:
+  - command: `/usr/local/bin/ananke startup --config /etc/ananke/ananke.yaml --execute --force-flux-branch main --auto-peer-failover --peer-wait-seconds 180`
+  - current run started: `2026-07-07T17:19:50Z`
+  - current phase observed: `critical-workloads`
+  - checklist observed: `10 passed / 0 failed / 2 running`
+  - auto-heals observed: pod recycling for `VaultInitStuck`, `ContainerRuntimeWedge`, `CrashLoopBackOff`, stale terminating pods, and Longhorn attach blocked on unready nodes.
+  - previous run failed at `2026-07-07T17:19:19Z` waiting on `logging/deployment/oauth2-proxy-logs`; systemd restarted the bootstrap service.
+
+## Observed Recovery Progress
+
+- Latest snapshot at approximately `2026-07-07T17:58Z`:
+  - Ananke is still running in `convergence-checks` with `17/19` checks passed, `0` failed, and `2` running.
+  - Ananke has recorded `31` auto-heal actions, mostly repeated stuck-pod recycling.
+  - `titan-22` and `titan-24` both recovered to `Ready=True` with fresh kubelet transitions (`titan-22` at `2026-07-07T17:56:29Z`, `titan-24` at `2026-07-07T17:55:11Z`).
+  - Metrics server is running and `kubectl top nodes` works for recovered nodes; `titan-22` had just rejoined and metrics may lag briefly.
+  - Remaining convergence blocker is mostly app workload readiness, led by `finance/firefly`.
+- Ananke completed the startup workflow successfully at `2026-07-07T17:59:00Z`:
+  - `startup_status=success`
+  - `startup_phase=complete`
+  - `startup_checklist_total=20 passed=19 failed=0 running=1`
+  - `startup_auto_heals=33`
+  - `intent=normal`
+- Vault recovered:
+  - `vault/vault-0` is `1/1 Running` on `titan-18`.
+  - Vault status reports `initialized=true`, `sealed=false`, `ha_enabled=true`.
+  - Vault became active at `2026-07-07T17:05:46Z`.
+  - `vault-k8s-auth-config-autoheal-*` completed.
+- Vault injector recovered after `titan-04` kubelet recovery:
+  - `vault-injector-agent-injector-*` became `1/1 Running`.
+  - Vault service endpoints and injector webhook endpoints are populated.
+- Vault-backed workloads started moving again:
+  - Ariadne's Vault init authenticated successfully and rendered `/vault/secrets/ariadne-env.sh`.
+  - Ariadne is now `2/2 Running` on `titan-08`.
+- Flux is mostly reconciled, but several app kustomizations are still waiting on app readiness or dependencies.
+- Longhorn no longer shows the earlier broad faulted volume picture; observed state later was `41 attached`, `1 attaching`, `47 detached`.
+
+## Current Known Issues
+
+- `titan-22` and `titan-24` are still `NotReady` / `Unknown` from Kubernetes.
+- `titan-05` is `Unknown` after power-event recovery and k3s-agent restart attempts:
+  - Ananke SSH route on port `2277` reaches the host.
+  - `systemctl restart k3s-agent` timed out while stopping the old process.
+  - systemd reported `Processes still around after SIGKILL`, then spawned a new `/usr/local/bin/k3s agent`.
+  - `k3s-agent.service` remained `activating/start`; local `127.0.0.1:6444` was open but kubelet `127.0.0.1:10250` was closed.
+  - Kubernetes node heartbeat stayed stale at `2026-07-07T17:21:57Z` with `node.kubernetes.io/unreachable` taints.
+  - `maintenance/k3s-agent-restart-*` on `titan-05` stayed `ContainerCreating`, which confirms the in-cluster helper cannot repair this class once the node runtime is wedged.
+- `titan-15` and `titan-17` are `Ready,SchedulingDisabled`; Ananke cordoned them because encrypted Longhorn mounts exposed missing `cryptsetup`.
+- Ananke tried to install `cryptsetup-bin` on `titan-15` and `titan-17`, but the SSH repair failed because `sudo` required a password for `atlas`.
+- Several workloads are still unready while Ananke recycles stale `VaultInitStuck`, `CrashLoopBackOff`, and `ImagePullBackOff` pods.
+
+## Manual Actions Taken
+
+- Verified cluster health with `kubectl` and Flux commands.
+- Verified Vault directly with `vault status` inside `vault-0`.
+- Verified Ananke coordinator service state through `titan-db`.
+- Did not start a second Ananke recovery process after discovering the active `ananke-bootstrap.service`.
+- Used the existing `maintenance/k3s-agent-restart` DaemonSet as a targeted host-level nudge by deleting its pods on `titan-04`, `titan-05`, `titan-06`, `titan-08`, and `titan-18`; this reruns the DaemonSet's `k3s-agent` restart command on those nodes.
+- The in-cluster helper successfully reran on `titan-08`, but helper startup/termination was itself wedged on other affected nodes. Queued direct `systemctl --no-block restart k3s-agent` over Ananke SSH for `titan-05`, `titan-06`, and `titan-18`. A direct restart attempt against `titan-04` hung and was stopped.
+- After discovering Ananke's SSH profile uses `atlas@node:2277`, queued `systemctl --no-block restart k3s-agent` on `titan-04` and `titan-05` through the same route.
+- `titan-04` recovered to `Ready` after the non-blocking restart and its `maintenance/k3s-agent-restart-*` pod became `Running`.
+- `titan-05` did not recover: systemd timed out killing the old `k3s-agent`, left containerd shims behind, and started a new agent process without reopening kubelet port `10250` or restoring node heartbeats.
+- A controlled reboot was issued for `titan-05` through Ananke's SSH route at approximately `2026-07-07T17:32Z` because the node met the proposed escalation criteria: stale Kubernetes heartbeat, kubelet `10250` closed, `k3s-agent.service` stuck in `activating/start`, and the in-cluster helper stuck `ContainerCreating`.
+- After the reboot command returned, `titan-05` temporarily stopped responding to ping and SSH, then recovered to Kubernetes `Ready=True` with fresh heartbeats and no `unreachable` taints by approximately `2026-07-07T17:35Z`.
+- Ananke later detected and logged `cordoned container-runtime-wedged node(s): titan-07 pods=4`; direct host checks showed `titan-07` reachable, `k3s-agent` active/running, and kubelet `10250` open. This should become a false-positive/auto-uncordon test case if Kubernetes readiness remains healthy.
+- Later `titan-07` showed a real runtime/API-state split: `oauth2-proxy-logs` reported container state `running`, but `kubectl exec` failed with `cannot exec in a deleted state` and `kubectl logs --previous` could not retrieve the container logs. Manual assist: queued `systemctl --no-block restart k3s-agent` on `titan-07` and deleted the stale `logging/oauth2-proxy-logs-*` pod.
+- `titan-07` recovered to `Ready=True` with `k3s-agent` active and kubelet `10250` open. `logging/oauth2-proxy-logs` was recreated on `titan-12` and became `2/2 Running`.
+- Ananke repeatedly failed and restarted on `logging/deployment/oauth2-proxy-logs` because the replacement pod on `titan-14` hit the same `failed to reserve container name` / `CreateContainerError` runtime wedge signature.
+- Manual assist: queued `systemctl --no-block restart k3s-agent` on `titan-14` through Ananke SSH and deleted the stuck `logging/oauth2-proxy-logs-5b4d4f87db-2tdzd` pod with `--wait=false` so the controller can recreate it after the runtime reset.
+- Follow-up on `titan-14`: the direct SSH command timed out from the operator view, but a later status check showed `k3s-agent.service` had a new MainPID and was `activating/start`; the existing `maintenance/k3s-agent-restart` DaemonSet pod for `titan-14` was also deleted to rerun the in-cluster helper.
+- `titan-14` recovered to fresh `Ready=True` heartbeats and the replacement `maintenance/k3s-agent-restart-*` pod became `Running`.
+- After the `titan-05` reboot, `logging/opensearch-0` recovered to `1/1 Running`; `logging/oauth2-proxy-logs` was recreated on `titan-07` and was still initializing at the time of note.
+- Ananke also entered a short retry loop where `kubectl exec vault-0 ... vault status` was killed even though Vault HTTP health through the Kubernetes pod proxy reported `initialized=true`, `sealed=false`, `standby=false`. This should become a separate robustness fix: startup should not fail outright on a transient killed `kubectl exec` when service endpoints/HTTP health prove Vault is active.
+- After Keycloak completed its slow post-attach startup at approximately `2026-07-07T17:45:55Z`, OIDC-dependent OAuth proxies began recovering.
+- Ananke advanced past `critical-workloads` into `convergence-checks` by approximately `2026-07-07T17:50:50Z`.
+- Manual assist: force-deleted stale pod `finance/firefly-6669f46874-q8rtq` after Ananke repeatedly issued ordinary deletes but the pod remained `Terminating` with no finalizers and continued to block `firefly-storage` attachment. The Firefly Longhorn volume moved from `titan-04` to `titan-06` and reported `attached`/`healthy` on the replacement node.
+- Post-Ananke-completion manual assist: `titan-04` was still `Ready` with kubelet `10250` open, but several newly scheduled pods showed `FailedCreatePodSandBox`, `failed to reserve sandbox name`, and `context deadline exceeded`. Queued `systemctl --no-block restart k3s-agent` on `titan-04` at approximately `2026-07-07T18:02:55Z`.
+- `titan-04` did not recover from the non-blocking `k3s-agent` restart. It moved to Kubernetes `Ready=Unknown` with `node.kubernetes.io/unreachable` taints; host checks showed `k3s-agent.service` stuck in `deactivating/stop-sigkill`, `Result=timeout`, and kubelet `10250` closed. Manual assist: issued a controlled reboot for `titan-04` through Ananke's SSH route shortly after `2026-07-07T18:06Z`.
+- `titan-04` recovered after reboot by approximately `2026-07-07T18:10Z`: Kubernetes reported `Ready=True`, taints cleared, `k3s-agent` was `active/running`, and kubelet `10250` was open.
+- Firefly hit a second stale-owner handoff after the first cleanup. Replacement pod `firefly-6669f46874-pjhg6` on `titan-07` was blocked while terminating pod `firefly-6669f46874-lrz5d` on `titan-06` still had the main `firefly` container running and mounted `firefly-storage`. No force delete was issued for this second handoff because the application container was live and PVC-writing. The old pod later disappeared on its own; Longhorn then attached `pvc-358b4319-60cb-4322-b948-776c34a414a9` to `titan-07` as `attached`/`healthy`.
+- Attempted to inspect container runtime state on `titan-06` with `crictl`, but `atlas` required a sudo password. This should be part of the automation fix: Ananke either needs a noninteractive, bounded runtime-inspection primitive or must report that it cannot prove a PVC-writing container is stopped.
+- Manual assist: uncordoned `titan-15` and `titan-17` after confirming Kubernetes `Ready=True`, Longhorn `allowScheduling=true`, and Longhorn disks ready. The remaining blocker for Longhorn on both nodes was Kubernetes `SchedulingDisabled`; after uncordon, Longhorn `Schedulable=True` on both nodes.
+- Manual assist: resolved the `titan-22` Kubernetes/Longhorn readiness discrepancy. Kubernetes reported `Ready=True`, but Longhorn node `Ready=False` with `ManagerPodMissing`; the Longhorn manager DaemonSet selected `longhorn-host=true`, and `titan-22` was missing that label. Restored `longhorn-host=true`, which started `longhorn-manager-jtsjw` on `titan-22` and brought Longhorn node `Ready=True`.
+- Manual assist: deleted pending controller-owned pod `crypto/wallet-monero-temp-844bd949c7-28r4p` to force a fresh volume-attach retry after the `titan-22` Longhorn manager came up.
+- Manual assist: deleted stale unattached VolumeAttachment `csi-0bac99e18b687f6a50647dd959c26d6e2a1b2bf1a4f844de4f646cf3408d41c5` after it retained the pre-fix "node titan-22 is not ready" Longhorn error. Longhorn then attached `pvc-c6a4bf9b-17c7-4bdf-8a88-ca460ad0da5b` to `titan-22` as `attached`/`healthy`.
+
+## Automation Growth Opportunities
+
+- Ananke should expose from a remote operator machine whether the coordinator bootstrap is already running, so an operator does not risk starting a duplicate recovery.
+- The local Ananke config path assumes `/etc/ananke/ananke.yaml` and `/var/lib/ananke`; operator-side `status` failed without root. A read-only remote status wrapper would make this smoother.
+- Ananke should handle the `cryptsetup-bin` host repair on managed nodes without requiring interactive sudo, or clearly preflight and report the missing node privilege before a power event.
+- Ananke should separate `Vault pod/data volume unavailable` from `Vault sealed` in status. It did recover correctly, but early status said the unseal check passed while deferring because `vault-0` was pending.
+- Ananke should avoid repeatedly recycling the same ImagePullBackOff pods if the underlying registry/image problem is unchanged; this creates noisy recovery logs.
+- Ananke should surface current blocking workloads in a compact operator summary rather than requiring journal inspection.
+- Ananke should detect pods stuck in `PodInitializing` after init completion, especially when events show `FailedCreatePodSandbox`, `failed to reserve sandbox name`, `FailedKillPod`, or image pulls that never transition to `ImagePullBackOff`.
+- Ananke should be able to rerun the existing `k3s-agent-restart` maintenance primitive on affected nodes when containerd/kubelet sandbox state is wedged after a power event.
+- Ananke should fall back from the in-cluster restart helper to SSH `systemctl --no-block restart k3s-agent` when the helper pod cannot start because kubelet/containerd is already wedged. Blocking `systemctl restart` over SSH can hang the operator path.
+- Ananke should check host and CoreDNS upstream DNS health separately from node SSH reachability. During this incident, node IP reachability passed while hostname resolution and image pulls were intermittently failing.
+- Ananke should stop repeatedly issuing ordinary pod deletes when the same controller pod remains `Terminating` and continues to own exclusive resources. It should escalate to a stale-owner cleanup flow that proves the old workload is not writing, force-removes the orphaned API object when safe, then verifies Longhorn detach/attach and replacement pod startup.
+- Ananke should reconcile Kubernetes node labels against Longhorn manager scheduling requirements. During this incident `titan-22` was Kubernetes `Ready=True`, had CSI/engine Longhorn pods, and had a Longhorn Node CR, but no `longhorn-manager` pod because `longhorn-host=true` was missing. Kubernetes Ready alone was not sufficient for PVC workloads.
+- Ananke should understand that Kubernetes cordon directly affects Longhorn node schedulability. `titan-15` and `titan-17` had Longhorn-ready disks and `allowScheduling=true`, but Longhorn `Schedulable=False` solely because the Kubernetes nodes were cordoned.
+
+### Generalized Node Runtime Wedge Case
+
+This incident exposed a reusable failure mode that Ananke should treat as a node-runtime recovery problem, not just an app-pod recovery problem.
+
+Detection signals:
+
+- Node is `Ready=Unknown` or has a stale heartbeat while SSH on the managed Ananke route still works.
+- Host is pingable and local k3s load balancer port `127.0.0.1:6444` may be open, but kubelet port `127.0.0.1:10250` is closed or API proxy calls fail.
+- `systemctl restart k3s-agent` hangs, or `systemctl show k3s-agent` reports `deactivating`, `final-sigterm`, `final-sigkill`, `activating/start`, or `Result=timeout` for longer than a bounded recovery window.
+- `journalctl -u k3s-agent` includes `Processes still around after SIGKILL`, `State 'stop-sigterm' timed out`, or `State 'final-sigterm' timed out`.
+- Pods on the node show `FailedCreatePodSandBox`, `failed to reserve sandbox name`, `CreateContainerError`, `FailedKillPod`, or long-lived `ContainerCreating`/`PodInitializing`.
+- The `maintenance/k3s-agent-restart` DaemonSet pod for that node cannot start or stays `ContainerCreating`, which means Kubernetes-based repair is unavailable.
+
+Suggested automated escalation ladder:
+
+- Mark the node as in node-runtime recovery and pause noisy repeated pod recycling for pods pinned to that node.
+- Cordon the node or confirm it already has `unreachable` taints before host-level repair.
+- Try a bounded SSH health probe using Ananke's configured SSH user, port, config, and identity.
+- Prefer `systemctl --no-block restart k3s-agent` over blocking `systemctl restart k3s-agent`.
+- After a short wait, require both Kubernetes heartbeat freshness and local kubelet port `10250` to recover.
+- If the service remains `deactivating`, `final-sigkill`, or `activating/start` with stale heartbeats, escalate to a configured host reboot path.
+- After reboot, wait for SSH, k3s-agent active/running, kubelet `10250`, Kubernetes `Ready=True`, and removal of `unreachable` taints before resuming app-pod recycling for that node.
+
+Implementation notes:
+
+- This should be a generic managed-node repair primitive, not hard-coded to `titan-05`.
+- The primitive should emit one compact operator summary: node, phase, trigger signals, action taken, elapsed time, and next escalation.
+- Tests should cover a node with SSH available but kubelet closed, a blocking restart timeout, leftover shim evidence, a helper DaemonSet stuck in `ContainerCreating`, and successful recovery through reboot.
+- The post-completion `titan-04` recurrence confirms the primitive must continue monitoring after Ananke declares cluster startup success. A Ready node with fresh-enough status can still be unable to create new sandboxes, and a later restart can transition it into the same hard `deactivating/stop-sigkill` state seen on `titan-05`.
+
+### Generalized Stale Pod / Volume Ownership Case
+
+This incident also exposed a reusable app-convergence failure mode that Ananke should treat as stale Kubernetes/runtime ownership rather than simple pod unhealthiness.
+
+Concrete example:
+
+- `finance/firefly-6669f46874-q8rtq` entered deletion at `2026-07-07T17:43:55Z` and remained `Terminating` for more than 25 minutes.
+- The pod had no finalizers but still reported a running `vault-agent` sidecar on `titan-04`.
+- The replacement pod `finance/firefly-6669f46874-lrz5d` on `titan-06` stayed `Init:0/2`.
+- Events showed `FailedAttachVolume` / `Multi-Attach` for PVC `firefly-storage` because volume `pvc-358b4319-60cb-4322-b948-776c34a414a9` was still attached to `titan-04`.
+- Longhorn reported the volume `attached`, `healthy`, current node `titan-04`.
+- Ananke repeatedly logged ordinary recycling of the same stuck pod, but did not escalate to clearing the stale owner.
+- Manual force deletion of the stale pod cleared the stale API owner and allowed Longhorn to attach `firefly-storage` to the replacement pod's node.
+
+Detection signals:
+
+- A controller-owned pod has `metadata.deletionTimestamp` older than a bounded threshold and no finalizers.
+- A replacement pod from the same controller is pending or initializing on another node.
+- Events include `Multi-Attach`, `FailedAttachVolume`, `Volume is already used by pod(s)`, or `Volume is already exclusively attached to one node`.
+- The PVC is `ReadWriteOnce` and the old and new pod nodes differ.
+- Longhorn/CNI/runtime state still shows the old node as the attachment owner.
+- Ordinary `kubectl delete pod --wait=false` is repeated without changing the deletion timestamp or clearing the volume attachment.
+
+Suggested automated escalation ladder:
+
+- Group the old pod, replacement pod, PVC, VolumeAttachment, Longhorn volume, controller, and node into one recovery incident.
+- Verify whether any application container that mounts the PVC is still running. Sidecars that do not mount the PVC should not block stale-owner cleanup by themselves.
+- Check node health for the old owner node. If kubelet/runtime is unhealthy, route through the managed-node recovery primitive first.
+- If the old owner node is healthy but the pod remains deleting past the threshold, issue a bounded force deletion for the stale pod.
+- Wait for VolumeAttachment/Longhorn ownership to detach from the old node or attach to the replacement node.
+- Recheck the replacement pod init sequence and controller readiness before declaring the app recovered.
+- Emit one compact operator event instead of repeatedly logging the same pod recycle.
+
+Implementation notes:
+
+- This should be generic across any `ReadWriteOnce` PVC workload, not hard-coded to Firefly.
+- The safety check should reason at container/mount level: a sidecar that only mounts service-account, Vault secret, or emptyDir volumes is materially different from an app container actively mounting the PVC.
+- Tests should cover a terminating pod without finalizers holding an RWO PVC, an old node with healthy kubelet, an old node with wedged kubelet, a sidecar-only survivor, and a replacement pod blocked by `Multi-Attach`.
+- Tests should also cover the unsafe variant where the terminating pod's application container is still running and mounts the PVC. In that case Ananke should wait, stop the container through a controlled runtime path if allowed, or report that it lacks the privilege to prove storage safety; it should not blindly force-delete the API object.
+
+### Generalized Longhorn / Kubernetes Readiness Drift Case
+
+This incident exposed a reusable storage readiness gap: Kubernetes `Ready=True` is not enough for PVC workloads when Longhorn's node model disagrees.
+
+Detection signals:
+
+- Kubernetes node is `Ready=True`, untainted, and schedulable, but Longhorn Node `Ready=False`.
+- Longhorn Node reason is `ManagerPodMissing`, or the Longhorn manager DaemonSet has no pod on the node.
+- Longhorn manager DaemonSet selects a node label such as `longhorn-host=true`, and the Kubernetes node is missing it.
+- Longhorn CSI/engine pods or a Longhorn Node CR exist for the node, proving the node participates in Longhorn even though manager scheduling is broken.
+- VolumeAttachment errors say Longhorn cannot attach because the node is not ready, despite Kubernetes Ready=True.
+
+Suggested automated escalation ladder:
+
+- Compare Kubernetes node readiness, labels, taints, Longhorn Node readiness/schedulability, Longhorn manager DaemonSet selector, manager pod presence, CSI pod presence, and VolumeAttachment errors.
+- If desired-state evidence proves the node should be a Longhorn node, restore the missing manager selector label and wait for the manager pod and Longhorn Node `Ready=True`.
+- If desired-state evidence is missing, report a clear unsafe label-drift blocker rather than guessing.
+- After Longhorn Node `Ready=True`, retry stale VolumeAttachments by recreating the affected controller-owned pending pod or using a safe VolumeAttachment retry policy.
+- For Kubernetes-cordoned Longhorn nodes with `allowScheduling=true` and ready disks, either complete the host repair and uncordon or report the explicit blocker; do not leave Longhorn capacity disabled silently.
+
+Implementation notes:
+
+- This must be generic across Longhorn nodes and manager DaemonSet selectors, not hard-coded to `titan-22` or `longhorn-host`.
+- Tests should cover missing Longhorn selector labels, manager pod missing, Kubernetes Ready but Longhorn NotReady, Kubernetes cordon causing Longhorn Schedulable=False, and stale attach errors that clear after Longhorn readiness is restored.

internal/cluster/orchestrator_access_fluxsource.go

@@ -38,8 +38,8 @@ func (o *Orchestrator) reconcileNodeAccess(ctx context.Context, nodes []string)
 			defer wg.Done()
 			sem <- struct{}{}
 			defer func() { <-sem }()
-			if _, err := o.ssh(ctx, node, "sudo -n /usr/bin/systemctl --version"); err != nil {
+			if err := o.preflightHostPrivilege(ctx, node); err != nil {
-				errCh <- fmt.Errorf("%s: missing sudo access to /usr/bin/systemctl (--version): %w", node, err)
+				errCh <- fmt.Errorf("%s: host privilege preflight failed: %w", node, err)
 			}
 		}()
 	}
@@ -49,13 +49,16 @@ func (o *Orchestrator) reconcileNodeAccess(ctx context.Context, nodes []string)
 		return nil
 	}
 	samples := []string{}
+	errCount := 0
 	for err := range errCh {
+		errCount++
 		samples = append(samples, err.Error())
 		if len(samples) >= 4 {
 			break
 		}
 	}
-	return fmt.Errorf("access validation had %d errors (first: %s)", len(errCh), strings.Join(samples, " | "))
+	errCount += len(errCh)
+	return fmt.Errorf("access validation had %d errors (first: %s)", errCount, strings.Join(samples, " | "))
 }
 
 // waitForNodeSSHAuth runs one orchestration or CLI step.

internal/cluster/orchestrator_autorepair.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 )
@@ -17,6 +18,7 @@ type nodeReadyItem struct {
 	Metadata struct {
 		Name        string            `json:"name"`
 		Annotations map[string]string `json:"annotations"`
+		Labels      map[string]string `json:"labels"`
 	} `json:"metadata"`
 	Spec struct {
 		Unschedulable bool `json:"unschedulable"`
@@ -75,6 +77,14 @@ func (o *Orchestrator) postStartAutoHeal(ctx context.Context) error {
 		errs = append(errs, fmt.Sprintf("required node labels: %v", err))
 	}
 
+	longhornRepairs, err := o.reconcileLonghornKubernetesReadiness(ctx)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("longhorn/kubernetes readiness drift: %v", err))
+	} else if longhornRepairs > 0 {
+		requestReconcile = true
+		o.log.Printf("post-start auto-heal repaired %d Longhorn/Kubernetes readiness drift item(s)", longhornRepairs)
+	}
+
 	releasedCordons, err := o.enforceRecoveryCordonLeases(ctx)
 	if err != nil {
 		errs = append(errs, fmt.Sprintf("recovery cordon lease check: %v", err))
@@ -99,6 +109,15 @@ func (o *Orchestrator) postStartAutoHeal(ctx context.Context) error {
 		requestReconcile = true
 	}
 
+	serviceRepairs, err := o.healUnreadyConfiguredServiceBackends(ctx)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("configured service backend repair: %v", err))
+	} else if len(serviceRepairs) > 0 {
+		requestReconcile = true
+		sort.Strings(serviceRepairs)
+		o.log.Printf("post-start auto-heal repaired configured service backend(s): %s", joinLimited(serviceRepairs, 8))
+	}
+
 	repairedProxies, err := o.repairBrokenKubeletProxies(ctx)
 	if err != nil {
 		errs = append(errs, fmt.Sprintf("kubelet proxy auto-repair: %v", err))
@@ -265,38 +284,10 @@ func (o *Orchestrator) repairBrokenKubeletProxies(ctx context.Context) (int, err
 			continue
 		}
 
-		if !node.Unschedulable {
+		if err := o.recoverManagedNodeRuntime(ctx, node.Name, node.Unschedulable, "broken kubelet proxy before k3s-agent restart"); err != nil {
-			if err := o.cordonNodeWithLease(ctx, node.Name, cordonReasonKubeletProxy, "broken kubelet proxy before k3s-agent restart"); err != nil {
+			errs = append(errs, fmt.Sprintf("%s %v", node.Name, err))
-				errs = append(errs, fmt.Sprintf("%s cordon before kubelet restart: %v", node.Name, err))
 			continue
 		}
-		}
-
-		o.log.Printf("warning: detected broken kubelet proxy on Ready node %s; restarting k3s-agent", node.Name)
-		if _, err := o.sshWithTimeout(ctx, node.Name, "sudo -n systemctl restart k3s-agent", 90*time.Second); err != nil {
-			if !node.Unschedulable {
-				o.bestEffort("uncordon node after failed kubelet proxy repair", func() error {
-					return o.uncordonAndClearCordonLease(ctx, node.Name, cordonReasonKubeletProxy)
-				})
-			}
-			errs = append(errs, fmt.Sprintf("%s restart k3s-agent: %v", node.Name, err))
-			continue
-		}
-		if _, err := o.kubectl(ctx, 140*time.Second, "wait", "node/"+node.Name, "--for=condition=Ready", "--timeout=120s"); err != nil {
-			errs = append(errs, fmt.Sprintf("%s wait Ready after k3s-agent restart: %v", node.Name, err))
-			continue
-		}
-		healthy, checkErr = o.kubeletProxyHealthy(ctx, node.Name)
-		if !healthy {
-			errs = append(errs, fmt.Sprintf("%s proxy still broken after k3s-agent restart: %v", node.Name, checkErr))
-			continue
-		}
-		if !node.Unschedulable {
-			if err := o.uncordonAndClearCordonLease(ctx, node.Name, cordonReasonKubeletProxy); err != nil {
-				errs = append(errs, fmt.Sprintf("%s uncordon after kubelet proxy repair: %v", node.Name, err))
-				continue
-			}
-		}
 		repaired++
 	}
 

internal/cluster/orchestrator_autorepair_proxy_test.go

@@ -41,9 +41,11 @@ func TestRepairBrokenKubeletProxiesRestartsSchedulableNode(t *testing.T) {
 		case name == "kubectl" && strings.Contains(joined, "cordon titan-07"):
 			cordoned = true
 			return "", nil
-		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl restart k3s-agent"):
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl --no-block restart k3s-agent"):
 			restarted = true
 			return "", nil
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl is-active k3s-agent"):
+			return "active", nil
 		case name == "kubectl" && strings.Contains(joined, "wait node/titan-07 --for=condition=Ready --timeout=120s"):
 			waited = true
 			return "", nil
@@ -100,9 +102,11 @@ func TestRepairBrokenKubeletProxiesPreservesExistingCordon(t *testing.T) {
 		case name == "kubectl" && (strings.Contains(joined, "cordon titan-18") || strings.Contains(joined, "uncordon titan-18")):
 			cordonTouched = true
 			return "", nil
-		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl restart k3s-agent"):
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl --no-block restart k3s-agent"):
 			restarted = true
 			return "", nil
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl is-active k3s-agent"):
+			return "active", nil
 		case name == "kubectl" && strings.Contains(joined, "wait node/titan-18 --for=condition=Ready --timeout=120s"):
 			return "", nil
 		default:
@@ -243,6 +247,7 @@ func TestRepairBrokenKubeletProxiesFailureBranches(t *testing.T) {
 			{cmd: "restart"},
 			{cmd: "wait"},
 			{cmd: "health", out: "ok"},
+			{cmd: "active", out: "active"},
 			{cmd: "uncordon", err: errors.New("uncordon denied")},
 		})
 		repaired, err := orch.repairBrokenKubeletProxies(context.Background())
@@ -280,8 +285,10 @@ func kubeletProxyRepairStub(t *testing.T, cfg config.Config, unschedulable bool,
 			actual = "uncordon"
 		case name == "kubectl" && strings.Contains(joined, "cordon titan-07"):
 			actual = "cordon"
-		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl restart k3s-agent"):
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl --no-block restart k3s-agent"):
 			actual = "restart"
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl is-active k3s-agent"):
+			actual = "active"
 		case name == "kubectl" && strings.Contains(joined, "wait node/titan-07 --for=condition=Ready --timeout=120s"):
 			actual = "wait"
 		default:

internal/cluster/orchestrator_autorepair_service_test.go

@@ -0,0 +1,64 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestConfiguredServiceBackendRepairContinuesPastNoopEndpoint runs one orchestration or CLI step.
+// Signature: TestConfiguredServiceBackendRepairContinuesPastNoopEndpoint(t *testing.T).
+// Why: one unhealthy endpoint without a matching workload must not prevent later
+// critical services, especially Mailu, from getting their backend repair pass.
+func TestConfiguredServiceBackendRepairContinuesPastNoopEndpoint(t *testing.T) {
+	cfg := config.Config{
+		Startup: config.Startup{
+			CriticalServiceEndpoints: []string{
+				"orphan/missing-service",
+				"mailu-mailserver/mailu-admin",
+			},
+		},
+	}
+	orch := buildOrchestratorWithStubs(t, cfg, nil)
+
+	scaledMail := false
+	rolledOutMail := false
+	dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		if name != "kubectl" {
+			return "", nil
+		}
+		joined := strings.Join(args, " ")
+		switch {
+		case strings.Contains(joined, "-n orphan get endpoints missing-service"):
+			return "", nil
+		case strings.Contains(joined, "-n orphan scale deployment missing-service --replicas=1"):
+			return "", errors.New("not found")
+		case strings.Contains(joined, "-n orphan scale statefulset missing-service --replicas=1"):
+			return "", errors.New("not found")
+		case strings.Contains(joined, "-n mailu-mailserver get endpoints mailu-admin"):
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver scale deployment mailu-admin --replicas=1"):
+			scaledMail = true
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver rollout status deployment/mailu-admin"):
+			rolledOutMail = true
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver scale statefulset mailu-admin --replicas=1"):
+			return "", errors.New("not found")
+		default:
+			return "", nil
+		}
+	}
+	orch.SetCommandOverrides(dispatch, dispatch)
+
+	if _, err := orch.healUnreadyConfiguredServiceBackends(context.Background()); err != nil {
+		t.Fatalf("healUnreadyConfiguredServiceBackends failed: %v", err)
+	}
+	if !scaledMail || !rolledOutMail {
+		t.Fatalf("expected later mail endpoint to be repaired, scaled=%v rolledOut=%v", scaledMail, rolledOutMail)
+	}
+}

internal/cluster/orchestrator_autorepair_test.go

@@ -177,6 +177,77 @@ func TestPostStartAutoHealSkipsWhenClusterIsHealthy(t *testing.T) {
 	}
 }
 
+// TestPostStartAutoHealRepairsConfiguredMailBackends runs one orchestration or CLI step.
+// Signature: TestPostStartAutoHealRepairsConfiguredMailBackends(t *testing.T).
+// Why: Mailu readiness problems should be handled by Ananke's daemon repair
+// loop after startup, not only by a fresh bootstrap run.
+func TestPostStartAutoHealRepairsConfiguredMailBackends(t *testing.T) {
+	cfg := config.Config{
+		Startup: config.Startup{
+			DeadNodeCleanupGraceSeconds:     300,
+			CriticalServiceEndpoints:        []string{"mailu-mailserver/mailu-admin"},
+			NodeRuntimeRestartWaitSeconds:   1,
+			NodeRuntimeRebootWaitSeconds:    1,
+			HostPrivilegedCommandTimeoutSec: 1,
+		},
+	}
+	orch := buildOrchestratorWithStubs(t, cfg, nil)
+
+	scaled := false
+	rolledOut := false
+	reconciled := false
+	dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		if name != "kubectl" {
+			return "", nil
+		}
+		joined := strings.Join(args, " ")
+		switch {
+		case strings.Contains(joined, "-n longhorn-system get nodes.longhorn.io -o json"):
+			return `{"items":[]}`, nil
+		case strings.Contains(joined, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
+			return "", nil
+		case strings.Contains(joined, "get nodes -o json"):
+			return `{"items":[]}`, nil
+		case strings.Contains(joined, "get pods -A -o json"):
+			return `{"items":[]}`, nil
+		case strings.Contains(joined, "get events -A -o json"):
+			return `{"items":[]}`, nil
+		case strings.Contains(joined, "get pvc -A -o json"):
+			return `{"items":[]}`, nil
+		case strings.Contains(joined, "-n mailu-mailserver get endpoints mailu-admin"):
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver scale deployment mailu-admin --replicas=1"):
+			scaled = true
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver rollout status deployment/mailu-admin"):
+			rolledOut = true
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver scale statefulset mailu-admin --replicas=1"):
+			return "", errors.New("not found")
+		case strings.Contains(joined, "-n flux-system annotate gitrepository flux-system reconcile.fluxcd.io/requestedAt="):
+			reconciled = true
+			return "", nil
+		case strings.Contains(joined, "-n flux-system annotate kustomizations.kustomize.toolkit.fluxcd.io --all reconcile.fluxcd.io/requestedAt="):
+			return "", nil
+		case strings.Contains(joined, "annotate --all-namespaces helmreleases.helm.toolkit.fluxcd.io --all reconcile.fluxcd.io/requestedAt="):
+			return "", nil
+		default:
+			return "", nil
+		}
+	}
+	orch.SetCommandOverrides(dispatch, dispatch)
+
+	if err := orch.postStartAutoHeal(context.Background()); err != nil {
+		t.Fatalf("postStartAutoHeal failed: %v", err)
+	}
+	if !scaled || !rolledOut {
+		t.Fatalf("expected mailu-admin deployment to be scaled and rolled out, scaled=%v rolledOut=%v", scaled, rolledOut)
+	}
+	if !reconciled {
+		t.Fatalf("expected flux reconcile after mail backend repair")
+	}
+}
+
 // TestRunPostStartAutoHealDryRun runs one orchestration or CLI step.
 // Signature: TestRunPostStartAutoHealDryRun(t *testing.T).
 // Why: covers the exported wrapper and the top-level dry-run guard so daemon

internal/cluster/orchestrator_core.go

@@ -20,6 +20,7 @@ type Orchestrator struct {
 	log                  *log.Logger
 	runOverride          func(timeoutCtx context.Context, timeout time.Duration, name string, args ...string) (string, error)
 	runSensitiveOverride func(timeoutCtx context.Context, timeout time.Duration, name string, args ...string) (string, error)
+	sshInputOverride     func(timeoutCtx context.Context, timeout time.Duration, node string, command string, input string) (string, error)
 	startupReportMu      sync.Mutex
 	activeStartupReport  *startupReport
 }
@@ -122,3 +123,11 @@ func (o *Orchestrator) SetCommandOverrides(run commandOverrideFunc, runSensitive
 	o.runOverride = run
 	o.runSensitiveOverride = runSensitive
 }
+
+// SetSSHInputOverride runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) SetSSHInputOverride(run func(context.Context, time.Duration, string, string, string) (string, error)).
+// Why: password-backed host-repair tests need to assert stdin behavior without
+// exposing secrets through ordinary command-argument stubs.
+func (o *Orchestrator) SetSSHInputOverride(run func(context.Context, time.Duration, string, string, string) (string, error)) {
+	o.sshInputOverride = run
+}

internal/cluster/orchestrator_critical_endpoints.go

@@ -2,12 +2,22 @@ package cluster
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"sort"
 	"strings"
 	"time"
 )
 
+type endpointResource struct {
+	Subsets []struct {
+		Addresses []struct {
+			IP string `json:"ip"`
+		} `json:"addresses"`
+	} `json:"subsets"`
+}
+
 // waitForCriticalServiceEndpoints runs one orchestration or CLI step.
 // Signature: (o *Orchestrator) waitForCriticalServiceEndpoints(ctx context.Context) error.
 // Why: some externally-healthy services (like Grafana) still require backend
@@ -25,8 +35,10 @@ func (o *Orchestrator) waitForCriticalServiceEndpoints(ctx context.Context) erro
 	lastFailure := "unknown"
 	lastLogged := time.Time{}
 	lastHealAttempt := time.Time{}
+	lastRecycleAttempt := time.Time{}
 
 	for {
+		o.maybeAutoRecycleStuckPods(ctx, &lastRecycleAttempt)
 		ready, detail, failedNamespace, failedService, err := o.criticalServiceEndpointsReady(ctx)
 		if err != nil {
 			lastFailure = err.Error()
@@ -44,6 +56,16 @@ func (o *Orchestrator) waitForCriticalServiceEndpoints(ctx context.Context) erro
 			lastHealAttempt = now
 			healed, healErr := o.maybeHealCriticalEndpointBackends(ctx, failedNamespace, failedService)
 			if healErr != nil {
+				probeRepairs, probeErr := o.maybeRepairCriticalBackendStartupProbes(ctx, failedNamespace, failedService)
+				if probeErr != nil {
+					o.log.Printf("warning: critical endpoint startup-probe repair failed for %s/%s: %v", failedNamespace, failedService, probeErr)
+				}
+				if len(probeRepairs) > 0 {
+					sort.Strings(probeRepairs)
+					repairDetail := fmt.Sprintf("repaired critical endpoint startup probes: %s", joinLimited(probeRepairs, 8))
+					o.log.Printf("%s", repairDetail)
+					o.noteStartupAutoHeal(repairDetail)
+				}
 				o.log.Printf("warning: critical endpoint backend auto-heal failed for %s/%s: %v", failedNamespace, failedService, healErr)
 			}
 			if len(healed) > 0 {
@@ -120,7 +142,7 @@ func (o *Orchestrator) maybeHealCriticalEndpointBackends(ctx context.Context, na
 			}
 			return healed, fmt.Errorf("scale %s/%s/%s to 1: %w", namespace, kind, service, err)
 		}
-		if err := o.waitWorkloadReady(ctx, workload); err != nil {
+		if err := o.workloadRolloutReadyOnce(ctx, workload); err != nil {
 			if !isNotFoundErr(err) {
 				return healed, err
 			}
@@ -131,6 +153,83 @@ func (o *Orchestrator) maybeHealCriticalEndpointBackends(ctx context.Context, na
 	return healed, nil
 }
 
+// workloadRolloutReadyOnce runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) workloadRolloutReadyOnce(ctx context.Context, w startupWorkload) error.
+// Why: endpoint backend healing should issue one bounded readiness nudge and
+// let the outer endpoint wait enforce the configured convergence deadline.
+func (o *Orchestrator) workloadRolloutReadyOnce(ctx context.Context, w startupWorkload) error {
+	_, err := o.kubectl(
+		ctx,
+		45*time.Second,
+		"-n",
+		w.Namespace,
+		"rollout",
+		"status",
+		fmt.Sprintf("%s/%s", w.Kind, w.Name),
+		"--timeout=30s",
+	)
+	return err
+}
+
+// healUnreadyConfiguredServiceBackends runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) healUnreadyConfiguredServiceBackends(ctx context.Context) ([]string, error).
+// Why: the daemon's post-start loop should be able to repair configured service
+// backends, including Mailu, without rerunning the whole startup workflow.
+func (o *Orchestrator) healUnreadyConfiguredServiceBackends(ctx context.Context) ([]string, error) {
+	healed := []string{}
+	if len(o.cfg.Startup.CriticalServiceEndpoints) > 0 {
+		errs := []string{}
+		for _, entry := range o.cfg.Startup.CriticalServiceEndpoints {
+			namespace, service, err := parseCriticalServiceEndpoint(entry)
+			if err != nil {
+				errs = append(errs, err.Error())
+				continue
+			}
+			count, err := o.endpointAddressCount(ctx, namespace, service)
+			if err != nil && !isNotFoundErr(err) {
+				errs = append(errs, fmt.Sprintf("query endpoints %s/%s: %v", namespace, service, err))
+				continue
+			}
+			if err == nil && count > 0 {
+				continue
+			}
+			detail := fmt.Sprintf("%s/%s endpoints=0", namespace, service)
+			if err != nil {
+				detail = fmt.Sprintf("%s/%s not found", namespace, service)
+			}
+			o.log.Printf("warning: configured critical endpoint unhealthy (%s); attempting backend repair", detail)
+			items, healErr := o.maybeHealCriticalEndpointBackends(ctx, namespace, service)
+			healed = append(healed, items...)
+			if healErr != nil {
+				probeItems, probeErr := o.maybeRepairCriticalBackendStartupProbes(ctx, namespace, service)
+				healed = append(healed, probeItems...)
+				if probeErr != nil {
+					errs = append(errs, fmt.Sprintf("heal critical endpoint %s/%s (%s): %v; startup-probe repair: %v", namespace, service, detail, healErr, probeErr))
+					continue
+				}
+				if o.cfg.Startup.AutoRecycleStuckPods {
+					o.bestEffort("recycle stuck pods after configured service backend repair failure", func() error {
+						return o.recycleStuckControllerPods(ctx)
+					})
+				}
+				if len(probeItems) == 0 {
+					errs = append(errs, fmt.Sprintf("heal critical endpoint %s/%s (%s): %v", namespace, service, detail, healErr))
+				}
+			}
+		}
+		if len(errs) > 0 {
+			return healed, errors.New(strings.Join(errs, "; "))
+		}
+	}
+
+	tcpHealed, err := o.healFailedTCPServiceBackends(ctx)
+	healed = append(healed, tcpHealed...)
+	if err != nil {
+		return healed, err
+	}
+	return healed, nil
+}
+
 // endpointAddressCount runs one orchestration or CLI step.
 // Signature: (o *Orchestrator) endpointAddressCount(ctx context.Context, namespace string, service string) (int, error).
 // Why: endpoint address counts provide an objective service-backend readiness
@@ -145,12 +244,43 @@ func (o *Orchestrator) endpointAddressCount(ctx context.Context, namespace strin
 		"endpoints",
 		service,
 		"-o",
-		`jsonpath={range .subsets[*].addresses[*]}{.ip}{"\n"}{end}`,
+		"json",
 	)
 	if err != nil {
 		return 0, err
 	}
-	return len(lines(out)), nil
+	cleaned := stripKubectlWarnings(out)
+	if strings.HasPrefix(cleaned, "{") {
+		var endpoint endpointResource
+		if err := json.Unmarshal([]byte(cleaned), &endpoint); err != nil {
+			return 0, fmt.Errorf("decode endpoints %s/%s: %w", namespace, service, err)
+		}
+		count := 0
+		for _, subset := range endpoint.Subsets {
+			for _, address := range subset.Addresses {
+				if strings.TrimSpace(address.IP) != "" {
+					count++
+				}
+			}
+		}
+		return count, nil
+	}
+	return len(lines(cleaned)), nil
+}
+
+// stripKubectlWarnings runs one orchestration or CLI step.
+// Signature: stripKubectlWarnings(out string) string.
+// Why: kubectl writes deprecation warnings to stderr; command execution combines
+// streams, so readiness parsers must not mistake warning lines for health data.
+func stripKubectlWarnings(out string) string {
+	kept := []string{}
+	for _, line := range strings.Split(out, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(line), "Warning:") {
+			continue
+		}
+		kept = append(kept, line)
+	}
+	return strings.TrimSpace(strings.Join(kept, "\n"))
 }
 
 // parseCriticalServiceEndpoint runs one orchestration or CLI step.

internal/cluster/orchestrator_critical_probe_repair.go

@@ -0,0 +1,144 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+)
+
+type criticalBackendWorkload struct {
+	Spec struct {
+		Template struct {
+			Spec struct {
+				Containers []criticalBackendContainer `json:"containers"`
+			} `json:"spec"`
+		} `json:"template"`
+	} `json:"spec"`
+}
+
+type criticalBackendContainer struct {
+	Name          string         `json:"name"`
+	LivenessProbe map[string]any `json:"livenessProbe"`
+	StartupProbe  map[string]any `json:"startupProbe"`
+}
+
+// maybeRepairCriticalBackendStartupProbes runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) maybeRepairCriticalBackendStartupProbes(ctx context.Context, namespace string, service string) ([]string, error).
+// Why: critical backends that are slow after outage recovery can be killed by
+// liveness probes before they ever become ready; adding a startup probe lets
+// Kubernetes distinguish slow boot from dead process without app-specific code.
+func (o *Orchestrator) maybeRepairCriticalBackendStartupProbes(ctx context.Context, namespace string, service string) ([]string, error) {
+	if !o.cfg.Startup.CriticalServiceStartupProbeRepair {
+		return nil, nil
+	}
+	namespace = strings.TrimSpace(namespace)
+	service = strings.TrimSpace(service)
+	if namespace == "" || service == "" {
+		return nil, nil
+	}
+	repaired := []string{}
+	errs := []string{}
+	for _, kind := range []string{"deployment", "statefulset"} {
+		items, err := o.repairWorkloadStartupProbes(ctx, namespace, kind, service)
+		if err != nil {
+			if isNotFoundErr(err) {
+				continue
+			}
+			errs = append(errs, err.Error())
+			continue
+		}
+		repaired = append(repaired, items...)
+	}
+	if len(errs) > 0 {
+		return repaired, errors.New(strings.Join(errs, "; "))
+	}
+	return repaired, nil
+}
+
+// repairWorkloadStartupProbes runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) repairWorkloadStartupProbes(ctx context.Context, namespace string, kind string, name string) ([]string, error).
+// Why: the startup-probe patch is deliberately limited to an existing
+// deployment/statefulset with a liveness probe and no startup probe.
+func (o *Orchestrator) repairWorkloadStartupProbes(ctx context.Context, namespace string, kind string, name string) ([]string, error) {
+	out, err := o.kubectl(ctx, 20*time.Second, "-n", namespace, "get", kind, name, "-o", "json")
+	if err != nil {
+		if strings.TrimSpace(out) != "" {
+			return nil, fmt.Errorf("%w: %s", err, strings.TrimSpace(out))
+		}
+		return nil, err
+	}
+	var workload criticalBackendWorkload
+	if err := json.Unmarshal([]byte(out), &workload); err != nil {
+		return nil, fmt.Errorf("decode %s/%s/%s for startup probe repair: %w", namespace, kind, name, err)
+	}
+
+	containers := []map[string]any{}
+	threshold := o.cfg.Startup.CriticalServiceStartupProbeThreshold
+	if threshold <= 0 {
+		threshold = 30
+	}
+	for _, container := range workload.Spec.Template.Spec.Containers {
+		containerName := strings.TrimSpace(container.Name)
+		if containerName == "" || len(container.LivenessProbe) == 0 || len(container.StartupProbe) > 0 {
+			continue
+		}
+		startupProbe := cloneMap(container.LivenessProbe)
+		startupProbe["failureThreshold"] = threshold
+		if _, ok := startupProbe["successThreshold"]; !ok {
+			startupProbe["successThreshold"] = 1
+		}
+		containers = append(containers, map[string]any{
+			"name":         containerName,
+			"startupProbe": startupProbe,
+		})
+	}
+	if len(containers) == 0 {
+		return nil, nil
+	}
+
+	now := time.Now().UTC().Format(time.RFC3339)
+	patch := map[string]any{
+		"spec": map[string]any{
+			"template": map[string]any{
+				"metadata": map[string]any{
+					"annotations": map[string]string{
+						"ananke.bstein.dev/startup-probe-repair-at":     now,
+						"ananke.bstein.dev/startup-probe-repair-reason": "critical-service-backend-unready",
+					},
+				},
+				"spec": map[string]any{
+					"containers": containers,
+				},
+			},
+		},
+	}
+	rawPatch, err := json.Marshal(patch)
+	if err != nil {
+		return nil, fmt.Errorf("encode startup probe repair patch: %w", err)
+	}
+	if _, err := o.kubectl(ctx, 25*time.Second, "-n", namespace, "patch", kind, name, "--type=strategic", "-p", string(rawPatch)); err != nil {
+		return nil, fmt.Errorf("patch startup probes on %s/%s/%s: %w", namespace, kind, name, err)
+	}
+	o.log.Printf("repaired startup probes on critical backend %s/%s/%s containers=%d", namespace, kind, name, len(containers))
+	o.noteStartupAutoHeal(fmt.Sprintf("repaired startup probes on %s/%s/%s", namespace, kind, name))
+	return []string{namespace + "/" + kind + "/" + name}, nil
+}
+
+// cloneMap runs one orchestration or CLI step.
+// Signature: cloneMap(in map[string]any) map[string]any.
+// Why: Kubernetes probe objects are patch payloads; cloning avoids mutating the
+// decoded workload fixture while changing the generated startup probe threshold.
+func cloneMap(in map[string]any) map[string]any {
+	out := map[string]any{}
+	for key, value := range in {
+		if nested, ok := value.(map[string]any); ok {
+			out[key] = cloneMap(nested)
+			continue
+		}
+		out[key] = value
+	}
+	return out
+}

internal/cluster/orchestrator_critical_vault.go

@@ -178,6 +178,9 @@ func (o *Orchestrator) waitWorkloadReady(ctx context.Context, w startupWorkload)
 		if err == nil {
 			return nil
 		}
+		if isNotFoundErr(err) {
+			return err
+		}
 		lastErr = err
 
 		select {
@@ -318,28 +321,6 @@ func (o *Orchestrator) ensureVaultUnsealed(ctx context.Context) error {
 	return fmt.Errorf("vault remained sealed after 5 auto-unseal attempts")
 }
 
-// vaultSealed runs one orchestration or CLI step.
-// Signature: (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error).
-// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
-func (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error) {
-	out, err := o.kubectl(
-		ctx,
-		25*time.Second,
-		"-n", "vault",
-		"exec", "vault-0", "--",
-		"sh", "-lc",
-		"VAULT_ADDR=http://127.0.0.1:8200 vault status -format=json 2>/dev/null || true",
-	)
-	if err != nil {
-		return false, fmt.Errorf("vault status check failed: %w", err)
-	}
-	sealed, err := parseVaultSealed(out)
-	if err != nil {
-		return false, fmt.Errorf("parse vault status: %w", err)
-	}
-	return sealed, nil
-}
-
 // parseVaultSealed runs one orchestration or CLI step.
 // Signature: parseVaultSealed(raw string) (bool, error).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

internal/cluster/orchestrator_hardening_test.go

@@ -0,0 +1,455 @@
+package cluster
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"log"
+	"net"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret runs one orchestration or CLI step.
+// Signature: TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret(t *testing.T).
+// Why: host repair must recover from password-required sudo while keeping the
+// sudo password out of command arguments and logs.
+func TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret(t *testing.T) {
+	secret := "correct horse battery staple"
+	var logs bytes.Buffer
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{
+			HostSudoSecretNamespace:    "ops",
+			HostSudoSecretNameTemplate: "sudo-{node}",
+			HostSudoSecretPasswordKey:  "password",
+		},
+	}, nil)
+	orch.log = log.New(&logs, "", 0)
+
+	orch.SetCommandOverrides(func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		joined := strings.Join(args, " ")
+		switch {
+		case name == "ssh" && strings.Contains(joined, "sudo -n /usr/bin/systemctl --version"):
+			return "sudo: a password is required", errors.New("exit status 1")
+		case name == "kubectl" && strings.Contains(joined, "-n ops get secret sudo-titan-05 -o json"):
+			return `{"data":{"password":"` + base64.StdEncoding.EncodeToString([]byte(secret)) + `"}}`, nil
+		default:
+			return "", nil
+		}
+	}, nil)
+
+	inputSeen := ""
+	commandSeen := ""
+	orch.SetSSHInputOverride(func(_ context.Context, _ time.Duration, node string, command string, input string) (string, error) {
+		if node != "titan-05" {
+			t.Fatalf("unexpected node %q", node)
+		}
+		commandSeen = command
+		inputSeen = input
+		return "systemd 255", nil
+	})
+
+	if err := orch.preflightHostPrivilege(context.Background(), "titan-05"); err != nil {
+		t.Fatalf("preflightHostPrivilege failed: %v", err)
+	}
+	if !strings.Contains(commandSeen, "sudo -S -p '' /usr/bin/systemctl --version") {
+		t.Fatalf("expected sudo stdin command, got %q", commandSeen)
+	}
+	if inputSeen != secret+"\n" {
+		t.Fatalf("expected sudo password on stdin only")
+	}
+	if strings.Contains(logs.String(), secret) || strings.Contains(commandSeen, secret) {
+		t.Fatalf("sudo password leaked through logs or command")
+	}
+}
+
+// TestHostPrivilegeMissingSecretReportsUnavailable runs one orchestration or CLI step.
+// Signature: TestHostPrivilegeMissingSecretReportsUnavailable(t *testing.T).
+// Why: missing Vault/Kubernetes sudo material should be a classified blocker,
+// not an opaque loop of failed host commands.
+func TestHostPrivilegeMissingSecretReportsUnavailable(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{
+			match: matchContains("ssh", "sudo -n /usr/bin/systemctl --version"),
+			out:   "sudo: a password is required",
+			err:   errors.New("exit status 1"),
+		},
+	})
+	err := orch.preflightHostPrivilege(context.Background(), "titan-05")
+	if err == nil || !strings.Contains(err.Error(), "host-privilege-unavailable") {
+		t.Fatalf("expected host-privilege-unavailable, got %v", err)
+	}
+}
+
+// TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel runs one orchestration or CLI step.
+// Signature: TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel(t *testing.T).
+// Why: a Kubernetes Ready node with a Longhorn Node CR should regain missing
+// manager selector labels instead of leaving PVC workloads blocked.
+func TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel(t *testing.T) {
+	labeled := false
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{
+			match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"),
+			out:   `{"items":[{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`,
+		},
+		{
+			match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager", "-o", "json"),
+			out:   `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true"}}}}`,
+		},
+		{
+			match: matchContains("kubectl", "get", "nodes", "-o", "json"),
+			out:   `{"items":[{"metadata":{"name":"titan-22","labels":{"kubernetes.io/hostname":"titan-22"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`,
+		},
+		{
+			match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json", "-l", "longhorn-host=true"),
+			out:   `{"items":[]}`,
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "label", "node", "titan-22", "--overwrite", "longhorn-host=true")(name, args) {
+					return false
+				}
+				labeled = true
+				return true
+			},
+		},
+	})
+
+	repaired, err := orch.reconcileLonghornKubernetesReadiness(context.Background())
+	if err != nil {
+		t.Fatalf("reconcileLonghornKubernetesReadiness failed: %v", err)
+	}
+	if repaired != 1 || !labeled {
+		t.Fatalf("expected one Longhorn label repair, repaired=%d labeled=%v", repaired, labeled)
+	}
+}
+
+// TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter runs one orchestration or CLI step.
+// Signature: TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter(t *testing.T).
+// Why: stale RWO cleanup may force-delete sidecar-only owners but must not clear
+// an API object while a live application container still mounts the PVC.
+func TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter(t *testing.T) {
+	old := time.Now().Add(-30 * time.Minute).UTC().Format(time.RFC3339)
+	pvcJSON := `{"items":[{"metadata":{"namespace":"finance","name":"firefly-storage"},"spec":{"accessModes":["ReadWriteOnce"],"volumeName":"pvc-1"}}]}`
+	eventsJSON := `{"items":[{"involvedObject":{"kind":"Pod","namespace":"finance","name":"firefly-new"},"type":"Warning","reason":"FailedAttachVolume","message":"Multi-Attach error for volume pvc-1: Volume is already used by pod(s) firefly-old"}]}`
+
+	t.Run("sidecar-only force delete", func(t *testing.T) {
+		podsJSON := `{"items":[` +
+			staleRWOPodJSON("firefly-old", old, "titan-04", true, false) + `,` +
+			replacementRWOPodJSON("firefly-new", "titan-06") +
+			`]}`
+		orch := staleRWOOrchestrator(t, podsJSON, pvcJSON, eventsJSON)
+
+		var pods podList
+		if err := jsonUnmarshal(podsJSON, &pods); err != nil {
+			t.Fatalf("decode pods: %v", err)
+		}
+		decisions, err := orch.staleRWOPVCOwnerDecisions(context.Background(), pods, 5*time.Minute)
+		if err != nil {
+			t.Fatalf("staleRWOPVCOwnerDecisions failed: %v", err)
+		}
+		decision := decisions["finance/firefly-old"]
+		if !decision.ForceDelete || decision.Unsafe {
+			t.Fatalf("expected sidecar-only force delete decision, got %#v", decision)
+		}
+	})
+
+	t.Run("live writer unsafe", func(t *testing.T) {
+		podsJSON := `{"items":[` +
+			staleRWOPodJSON("firefly-old", old, "titan-04", true, true) + `,` +
+			replacementRWOPodJSON("firefly-new", "titan-06") +
+			`]}`
+		orch := staleRWOOrchestrator(t, podsJSON, pvcJSON, eventsJSON)
+
+		var pods podList
+		if err := jsonUnmarshal(podsJSON, &pods); err != nil {
+			t.Fatalf("decode pods: %v", err)
+		}
+		decisions, err := orch.staleRWOPVCOwnerDecisions(context.Background(), pods, 5*time.Minute)
+		if err != nil {
+			t.Fatalf("staleRWOPVCOwnerDecisions failed: %v", err)
+		}
+		decision := decisions["finance/firefly-old"]
+		if !decision.Unsafe || decision.ForceDelete {
+			t.Fatalf("expected live writer unsafe decision, got %#v", decision)
+		}
+	})
+}
+
+// TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS runs one orchestration or CLI step.
+// Signature: TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS(t *testing.T).
+// Why: DNS-caused image pulls should be reported as registry/DNS blockers rather
+// than recycled as if deleting the pod could fix name resolution.
+func TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{
+			match: matchContains("kubectl", "get", "events", "-A", "-o", "json"),
+			out:   `{"items":[{"involvedObject":{"kind":"Pod","namespace":"logging","name":"oauth2"},"type":"Warning","reason":"Failed","message":"Failed to pull image: lookup registry-1.docker.io: Try again"}]}`,
+		},
+	})
+	reasons, err := orch.imagePullDNSBlockerReasons(context.Background())
+	if err != nil {
+		t.Fatalf("imagePullDNSBlockerReasons failed: %v", err)
+	}
+	if got := reasons["logging/oauth2"]; got != "ImagePullDNSBlocker:registry-1.docker.io" {
+		t.Fatalf("unexpected image pull DNS reason %q", got)
+	}
+}
+
+// TestVaultSealedFallsBackToHTTPHealthWhenExecKilled runs one orchestration or CLI step.
+// Signature: TestVaultSealedFallsBackToHTTPHealthWhenExecKilled(t *testing.T).
+// Why: a transient killed kubectl exec should not block startup when Vault HTTP
+// health proves the pod is initialized and unsealed.
+func TestVaultSealedFallsBackToHTTPHealthWhenExecKilled(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{
+			match: matchContains("kubectl", "-n", "vault", "exec", "vault-0"),
+			err:   errors.New("signal: killed"),
+		},
+		{
+			match: matchContains("kubectl", "-n", "vault", "get", "--raw", "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health"),
+			out:   `{"initialized":true,"sealed":false,"standby":false}`,
+		},
+	})
+	sealed, err := orch.vaultSealed(context.Background())
+	if err != nil || sealed {
+		t.Fatalf("expected HTTP fallback to report unsealed, sealed=%v err=%v", sealed, err)
+	}
+}
+
+// TestTCPServiceChecklistReadyChecksMailProtocolBanner runs one orchestration or CLI step.
+// Signature: TestTCPServiceChecklistReadyChecksMailProtocolBanner(t *testing.T).
+// Why: Mailu is not covered by HTTP ingress checks, so Ananke needs direct
+// protocol probes that validate a TCP greeting and command response.
+func TestTCPServiceChecklistReadyChecksMailProtocolBanner(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	defer listener.Close()
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		conn, acceptErr := listener.Accept()
+		if acceptErr != nil {
+			return
+		}
+		defer conn.Close()
+		_, _ = conn.Write([]byte("220 mail.bstein.dev ESMTP ready\r\n"))
+		_, _ = bufio.NewReader(conn).ReadString('\n')
+		_, _ = conn.Write([]byte("221 2.0.0 Bye\r\n"))
+	}()
+
+	host, port, err := net.SplitHostPort(listener.Addr().String())
+	if err != nil {
+		t.Fatalf("split listener address: %v", err)
+	}
+	check := config.TCPServiceChecklistCheck{
+		Name:           "mail-smtp",
+		Host:           host,
+		Port:           atoiForTest(t, port),
+		Send:           "QUIT\r\n",
+		ExpectContains: "ESMTP",
+		TimeoutSeconds: 2,
+	}
+	orch := buildOrchestratorWithStubs(t, config.Config{}, nil)
+	ok, detail := orch.tcpServiceCheckReady(context.Background(), check)
+	if !ok {
+		t.Fatalf("expected TCP mail check to pass, detail=%s", detail)
+	}
+	<-done
+}
+
+// TestServiceChecklistReadyIncludesTCPFailures runs one orchestration or CLI step.
+// Signature: TestServiceChecklistReadyIncludesTCPFailures(t *testing.T).
+// Why: startup should fail clearly when a configured mail protocol endpoint is
+// down even if there are no HTTP checks in that config.
+func TestServiceChecklistReadyIncludesTCPFailures(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{
+			TCPServiceChecklist: []config.TCPServiceChecklistCheck{
+				{
+					Name:           "mail-smtp",
+					Host:           "127.0.0.1",
+					Port:           1,
+					ExpectContains: "ESMTP",
+					TimeoutSeconds: 1,
+				},
+			},
+		},
+	}, nil)
+	ok, detail := orch.serviceChecklistReady(context.Background())
+	if ok {
+		t.Fatalf("expected TCP-only checklist to fail")
+	}
+	if !strings.Contains(detail, "tcp mail-smtp") {
+		t.Fatalf("expected TCP failure detail, got %q", detail)
+	}
+}
+
+// TestEndpointAddressCountIgnoresKubectlWarnings runs one orchestration or CLI step.
+// Signature: TestEndpointAddressCountIgnoresKubectlWarnings(t *testing.T).
+// Why: Kubernetes endpoint deprecation warnings must not make an empty service
+// look ready by contributing a fake output line.
+func TestEndpointAddressCountIgnoresKubectlWarnings(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{
+			match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "endpoints", "mailu-admin", "-o", "json"),
+			out:   "Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\n{\"subsets\":[]}",
+		},
+	})
+	count, err := orch.endpointAddressCount(context.Background(), "mailu-mailserver", "mailu-admin")
+	if err != nil {
+		t.Fatalf("endpointAddressCount failed: %v", err)
+	}
+	if count != 0 {
+		t.Fatalf("expected empty endpoint count after warning strip, got %d", count)
+	}
+}
+
+// TestCriticalBackendStartupProbeRepairCopiesLivenessProbe runs one orchestration or CLI step.
+// Signature: TestCriticalBackendStartupProbeRepairCopiesLivenessProbe(t *testing.T).
+// Why: slow critical backends should get a bounded startup grace without
+// changing app-specific liveness semantics or requiring a Mailu special case.
+func TestCriticalBackendStartupProbeRepairCopiesLivenessProbe(t *testing.T) {
+	var patchPayload string
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{
+			CriticalServiceStartupProbeRepair:    true,
+			CriticalServiceStartupProbeThreshold: 24,
+		},
+	}, []commandStub{
+		{
+			match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "deployment", "mailu-admin", "-o", "json"),
+			out: `{"spec":{"template":{"spec":{"containers":[{"name":"admin","livenessProbe":{"httpGet":{"path":"/ping","port":"http"},"periodSeconds":10,"failureThreshold":3}},` +
+				`{"name":"sidecar"}]}}}}`,
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "mailu-mailserver", "patch", "deployment", "mailu-admin", "--type=strategic", "-p")(name, args) {
+					return false
+				}
+				for i, arg := range args {
+					if arg == "-p" && i+1 < len(args) {
+						patchPayload = args[i+1]
+					}
+				}
+				return true
+			},
+		},
+		{
+			match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "statefulset", "mailu-admin", "-o", "json"),
+			err:   errors.New("not found"),
+		},
+	})
+
+	repaired, err := orch.maybeRepairCriticalBackendStartupProbes(context.Background(), "mailu-mailserver", "mailu-admin")
+	if err != nil {
+		t.Fatalf("maybeRepairCriticalBackendStartupProbes failed: %v", err)
+	}
+	if len(repaired) != 1 || repaired[0] != "mailu-mailserver/deployment/mailu-admin" {
+		t.Fatalf("unexpected repaired list: %v", repaired)
+	}
+	if patchPayload == "" {
+		t.Fatalf("expected strategic merge patch payload")
+	}
+	if !strings.Contains(patchPayload, `"startupProbe"`) || !strings.Contains(patchPayload, `"failureThreshold":24`) || !strings.Contains(patchPayload, `"path":"/ping"`) {
+		t.Fatalf("unexpected startup-probe patch payload: %s", patchPayload)
+	}
+}
+
+// TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet runs one orchestration or CLI step.
+// Signature: TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet(t *testing.T).
+// Why: Mailu backends are Deployments; a kubectl NotFound message on the
+// StatefulSet probe path must not turn a successful/no-op Deployment check into
+// an auto-heal failure.
+func TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{
+			CriticalServiceStartupProbeRepair:    true,
+			CriticalServiceStartupProbeThreshold: 24,
+		},
+	}, []commandStub{
+		{
+			match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "deployment", "mailu-rspamd", "-o", "json"),
+			out:   `{"spec":{"template":{"spec":{"containers":[{"name":"rspamd","livenessProbe":{"httpGet":{"path":"/","port":"http"}},"startupProbe":{"httpGet":{"path":"/","port":"http"}}}]}}}}`,
+		},
+		{
+			match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "statefulset", "mailu-rspamd", "-o", "json"),
+			out:   `Error from server (NotFound): statefulsets.apps "mailu-rspamd" not found`,
+			err:   errors.New("exit status 1"),
+		},
+	})
+	repaired, err := orch.maybeRepairCriticalBackendStartupProbes(context.Background(), "mailu-mailserver", "mailu-rspamd")
+	if err != nil {
+		t.Fatalf("expected missing statefulset to be ignored, got %v", err)
+	}
+	if len(repaired) != 0 {
+		t.Fatalf("expected no repair for existing startup probe, got %v", repaired)
+	}
+}
+
+// atoiForTest runs one orchestration or CLI step.
+// Signature: atoiForTest(t *testing.T, raw string) int.
+// Why: TCP listener fixtures need the kernel-selected port as a typed config
+// value without obscuring test failures.
+func atoiForTest(t *testing.T, raw string) int {
+	t.Helper()
+	port, err := strconv.Atoi(raw)
+	if err != nil {
+		t.Fatalf("parse port %q: %v", raw, err)
+	}
+	return port
+}
+
+// staleRWOOrchestrator runs one orchestration or CLI step.
+// Signature: staleRWOOrchestrator(t *testing.T, podsJSON string, pvcJSON string, eventsJSON string) *Orchestrator.
+// Why: stale RWO tests need a tiny fake API surface for pods, PVCs, and events
+// without depending on a live Kubernetes cluster.
+func staleRWOOrchestrator(t *testing.T, podsJSON string, pvcJSON string, eventsJSON string) *Orchestrator {
+	t.Helper()
+	return buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: eventsJSON},
+		{match: matchContains("kubectl", "get", "pvc", "-A", "-o", "json"), out: pvcJSON},
+		{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: podsJSON},
+	})
+}
+
+// staleRWOPodJSON runs one orchestration or CLI step.
+// Signature: staleRWOPodJSON(name string, deletedAt string, node string, sidecarRunning bool, appRunning bool) string.
+// Why: the sidecar-only and live-writer fixtures differ only in container
+// status, so one builder keeps the safety distinction obvious.
+func staleRWOPodJSON(name string, deletedAt string, node string, sidecarRunning bool, appRunning bool) string {
+	statuses := []string{}
+	if sidecarRunning {
+		statuses = append(statuses, `{"name":"vault-agent","state":{"running":{"startedAt":"`+deletedAt+`"}}}`)
+	}
+	if appRunning {
+		statuses = append(statuses, `{"name":"firefly","state":{"running":{"startedAt":"`+deletedAt+`"}}}`)
+	}
+	return `{"metadata":{"namespace":"finance","name":"` + name + `","creationTimestamp":"` + deletedAt + `","deletionTimestamp":"` + deletedAt + `","ownerReferences":[{"kind":"ReplicaSet","name":"firefly"}]},"spec":{"nodeName":"` + node + `","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"firefly-storage"}},{"name":"vault-secret"}],"containers":[{"name":"firefly","volumeMounts":[{"name":"data","mountPath":"/var/www/html/storage"}]},{"name":"vault-agent","volumeMounts":[{"name":"vault-secret","mountPath":"/vault/secrets"}]}]},"status":{"phase":"Running","containerStatuses":[` + strings.Join(statuses, ",") + `]}}`
+}
+
+// replacementRWOPodJSON runs one orchestration or CLI step.
+// Signature: replacementRWOPodJSON(name string, node string) string.
+// Why: stale-owner tests need a controller sibling blocked in Pending on a
+// different node while sharing the same RWO claim.
+func replacementRWOPodJSON(name string, node string) string {
+	return `{"metadata":{"namespace":"finance","name":"` + name + `","creationTimestamp":"` + time.Now().Add(-10*time.Minute).UTC().Format(time.RFC3339) + `","ownerReferences":[{"kind":"ReplicaSet","name":"firefly"}]},"spec":{"nodeName":"` + node + `","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"firefly-storage"}}],"containers":[{"name":"firefly","volumeMounts":[{"name":"data","mountPath":"/var/www/html/storage"}]}]},"status":{"phase":"Pending","containerStatuses":[{"name":"firefly","state":{"waiting":{"reason":"ContainerCreating"}}}]}}`
+}
+
+// jsonUnmarshal runs one orchestration or CLI step.
+// Signature: jsonUnmarshal(raw string, target any) error.
+// Why: local JSON fixtures should use the same decoder semantics as Kubernetes
+// command output parsing.
+func jsonUnmarshal(raw string, target any) error {
+	return json.Unmarshal([]byte(raw), target)
+}

internal/cluster/orchestrator_host_privilege.go

@@ -0,0 +1,331 @@
+package cluster
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+type hostPrivilegedAction string
+
+const (
+	hostActionSudoPreflight     hostPrivilegedAction = "sudo-preflight"
+	hostActionK3sAgentShow      hostPrivilegedAction = "k3s-agent-show"
+	hostActionK3sAgentIsActive  hostPrivilegedAction = "k3s-agent-is-active"
+	hostActionK3sAgentRestart   hostPrivilegedAction = "k3s-agent-restart-no-block"
+	hostActionHostReboot        hostPrivilegedAction = "host-reboot"
+	hostActionInstallCryptsetup hostPrivilegedAction = "install-cryptsetup-bin"
+	hostActionModprobeDMCrypt   hostPrivilegedAction = "modprobe-dm-crypt"
+	hostActionCrictlPods        hostPrivilegedAction = "crictl-pods"
+	hostActionCrictlPs          hostPrivilegedAction = "crictl-ps"
+	hostActionCrictlInspect     hostPrivilegedAction = "crictl-inspect"
+	hostActionCrictlStop        hostPrivilegedAction = "crictl-stop"
+)
+
+type hostPrivilegeError struct {
+	Class  string
+	Node   string
+	Action hostPrivilegedAction
+	Detail string
+}
+
+// Error runs one orchestration or CLI step.
+// Signature: (e hostPrivilegeError) Error() string.
+// Why: host repair failures need stable operator-facing classes without
+// embedding secret material or raw command input.
+func (e hostPrivilegeError) Error() string {
+	parts := []string{strings.TrimSpace(e.Class)}
+	if strings.TrimSpace(e.Node) != "" {
+		parts = append(parts, "node="+strings.TrimSpace(e.Node))
+	}
+	if e.Action != "" {
+		parts = append(parts, "action="+string(e.Action))
+	}
+	if strings.TrimSpace(e.Detail) != "" {
+		parts = append(parts, strings.TrimSpace(e.Detail))
+	}
+	return strings.Join(parts, " ")
+}
+
+type hostSudoSecret struct {
+	Data map[string]string `json:"data"`
+}
+
+// preflightHostPrivilege verifies that Ananke can run harmless sudo on a node.
+// Signature: (o *Orchestrator) preflightHostPrivilege(ctx context.Context, node string) error.
+// Why: host repair should fail with a clear privilege class before an outage
+// path needs a package install, k3s-agent restart, runtime inspection, or reboot.
+func (o *Orchestrator) preflightHostPrivilege(ctx context.Context, node string) error {
+	_, err := o.runHostPrivilegedAction(ctx, node, hostActionSudoPreflight, 12*time.Second)
+	return err
+}
+
+// runHostPrivilegedAction executes one allowlisted sudo action on a managed host.
+// Signature: (o *Orchestrator) runHostPrivilegedAction(ctx context.Context, node string, action hostPrivilegedAction, timeout time.Duration, args ...string) (string, error).
+// Why: recovery must not turn Vault-backed sudo into arbitrary remote root
+// execution; every privileged operation is classified, bounded, and auditable.
+func (o *Orchestrator) runHostPrivilegedAction(ctx context.Context, node string, action hostPrivilegedAction, timeout time.Duration, args ...string) (string, error) {
+	node = strings.TrimSpace(node)
+	if node == "" {
+		return "", hostPrivilegeError{Class: "host-privilege-unavailable", Action: action, Detail: "node is empty"}
+	}
+	if !o.sshManaged(node) {
+		return "", hostPrivilegeError{Class: "host-privilege-unavailable", Node: node, Action: action, Detail: "node is not SSH-managed"}
+	}
+	commandArgs, err := hostPrivilegedCommandArgs(action, args...)
+	if err != nil {
+		return "", err
+	}
+	if timeout <= 0 {
+		timeout = o.hostPrivilegedCommandTimeout()
+	}
+
+	passwordless := sudoRemoteCommand(false, commandArgs)
+	out, err := o.sshWithTimeout(ctx, node, passwordless, timeout)
+	if err == nil {
+		o.logHostPrivilegedAction(node, action, "passwordless-sudo")
+		return out, nil
+	}
+	if !sudoMayNeedPassword(out, err) {
+		return out, classifyHostPrivilegeFailure(node, action, err, out)
+	}
+
+	password, secretClass, secretErr := o.hostSudoPassword(ctx, node)
+	if secretErr != nil {
+		return "", hostPrivilegeError{
+			Class:  "host-privilege-unavailable",
+			Node:   node,
+			Action: action,
+			Detail: fmt.Sprintf("%s: %v", secretClass, secretErr),
+		}
+	}
+	passwordBacked := sudoRemoteCommand(true, commandArgs)
+	out, err = o.sshWithInput(ctx, node, passwordBacked, password+"\n", timeout)
+	if err != nil {
+		return out, classifyHostPrivilegeFailure(node, action, err, out)
+	}
+	o.logHostPrivilegedAction(node, action, "vault-backed-sudo")
+	return out, nil
+}
+
+// hostPrivilegedCommandTimeout runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) hostPrivilegedCommandTimeout() time.Duration.
+// Why: every privileged host command must have a bounded default timeout even
+// when a focused unit test builds a partial config.
+func (o *Orchestrator) hostPrivilegedCommandTimeout() time.Duration {
+	seconds := o.cfg.Startup.HostPrivilegedCommandTimeoutSec
+	if seconds <= 0 {
+		seconds = 90
+	}
+	return time.Duration(seconds) * time.Second
+}
+
+// logHostPrivilegedAction runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) logHostPrivilegedAction(node string, action hostPrivilegedAction, mode string).
+// Why: audit logs should record the action class and sudo mode while omitting
+// command stdin, passwords, and secret contents.
+func (o *Orchestrator) logHostPrivilegedAction(node string, action hostPrivilegedAction, mode string) {
+	o.log.Printf("host privileged action completed node=%s action=%s mode=%s", node, action, mode)
+}
+
+// hostPrivilegedCommandArgs runs one orchestration or CLI step.
+// Signature: hostPrivilegedCommandArgs(action hostPrivilegedAction, args ...string) ([]string, error).
+// Why: the Vault-backed sudo path must stay on a strict allowlist instead of
+// accepting arbitrary remote root commands.
+func hostPrivilegedCommandArgs(action hostPrivilegedAction, args ...string) ([]string, error) {
+	switch action {
+	case hostActionSudoPreflight:
+		return []string{"/usr/bin/systemctl", "--version"}, nil
+	case hostActionK3sAgentShow:
+		return []string{"systemctl", "show", "k3s-agent", "--property=ActiveState,SubState,Result,MainPID", "--no-pager"}, nil
+	case hostActionK3sAgentIsActive:
+		return []string{"systemctl", "is-active", "k3s-agent"}, nil
+	case hostActionK3sAgentRestart:
+		return []string{"systemctl", "--no-block", "restart", "k3s-agent"}, nil
+	case hostActionHostReboot:
+		return []string{"systemctl", "reboot"}, nil
+	case hostActionInstallCryptsetup:
+		return []string{
+			"env",
+			"DEBIAN_FRONTEND=noninteractive",
+			"sh",
+			"-lc",
+			"apt-get update && apt-get install -y --no-install-recommends cryptsetup-bin && (command -v cryptsetup >/dev/null 2>&1 || test -x /usr/sbin/cryptsetup || test -x /usr/bin/cryptsetup) && echo __ANANKE_CRYPTSETUP_INSTALLED__",
+		}, nil
+	case hostActionModprobeDMCrypt:
+		return []string{"modprobe", "dm_crypt"}, nil
+	case hostActionCrictlPods:
+		return []string{"crictl", "pods", "-o", "json"}, nil
+	case hostActionCrictlPs:
+		return []string{"crictl", "ps", "-a", "-o", "json"}, nil
+	case hostActionCrictlInspect:
+		if len(args) != 1 || strings.TrimSpace(args[0]) == "" || strings.ContainsAny(args[0], " \t\r\n;&|`$<>") {
+			return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "crictl inspect requires one safe container id"}
+		}
+		return []string{"crictl", "inspect", strings.TrimSpace(args[0])}, nil
+	case hostActionCrictlStop:
+		if len(args) != 1 || strings.TrimSpace(args[0]) == "" || strings.ContainsAny(args[0], " \t\r\n;&|`$<>") {
+			return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "crictl stop requires one safe container id"}
+		}
+		return []string{"crictl", "stop", strings.TrimSpace(args[0])}, nil
+	default:
+		return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "action is not allowlisted"}
+	}
+}
+
+// sudoRemoteCommand runs one orchestration or CLI step.
+// Signature: sudoRemoteCommand(passwordBacked bool, commandArgs []string) string.
+// Why: passwordless and password-backed sudo share one command builder so
+// quoting and prompt suppression stay consistent.
+func sudoRemoteCommand(passwordBacked bool, commandArgs []string) string {
+	parts := []string{"sudo"}
+	if passwordBacked {
+		parts = append(parts, "-S", "-p", "")
+	} else {
+		parts = append(parts, "-n")
+	}
+	parts = append(parts, commandArgs...)
+	return shellJoin(parts)
+}
+
+// shellJoin runs one orchestration or CLI step.
+// Signature: shellJoin(parts []string) string.
+// Why: remote SSH commands need readable simple words and safe quoting for
+// shell-sensitive fragments such as package-manager scripts.
+func shellJoin(parts []string) string {
+	out := make([]string, 0, len(parts))
+	for _, part := range parts {
+		out = append(out, shellQuoteIfNeeded(part))
+	}
+	return strings.Join(out, " ")
+}
+
+// shellQuoteIfNeeded runs one orchestration or CLI step.
+// Signature: shellQuoteIfNeeded(part string) string.
+// Why: audit-friendly commands should avoid unnecessary quotes while still
+// protecting whitespace, quotes, and metacharacters.
+func shellQuoteIfNeeded(part string) string {
+	if part == "" {
+		return shellQuote(part)
+	}
+	for _, r := range part {
+		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
+			continue
+		}
+		switch r {
+		case '_', '-', '.', '/', '=', ':':
+			continue
+		default:
+			return shellQuote(part)
+		}
+	}
+	return part
+}
+
+// sudoMayNeedPassword runs one orchestration or CLI step.
+// Signature: sudoMayNeedPassword(out string, err error) bool.
+// Why: Ananke should try Vault-backed sudo only for sudo privilege failures, not
+// for unrelated SSH or command errors.
+func sudoMayNeedPassword(out string, err error) bool {
+	full := strings.ToLower(strings.TrimSpace(out + " " + fmt.Sprint(err)))
+	needles := []string{
+		"a password is required",
+		"password is required",
+		"sudo: a terminal is required",
+		"no tty present",
+		"may not run sudo",
+		"sudo denied",
+		"is not in the sudoers file",
+	}
+	for _, needle := range needles {
+		if strings.Contains(full, needle) {
+			return true
+		}
+	}
+	return false
+}
+
+// classifyHostPrivilegeFailure runs one orchestration or CLI step.
+// Signature: classifyHostPrivilegeFailure(node string, action hostPrivilegedAction, err error, out string) error.
+// Why: host repair needs distinct timeout, auth, SSH, and command-failure
+// classes so status output tells the operator what kind of help is needed.
+func classifyHostPrivilegeFailure(node string, action hostPrivilegedAction, err error, out string) error {
+	full := strings.ToLower(strings.TrimSpace(out + " " + fmt.Sprint(err)))
+	class := "host-command-failed"
+	switch {
+	case strings.Contains(full, "context deadline exceeded"), strings.Contains(full, "timed out"), strings.Contains(full, "timeout"):
+		class = "host-command-timeout"
+	case strings.Contains(full, "authentication failure"), strings.Contains(full, "incorrect password"), strings.Contains(full, "try again"):
+		class = "host-privilege-auth-failed"
+	case strings.Contains(full, "permission denied"), strings.Contains(full, "publickey"):
+		class = "host-ssh-unavailable"
+	}
+	return hostPrivilegeError{Class: class, Node: node, Action: action, Detail: scrubHostPrivilegeDetail(out, err)}
+}
+
+// scrubHostPrivilegeDetail runs one orchestration or CLI step.
+// Signature: scrubHostPrivilegeDetail(out string, err error) string.
+// Why: host command details should be compact enough for annotations and avoid
+// accidental multi-line or control-character log noise.
+func scrubHostPrivilegeDetail(out string, err error) string {
+	detail := strings.TrimSpace(fmt.Sprint(err))
+	if trimmed := strings.TrimSpace(out); trimmed != "" {
+		if detail != "" {
+			detail += ": "
+		}
+		detail += trimmed
+	}
+	return sanitizeCordonAnnotationValue(detail)
+}
+
+// hostSudoPassword runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) hostSudoPassword(ctx context.Context, node string) (string, string, error).
+// Why: sudo material should come from a configured Vault-synced Kubernetes
+// Secret and never be logged or passed through ordinary command arguments.
+func (o *Orchestrator) hostSudoPassword(ctx context.Context, node string) (string, string, error) {
+	namespace := strings.TrimSpace(o.cfg.Startup.HostSudoSecretNamespace)
+	template := strings.TrimSpace(o.cfg.Startup.HostSudoSecretNameTemplate)
+	key := strings.TrimSpace(o.cfg.Startup.HostSudoSecretPasswordKey)
+	if key == "" {
+		key = "password"
+	}
+	if namespace == "" || template == "" {
+		return "", "secret-lookup-unconfigured", fmt.Errorf("host sudo secret namespace/template is not configured")
+	}
+	secretName := strings.ReplaceAll(template, "{node}", node)
+	secretName = strings.TrimSpace(secretName)
+	if secretName == "" {
+		return "", "secret-lookup-invalid", fmt.Errorf("host sudo secret name resolved empty")
+	}
+
+	out, err := o.kubectl(ctx, 15*time.Second, "-n", namespace, "get", "secret", secretName, "-o", "json")
+	if err != nil {
+		return "", "secret-lookup-failed", err
+	}
+	var secret hostSudoSecret
+	if err := json.Unmarshal([]byte(out), &secret); err != nil {
+		return "", "secret-decode-failed", err
+	}
+	encoded := strings.TrimSpace(secret.Data[key])
+	if encoded == "" {
+		keys := make([]string, 0, len(secret.Data))
+		for k := range secret.Data {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		return "", "secret-key-missing", fmt.Errorf("password key %q missing (available=%s)", key, joinLimited(keys, 4))
+	}
+	decoded, err := base64.StdEncoding.DecodeString(encoded)
+	if err != nil {
+		return "", "secret-decode-failed", err
+	}
+	password := strings.TrimRight(string(decoded), "\r\n")
+	if password == "" {
+		return "", "secret-empty", fmt.Errorf("password key %q is empty", key)
+	}
+	return password, "secret-lookup-ok", nil
+}

internal/cluster/orchestrator_image_pull_dns.go

@@ -0,0 +1,88 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// imagePullDNSBlockerReasons classifies image pull failures caused by DNS.
+// Signature: (o *Orchestrator) imagePullDNSBlockerReasons(ctx context.Context) (map[string]string, error).
+// Why: deleting an unchanged ImagePullBackOff pod does not repair registry DNS;
+// Ananke should report the blocker and wait for DNS/registry health to change.
+func (o *Orchestrator) imagePullDNSBlockerReasons(ctx context.Context) (map[string]string, error) {
+	eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query events for image-pull DNS scan: %w", err)
+	}
+	reasons := map[string]string{}
+	if strings.TrimSpace(eventsOut) == "" {
+		return reasons, nil
+	}
+	var events eventList
+	if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
+		return nil, fmt.Errorf("decode events for image-pull DNS scan: %w", err)
+	}
+	for _, event := range events.Items {
+		if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
+			continue
+		}
+		reason := strings.TrimSpace(event.Reason)
+		if reason != "Failed" && reason != "FailedPull" && reason != "ErrImagePull" && reason != "ImagePullBackOff" {
+			continue
+		}
+		message := strings.TrimSpace(event.Message)
+		if !imagePullMessageHasDNSFailure(message) {
+			continue
+		}
+		key := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name)
+		if key == "/" {
+			continue
+		}
+		reasons[key] = "ImagePullDNSBlocker:" + imagePullRegistryHost(message)
+	}
+	return reasons, nil
+}
+
+// imagePullMessageHasDNSFailure runs one orchestration or CLI step.
+// Signature: imagePullMessageHasDNSFailure(message string) bool.
+// Why: image pull recycling should be suppressed only for resolver failures,
+// leaving ordinary pull/auth errors on their existing paths.
+func imagePullMessageHasDNSFailure(message string) bool {
+	lower := strings.ToLower(message)
+	if !strings.Contains(lower, "lookup ") {
+		return false
+	}
+	return strings.Contains(lower, "try again") ||
+		strings.Contains(lower, "no such host") ||
+		strings.Contains(lower, "server misbehaving") ||
+		strings.Contains(lower, "temporary failure") ||
+		strings.Contains(lower, "i/o timeout")
+}
+
+// imagePullRegistryHost runs one orchestration or CLI step.
+// Signature: imagePullRegistryHost(message string) string.
+// Why: operator status should group DNS blockers by registry host instead of
+// repeating full kubelet event text for every pod.
+func imagePullRegistryHost(message string) string {
+	lower := strings.ToLower(message)
+	idx := strings.Index(lower, "lookup ")
+	if idx < 0 {
+		return "unknown-registry"
+	}
+	rest := strings.TrimSpace(message[idx+len("lookup "):])
+	fields := strings.Fields(rest)
+	if len(fields) == 0 {
+		return "unknown-registry"
+	}
+	host := strings.Trim(fields[0], `"'[]():,`)
+	if host == "" {
+		return "unknown-registry"
+	}
+	return host
+}

internal/cluster/orchestrator_longhorn_reconcile.go

@@ -0,0 +1,265 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+type longhornNodeList struct {
+	Items []longhornNode `json:"items"`
+}
+
+type longhornNode struct {
+	Metadata struct {
+		Name string `json:"name"`
+	} `json:"metadata"`
+	Spec struct {
+		AllowScheduling *bool `json:"allowScheduling"`
+	} `json:"spec"`
+	Status struct {
+		Conditions []longhornCondition `json:"conditions"`
+	} `json:"status"`
+}
+
+type longhornCondition struct {
+	Type    string `json:"type"`
+	Status  string `json:"status"`
+	Reason  string `json:"reason"`
+	Message string `json:"message"`
+}
+
+type daemonSetResource struct {
+	Metadata struct {
+		Namespace string `json:"namespace"`
+		Name      string `json:"name"`
+	} `json:"metadata"`
+	Spec struct {
+		Selector struct {
+			MatchLabels map[string]string `json:"matchLabels"`
+		} `json:"selector"`
+	} `json:"spec"`
+}
+
+// reconcileLonghornKubernetesReadiness repairs safe Kubernetes/Longhorn drift.
+// Signature: (o *Orchestrator) reconcileLonghornKubernetesReadiness(ctx context.Context) (int, error).
+// Why: Kubernetes Ready=True is not enough for PVC scheduling when Longhorn's
+// manager, labels, and node conditions disagree with the Kubernetes node model.
+func (o *Orchestrator) reconcileLonghornKubernetesReadiness(ctx context.Context) (int, error) {
+	longhornNodes, err := o.queryLonghornNodes(ctx)
+	if err != nil {
+		return 0, err
+	}
+	if len(longhornNodes.Items) == 0 {
+		return 0, nil
+	}
+	managerDS, err := o.queryLonghornManagerDaemonSet(ctx)
+	if err != nil {
+		return 0, err
+	}
+	selector := normalizeSelectorLabels(managerDS.Spec.Selector.MatchLabels)
+	if len(selector) == 0 {
+		return 0, fmt.Errorf("longhorn manager daemonset has no matchLabels selector")
+	}
+	nodes, err := o.queryReadyNodes(ctx)
+	if err != nil {
+		return 0, err
+	}
+	kubeNodes := map[string]nodeReadyItem{}
+	for _, node := range nodes.Items {
+		name := strings.TrimSpace(node.Metadata.Name)
+		if name != "" {
+			kubeNodes[name] = node
+		}
+	}
+	managerPods, err := o.longhornManagerPodsByNode(ctx, selector)
+	if err != nil {
+		return 0, err
+	}
+
+	repaired := 0
+	errs := []string{}
+	for _, lhNode := range longhornNodes.Items {
+		name := strings.TrimSpace(lhNode.Metadata.Name)
+		if name == "" {
+			continue
+		}
+		kubeNode, ok := kubeNodes[name]
+		if !ok {
+			continue
+		}
+		if _, hasManager := managerPods[name]; hasManager {
+			continue
+		}
+		readyCond := longhornConditionByType(lhNode, "Ready")
+		if readyCond == nil || strings.EqualFold(strings.TrimSpace(readyCond.Status), "True") {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(readyCond.Reason), "ManagerPodMissing") {
+			continue
+		}
+		missing := missingSelectorLabels(kubeNode.Metadata.Labels, selector)
+		if len(missing) == 0 {
+			o.log.Printf("warning: Longhorn manager pod missing on node=%s but node already matches selector; leaving for DaemonSet scheduler reconciliation", name)
+			continue
+		}
+
+		labels := make([]string, 0, len(missing))
+		for key, value := range missing {
+			labels = append(labels, key+"="+value)
+		}
+		sort.Strings(labels)
+		args := append([]string{"label", "node", name, "--overwrite"}, labels...)
+		o.log.Printf("warning: restoring Longhorn manager selector labels on node=%s labels=%s reason=LonghornNodeExistsAndManagerPodMissing", name, strings.Join(labels, ","))
+		if _, err := o.kubectl(ctx, 25*time.Second, args...); err != nil {
+			errs = append(errs, fmt.Sprintf("%s restore Longhorn selector labels: %v", name, err))
+			continue
+		}
+		repaired++
+		o.noteStartupAutoHeal(fmt.Sprintf("restored Longhorn manager selector labels on %s: %s", name, strings.Join(labels, ",")))
+	}
+	if len(errs) > 0 {
+		return repaired, errors.New(strings.Join(errs, "; "))
+	}
+	return repaired, nil
+}
+
+// queryLonghornNodes runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) queryLonghornNodes(ctx context.Context) (longhornNodeList, error).
+// Why: Longhorn/Kubernetes drift reconciliation starts from Longhorn's own node
+// model rather than assuming Kubernetes Ready is sufficient for storage.
+func (o *Orchestrator) queryLonghornNodes(ctx context.Context) (longhornNodeList, error) {
+	out, err := o.kubectl(ctx, 30*time.Second, "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json")
+	if err != nil {
+		if isNotFoundErr(err) {
+			return longhornNodeList{}, nil
+		}
+		return longhornNodeList{}, fmt.Errorf("query longhorn nodes: %w", err)
+	}
+	if strings.TrimSpace(out) == "" {
+		return longhornNodeList{}, nil
+	}
+	var nodes longhornNodeList
+	if err := json.Unmarshal([]byte(out), &nodes); err != nil {
+		return longhornNodeList{}, fmt.Errorf("decode longhorn nodes: %w", err)
+	}
+	return nodes, nil
+}
+
+// queryLonghornManagerDaemonSet runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) queryLonghornManagerDaemonSet(ctx context.Context) (daemonSetResource, error).
+// Why: manager selector labels are installation policy, so repair must read
+// the live DaemonSet selector instead of hard-coding a Titan label.
+func (o *Orchestrator) queryLonghornManagerDaemonSet(ctx context.Context) (daemonSetResource, error) {
+	out, err := o.kubectl(ctx, 20*time.Second, "-n", "longhorn-system", "get", "daemonset", "longhorn-manager", "-o", "json")
+	if err != nil {
+		return daemonSetResource{}, fmt.Errorf("query longhorn manager daemonset: %w", err)
+	}
+	var ds daemonSetResource
+	if err := json.Unmarshal([]byte(out), &ds); err != nil {
+		return daemonSetResource{}, fmt.Errorf("decode longhorn manager daemonset: %w", err)
+	}
+	return ds, nil
+}
+
+// longhornManagerPodsByNode runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) longhornManagerPodsByNode(ctx context.Context, selector map[string]string) (map[string]struct{}, error).
+// Why: manager-pod presence is the postcondition for selector repair and
+// distinguishes label drift from a healthy Longhorn node.
+func (o *Orchestrator) longhornManagerPodsByNode(ctx context.Context, selector map[string]string) (map[string]struct{}, error) {
+	args := []string{"-n", "longhorn-system", "get", "pods", "-o", "json"}
+	if len(selector) > 0 {
+		args = append(args, "-l", selectorLabelString(selector))
+	}
+	out, err := o.kubectl(ctx, 20*time.Second, args...)
+	if err != nil {
+		return nil, fmt.Errorf("query longhorn manager pods: %w", err)
+	}
+	var pods podList
+	if err := json.Unmarshal([]byte(out), &pods); err != nil {
+		return nil, fmt.Errorf("decode longhorn manager pods: %w", err)
+	}
+	byNode := map[string]struct{}{}
+	for _, pod := range pods.Items {
+		node := strings.TrimSpace(pod.Spec.NodeName)
+		if node == "" {
+			continue
+		}
+		if podMatchesLabels(pod.Metadata.Labels, selector) {
+			byNode[node] = struct{}{}
+		}
+	}
+	return byNode, nil
+}
+
+// normalizeSelectorLabels runs one orchestration or CLI step.
+// Signature: normalizeSelectorLabels(labels map[string]string) map[string]string.
+// Why: empty selector fragments should not produce accidental node label writes.
+func normalizeSelectorLabels(labels map[string]string) map[string]string {
+	out := map[string]string{}
+	for key, value := range labels {
+		key = strings.TrimSpace(key)
+		value = strings.TrimSpace(value)
+		if key != "" && value != "" {
+			out[key] = value
+		}
+	}
+	return out
+}
+
+// selectorLabelString runs one orchestration or CLI step.
+// Signature: selectorLabelString(labels map[string]string) string.
+// Why: kubectl label selectors need stable ordering for deterministic tests and
+// readable logs.
+func selectorLabelString(labels map[string]string) string {
+	parts := make([]string, 0, len(labels))
+	for key, value := range labels {
+		parts = append(parts, key+"="+value)
+	}
+	sort.Strings(parts)
+	return strings.Join(parts, ",")
+}
+
+// podMatchesLabels runs one orchestration or CLI step.
+// Signature: podMatchesLabels(labels map[string]string, selector map[string]string) bool.
+// Why: Longhorn manager pod checks should use the DaemonSet selector rather
+// than assuming a specific app label.
+func podMatchesLabels(labels map[string]string, selector map[string]string) bool {
+	for key, value := range selector {
+		if labels == nil || strings.TrimSpace(labels[key]) != value {
+			return false
+		}
+	}
+	return true
+}
+
+// missingSelectorLabels runs one orchestration or CLI step.
+// Signature: missingSelectorLabels(labels map[string]string, selector map[string]string) map[string]string.
+// Why: label-drift repair should write exactly the manager selector labels that
+// are absent or wrong on the node.
+func missingSelectorLabels(labels map[string]string, selector map[string]string) map[string]string {
+	missing := map[string]string{}
+	for key, value := range selector {
+		if labels == nil || strings.TrimSpace(labels[key]) != value {
+			missing[key] = value
+		}
+	}
+	return missing
+}
+
+// longhornConditionByType runs one orchestration or CLI step.
+// Signature: longhornConditionByType(node longhornNode, condType string) *longhornCondition.
+// Why: Longhorn node condition parsing is shared by readiness and schedulability
+// checks and should remain case-insensitive.
+func longhornConditionByType(node longhornNode, condType string) *longhornCondition {
+	for i := range node.Status.Conditions {
+		if strings.EqualFold(strings.TrimSpace(node.Status.Conditions[i].Type), condType) {
+			return &node.Status.Conditions[i]
+		}
+	}
+	return nil
+}

internal/cluster/orchestrator_longhorn_recovery.go

@@ -104,25 +104,37 @@ func (o *Orchestrator) repairEncryptedVolumeMountPrereqs(ctx context.Context, po
 // Why: kubelet's encrypted Longhorn mount helper shells into the host namespace,
 // so the package must exist on the node host, not merely inside a workload pod.
 func (o *Orchestrator) ensureHostCryptsetup(ctx context.Context, node string) error {
-	command := strings.Join([]string{
+	checkCommand := strings.Join([]string{
 		"set -eu",
 		"if command -v cryptsetup >/dev/null 2>&1 || [ -x /usr/sbin/cryptsetup ] || [ -x /usr/bin/cryptsetup ]; then echo __ANANKE_CRYPTSETUP_PRESENT__; exit 0; fi",
 		"if ! command -v apt-get >/dev/null 2>&1; then echo __ANANKE_CRYPTSETUP_NO_APT__; exit 42; fi",
-		"sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update",
+		"echo __ANANKE_CRYPTSETUP_MISSING__",
-		"sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends cryptsetup-bin",
+		"exit 41",
-		"if command -v cryptsetup >/dev/null 2>&1 || [ -x /usr/sbin/cryptsetup ] || [ -x /usr/bin/cryptsetup ]; then echo __ANANKE_CRYPTSETUP_INSTALLED__; exit 0; fi",
-		"echo __ANANKE_CRYPTSETUP_INSTALL_FAILED__",
-		"exit 43",
 	}, "; ")
-	out, err := o.sshWithTimeout(ctx, node, command, 5*time.Minute)
+	out, err := o.sshWithTimeout(ctx, node, checkCommand, 25*time.Second)
+	if err == nil && strings.Contains(out, "__ANANKE_CRYPTSETUP_PRESENT__") {
+		o.log.Printf("ensured cryptsetup prerequisite on %s: __ANANKE_CRYPTSETUP_PRESENT__", node)
+		return nil
+	}
+	if strings.Contains(out, "__ANANKE_CRYPTSETUP_NO_APT__") {
+		return fmt.Errorf("install cryptsetup-bin: apt-get is unavailable on host")
+	}
+
+	out, err = o.runHostPrivilegedAction(ctx, node, hostActionInstallCryptsetup, 5*time.Minute)
 	if err != nil {
 		return fmt.Errorf("install cryptsetup-bin: %w (output=%s)", err, strings.TrimSpace(out))
 	}
-	trimmed := strings.TrimSpace(out)
+	if strings.Contains(out, "__ANANKE_CRYPTSETUP_INSTALLED__") {
-	o.log.Printf("ensured cryptsetup prerequisite on %s: %s", node, trimmed)
+		o.log.Printf("ensured cryptsetup prerequisite on %s: __ANANKE_CRYPTSETUP_INSTALLED__", node)
-	if strings.Contains(trimmed, "__ANANKE_CRYPTSETUP_INSTALLED__") {
 		o.noteStartupAutoHeal(fmt.Sprintf("installed cryptsetup on %s", node))
+		return nil
 	}
+	verifyOut, verifyErr := o.sshWithTimeout(ctx, node, checkCommand, 25*time.Second)
+	if verifyErr != nil || !strings.Contains(verifyOut, "__ANANKE_CRYPTSETUP_PRESENT__") {
+		return fmt.Errorf("cryptsetup-bin install completed but verification failed: %w (output=%s)", verifyErr, strings.TrimSpace(verifyOut))
+	}
+	o.log.Printf("ensured cryptsetup prerequisite on %s: __ANANKE_CRYPTSETUP_INSTALLED__", node)
+	o.noteStartupAutoHeal(fmt.Sprintf("installed cryptsetup on %s", node))
 	return nil
 }
 

internal/cluster/orchestrator_node_runtime_recovery.go

@@ -0,0 +1,162 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// recoverManagedNodeRuntime runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) recoverManagedNodeRuntime(ctx context.Context, node string, alreadyUnschedulable bool, reason string) error.
+// Why: kubelet/containerd wedges need one bounded escalation ladder: cordon,
+// non-blocking k3s-agent restart, health proof, and optional controlled reboot.
+func (o *Orchestrator) recoverManagedNodeRuntime(ctx context.Context, node string, alreadyUnschedulable bool, reason string) error {
+	node = strings.TrimSpace(node)
+	if node == "" {
+		return fmt.Errorf("node is empty")
+	}
+	if !alreadyUnschedulable {
+		if err := o.cordonNodeWithLease(ctx, node, cordonReasonKubeletProxy, reason); err != nil {
+			return fmt.Errorf("cordon before kubelet restart: %w", err)
+		}
+	}
+
+	o.log.Printf("warning: detected node runtime repair signal on %s; issuing non-blocking k3s-agent restart reason=%s", node, sanitizeCordonAnnotationValue(reason))
+	if _, err := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentRestart, 30*time.Second); err != nil {
+		if !alreadyUnschedulable {
+			o.bestEffort("uncordon node after failed kubelet proxy repair", func() error {
+				return o.uncordonAndClearCordonLease(ctx, node, cordonReasonKubeletProxy)
+			})
+		}
+		return fmt.Errorf("restart k3s-agent: %w", err)
+	}
+
+	readyErr := o.waitForManagedNodeRuntimeReady(ctx, node, o.nodeRuntimeRestartWait())
+	if readyErr == nil {
+		return o.releaseRuntimeRecoveryCordon(ctx, node, alreadyUnschedulable)
+	}
+
+	showOut, showErr := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentShow, 20*time.Second)
+	if showErr != nil {
+		return fmt.Errorf("%v; k3s-agent state check failed: %w", readyErr, showErr)
+	}
+	if !k3sAgentStateRequiresReboot(showOut) {
+		return readyErr
+	}
+	if !o.cfg.Startup.HostRepairAllowReboot {
+		return fmt.Errorf("node runtime requires controlled reboot but host_repair_allow_reboot is false; k3s-agent state=%s", summarizeSystemctlShow(showOut))
+	}
+
+	o.log.Printf("warning: escalating node runtime recovery to controlled reboot node=%s state=%s", node, summarizeSystemctlShow(showOut))
+	if _, err := o.runHostPrivilegedAction(ctx, node, hostActionHostReboot, 30*time.Second); err != nil {
+		return fmt.Errorf("controlled reboot command failed: %w", err)
+	}
+	if err := o.waitForManagedNodeRuntimeReady(ctx, node, o.nodeRuntimeRebootWait()); err != nil {
+		return fmt.Errorf("node did not recover after controlled reboot: %w", err)
+	}
+	return o.releaseRuntimeRecoveryCordon(ctx, node, alreadyUnschedulable)
+}
+
+// releaseRuntimeRecoveryCordon runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) releaseRuntimeRecoveryCordon(ctx context.Context, node string, alreadyUnschedulable bool) error.
+// Why: nodes Ananke cordoned for repair should return to service after proof of
+// recovery, while pre-existing cordons remain under their original owner.
+func (o *Orchestrator) releaseRuntimeRecoveryCordon(ctx context.Context, node string, alreadyUnschedulable bool) error {
+	if alreadyUnschedulable {
+		return nil
+	}
+	if err := o.uncordonAndClearCordonLease(ctx, node, cordonReasonKubeletProxy); err != nil {
+		return fmt.Errorf("uncordon after kubelet proxy repair: %w", err)
+	}
+	return nil
+}
+
+// nodeRuntimeRestartWait runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) nodeRuntimeRestartWait() time.Duration.
+// Why: k3s-agent restart recovery needs a bounded wait even when tests build a
+// partial startup config.
+func (o *Orchestrator) nodeRuntimeRestartWait() time.Duration {
+	seconds := o.cfg.Startup.NodeRuntimeRestartWaitSeconds
+	if seconds <= 0 {
+		seconds = 140
+	}
+	return time.Duration(seconds) * time.Second
+}
+
+// nodeRuntimeRebootWait runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) nodeRuntimeRebootWait() time.Duration.
+// Why: controlled reboot recovery has a longer but still finite postcondition
+// window for SSH, systemd, kubelet, and Kubernetes readiness.
+func (o *Orchestrator) nodeRuntimeRebootWait() time.Duration {
+	seconds := o.cfg.Startup.NodeRuntimeRebootWaitSeconds
+	if seconds <= 0 {
+		seconds = 420
+	}
+	return time.Duration(seconds) * time.Second
+}
+
+// waitForManagedNodeRuntimeReady runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) waitForManagedNodeRuntimeReady(ctx context.Context, node string, wait time.Duration) error.
+// Why: runtime repair is not complete until Kubernetes Ready, kubelet proxy, and
+// k3s-agent service state all agree the node is usable.
+func (o *Orchestrator) waitForManagedNodeRuntimeReady(ctx context.Context, node string, wait time.Duration) error {
+	if wait <= 0 {
+		wait = 140 * time.Second
+	}
+	kubeWait := wait - 20*time.Second
+	if kubeWait < 10*time.Second {
+		kubeWait = 10 * time.Second
+	}
+	if _, err := o.kubectl(ctx, wait, "wait", "node/"+node, "--for=condition=Ready", "--timeout="+fmt.Sprintf("%.0fs", kubeWait.Seconds())); err != nil {
+		return fmt.Errorf("wait Ready after k3s-agent restart: %w", err)
+	}
+	healthy, err := o.kubeletProxyHealthy(ctx, node)
+	if !healthy {
+		return fmt.Errorf("proxy still broken after k3s-agent restart: %v", err)
+	}
+	out, err := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentIsActive, 12*time.Second)
+	if err != nil {
+		return fmt.Errorf("k3s-agent active check: %w", err)
+	}
+	if strings.TrimSpace(out) != "active" {
+		return fmt.Errorf("k3s-agent is-active returned %q", strings.TrimSpace(out))
+	}
+	return nil
+}
+
+// k3sAgentStateRequiresReboot runs one orchestration or CLI step.
+// Signature: k3sAgentStateRequiresReboot(out string) bool.
+// Why: only the stuck systemd states observed during hard runtime wedges should
+// escalate from non-blocking restart to controlled reboot.
+func k3sAgentStateRequiresReboot(out string) bool {
+	lower := strings.ToLower(out)
+	needles := []string{
+		"activestate=deactivating",
+		"activestate=activating",
+		"substate=stop-sigterm",
+		"substate=stop-sigkill",
+		"substate=final-sigterm",
+		"substate=final-sigkill",
+		"substate=start",
+		"result=timeout",
+	}
+	for _, needle := range needles {
+		if strings.Contains(lower, needle) {
+			return true
+		}
+	}
+	return false
+}
+
+// summarizeSystemctlShow runs one orchestration or CLI step.
+// Signature: summarizeSystemctlShow(out string) string.
+// Why: operator-facing runtime incidents need compact systemd state summaries
+// rather than full multi-line command output.
+func summarizeSystemctlShow(out string) string {
+	lines := lines(out)
+	if len(lines) == 0 {
+		return "unknown"
+	}
+	return sanitizeCordonAnnotationValue(joinLimited(lines, 4))
+}

internal/cluster/orchestrator_poststart_exec.go

@@ -187,6 +187,81 @@ func (o *Orchestrator) ssh(ctx context.Context, node string, command string) (st
 // Signature: (o *Orchestrator) sshWithTimeout(ctx context.Context, node string, command string, timeout time.Duration) (string, error).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
 func (o *Orchestrator) sshWithTimeout(ctx context.Context, node string, command string, timeout time.Duration) (string, error) {
+	attempts, attemptNames, knownHostsFiles, repairHosts := o.sshCommandAttempts(node, command)
+
+	var lastOut string
+	var lastErr error
+	for i, args := range attempts {
+		out, err := o.run(ctx, timeout, "ssh", args...)
+		if err == nil {
+			if i > 0 {
+				o.log.Printf("warning: ssh %s path failed for %s, using %s path", attemptNames[i-1], node, attemptNames[i])
+			}
+			return out, nil
+		}
+		if sshutil.ShouldAttemptKnownHostsRepair(out, err) {
+			o.log.Printf("warning: ssh failure on %s via %s path may be host-key related; repairing known_hosts and retrying once", node, attemptNames[i])
+			sshutil.RepairKnownHosts(ctx, o.log, knownHostsFiles, repairHosts, o.cfg.SSHPort)
+			retryOut, retryErr := o.run(ctx, timeout, "ssh", args...)
+			if retryErr == nil {
+				return retryOut, nil
+			}
+			out = retryOut
+			err = retryErr
+		}
+		lastOut = out
+		lastErr = err
+		if i < len(attempts)-1 {
+			o.log.Printf("warning: ssh %s path failed for %s: %v; trying %s path", attemptNames[i], node, err, attemptNames[i+1])
+		}
+	}
+	return lastOut, lastErr
+}
+
+// sshWithInput runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) sshWithInput(ctx context.Context, node string, command string, input string, timeout time.Duration) (string, error).
+// Why: Vault-backed sudo must send passwords over stdin, not in the SSH command
+// line or logs, while still using Ananke's configured node route.
+func (o *Orchestrator) sshWithInput(ctx context.Context, node string, command string, input string, timeout time.Duration) (string, error) {
+	if o.sshInputOverride != nil {
+		return o.sshInputOverride(ctx, timeout, node, command, input)
+	}
+	attempts, attemptNames, knownHostsFiles, repairHosts := o.sshCommandAttempts(node, command)
+
+	var lastOut string
+	var lastErr error
+	for i, args := range attempts {
+		out, err := o.runSensitiveWithInput(ctx, timeout, input, "ssh", args...)
+		if err == nil {
+			if i > 0 {
+				o.log.Printf("warning: ssh %s path failed for %s, using %s path", attemptNames[i-1], node, attemptNames[i])
+			}
+			return out, nil
+		}
+		if sshutil.ShouldAttemptKnownHostsRepair(out, err) {
+			o.log.Printf("warning: ssh failure on %s via %s path may be host-key related; repairing known_hosts and retrying once", node, attemptNames[i])
+			sshutil.RepairKnownHosts(ctx, o.log, knownHostsFiles, repairHosts, o.cfg.SSHPort)
+			retryOut, retryErr := o.runSensitiveWithInput(ctx, timeout, input, "ssh", args...)
+			if retryErr == nil {
+				return retryOut, nil
+			}
+			out = retryOut
+			err = retryErr
+		}
+		lastOut = out
+		lastErr = err
+		if i < len(attempts)-1 {
+			o.log.Printf("warning: ssh %s path failed for %s: %v; trying %s path", attemptNames[i], node, err, attemptNames[i+1])
+		}
+	}
+	return lastOut, lastErr
+}
+
+// sshCommandAttempts runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) sshCommandAttempts(node string, command string) ([][]string, []string, []string, []string).
+// Why: ordinary SSH and stdin-backed SSH must use identical host, identity,
+// port, jump-host, and known-host repair behavior.
+func (o *Orchestrator) sshCommandAttempts(node string, command string) ([][]string, []string, []string, []string) {
 	host := node
 	if mapped, ok := o.cfg.SSHNodeHosts[node]; ok && strings.TrimSpace(mapped) != "" {
 		host = strings.TrimSpace(mapped)
@@ -240,34 +315,7 @@ func (o *Orchestrator) sshWithTimeout(ctx context.Context, node string, command
 	direct = append(direct, target, command)
 	attempts = append(attempts, direct)
 	attemptNames = append(attemptNames, "direct")
-
+	return attempts, attemptNames, knownHostsFiles, repairHosts
-	var lastOut string
-	var lastErr error
-	for i, args := range attempts {
-		out, err := o.run(ctx, timeout, "ssh", args...)
-		if err == nil {
-			if i > 0 {
-				o.log.Printf("warning: ssh %s path failed for %s, using %s path", attemptNames[i-1], node, attemptNames[i])
-			}
-			return out, nil
-		}
-		if sshutil.ShouldAttemptKnownHostsRepair(out, err) {
-			o.log.Printf("warning: ssh failure on %s via %s path may be host-key related; repairing known_hosts and retrying once", node, attemptNames[i])
-			sshutil.RepairKnownHosts(ctx, o.log, knownHostsFiles, repairHosts, o.cfg.SSHPort)
-			retryOut, retryErr := o.run(ctx, timeout, "ssh", args...)
-			if retryErr == nil {
-				return retryOut, nil
-			}
-			out = retryOut
-			err = retryErr
-		}
-		lastOut = out
-		lastErr = err
-		if i < len(attempts)-1 {
-			o.log.Printf("warning: ssh %s path failed for %s: %v; trying %s path", attemptNames[i], node, err, attemptNames[i+1])
-		}
-	}
-	return lastOut, lastErr
 }
 
 // run runs one orchestration or CLI step.
@@ -308,6 +356,31 @@ func (o *Orchestrator) runSensitive(ctx context.Context, timeout time.Duration,
 	return trimmed, nil
 }
 
+// runSensitiveWithInput runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) runSensitiveWithInput(ctx context.Context, timeout time.Duration, input string, name string, args ...string) (string, error).
+// Why: sudo passwords and similar secret stdin must not be passed through
+// ordinary command arguments or dry-run logging.
+func (o *Orchestrator) runSensitiveWithInput(ctx context.Context, timeout time.Duration, input string, name string, args ...string) (string, error) {
+	runCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(runCtx, name, args...)
+	cmd.Env = os.Environ()
+	if o.runner.Kubeconfig != "" {
+		cmd.Env = append(cmd.Env, "KUBECONFIG="+o.runner.Kubeconfig)
+	}
+	cmd.Stdin = strings.NewReader(input)
+	out, err := cmd.CombinedOutput()
+	trimmed := strings.TrimSpace(string(out))
+	if err != nil {
+		if trimmed == "" {
+			return "", fmt.Errorf("%s failed: %w", name, err)
+		}
+		return trimmed, fmt.Errorf("%s failed: %w", name, err)
+	}
+	return trimmed, nil
+}
+
 // lines runs one orchestration or CLI step.
 // Signature: lines(in string) []string.
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

internal/cluster/orchestrator_service_stability.go

@@ -99,6 +99,7 @@ func (o *Orchestrator) waitForServiceChecklist(ctx context.Context) error {
 	lastRecycleAttempt := time.Time{}
 	lastReplicaHeal := time.Time{}
 	lastIngressHeal := time.Time{}
+	lastTCPHeal := time.Time{}
 	for {
 		o.maybeAutoRecycleStuckPods(ctx, &lastRecycleAttempt)
 		o.maybeAutoHealCriticalWorkloadReplicas(ctx, &lastReplicaHeal)
@@ -110,6 +111,7 @@ func (o *Orchestrator) waitForServiceChecklist(ctx context.Context) error {
 			return nil
 		}
 		o.maybeAutoHealIngressHostBackends(ctx, &lastIngressHeal, lastFailure)
+		o.maybeAutoHealTCPServiceBackends(ctx, &lastTCPHeal)
 		if lastFailure != prevFailure || time.Since(lastLogged) >= 30*time.Second {
 			remaining := time.Until(deadline).Round(time.Second)
 			if remaining < 0 {
@@ -134,7 +136,7 @@ func (o *Orchestrator) waitForServiceChecklist(ctx context.Context) error {
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
 func (o *Orchestrator) serviceChecklistReady(ctx context.Context) (bool, string) {
 	checks := o.cfg.Startup.ServiceChecklist
-	if len(checks) == 0 {
+	if len(checks) == 0 && len(o.cfg.Startup.TCPServiceChecklist) == 0 {
 		return true, "no checklist items configured"
 	}
 	for _, check := range checks {
@@ -147,7 +149,10 @@ func (o *Orchestrator) serviceChecklistReady(ctx context.Context) (bool, string)
 			return false, fmt.Sprintf("%s: %s", name, detail)
 		}
 	}
-	return true, fmt.Sprintf("checks=%d", len(checks))
+	if ok, detail := o.tcpServiceChecklistReady(ctx); !ok {
+		return false, "tcp " + detail
+	}
+	return true, fmt.Sprintf("checks=%d tcp-checks=%d", len(checks), len(o.cfg.Startup.TCPServiceChecklist))
 }
 
 // serviceCheckReady runs one orchestration or CLI step.

internal/cluster/orchestrator_stale_rwo_owner.go

@@ -0,0 +1,319 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+type staleRWOOwnerDecision struct {
+	Reason      string
+	ForceDelete bool
+	Unsafe      bool
+}
+
+type persistentVolumeClaimList struct {
+	Items []persistentVolumeClaimResource `json:"items"`
+}
+
+type persistentVolumeClaimResource struct {
+	Metadata struct {
+		Namespace string `json:"namespace"`
+		Name      string `json:"name"`
+	} `json:"metadata"`
+	Spec struct {
+		AccessModes []string `json:"accessModes"`
+		VolumeName  string   `json:"volumeName"`
+	} `json:"spec"`
+}
+
+// staleRWOPVCOwnerDecisions finds stale terminating single-writer pods safely.
+// Signature: (o *Orchestrator) staleRWOPVCOwnerDecisions(ctx context.Context, pods podList, grace time.Duration) (map[string]staleRWOOwnerDecision, error).
+// Why: RWO volume handoff needs container/mount-level safety, not blind repeated
+// pod deletion, so sidecar-only stale owners can clear while live writers block.
+func (o *Orchestrator) staleRWOPVCOwnerDecisions(ctx context.Context, pods podList, grace time.Duration) (map[string]staleRWOOwnerDecision, error) {
+	eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query events for stale RWO owner scan: %w", err)
+	}
+	var events eventList
+	if strings.TrimSpace(eventsOut) != "" {
+		if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
+			return nil, fmt.Errorf("decode events for stale RWO owner scan: %w", err)
+		}
+	}
+	pvcs, err := o.queryPVCs(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	now := time.Now()
+	podsByController := map[string][]podResource{}
+	for _, pod := range pods.Items {
+		controllerKey := podControllerKey(pod)
+		if controllerKey == "" {
+			continue
+		}
+		podsByController[controllerKey] = append(podsByController[controllerKey], pod)
+	}
+
+	decisions := map[string]staleRWOOwnerDecision{}
+	for _, oldPod := range pods.Items {
+		oldKey := podKey(oldPod)
+		if oldKey == "" || oldPod.Metadata.DeletionTimestamp == nil || !podControllerOwned(oldPod) {
+			continue
+		}
+		if now.Sub(*oldPod.Metadata.DeletionTimestamp) < grace {
+			continue
+		}
+		oldNode := strings.TrimSpace(oldPod.Spec.NodeName)
+		if oldNode == "" {
+			continue
+		}
+		blockedClaims := podRWOPVCVolumeNames(oldPod, pvcs)
+		if len(blockedClaims) == 0 {
+			continue
+		}
+		controllerKey := podControllerKey(oldPod)
+		replacements := podsByController[controllerKey]
+		replacementKey := ""
+		for _, replacement := range replacements {
+			if podKey(replacement) == oldKey {
+				continue
+			}
+			if strings.TrimSpace(replacement.Spec.NodeName) == "" || strings.TrimSpace(replacement.Spec.NodeName) == oldNode {
+				continue
+			}
+			if !replacementPodPendingForAttach(replacement) {
+				continue
+			}
+			if !podHasBlockedAttachEvent(replacement, events) {
+				continue
+			}
+			if !podsShareAnyClaim(oldPod, replacement, blockedClaims) {
+				continue
+			}
+			replacementKey = podKey(replacement)
+			break
+		}
+		if replacementKey == "" {
+			continue
+		}
+
+		unsafe := runningContainersMountAnyVolume(oldPod, blockedClaims)
+		claims := make([]string, 0, len(blockedClaims))
+		for _, claim := range blockedClaims {
+			claims = append(claims, claim)
+		}
+		sort.Strings(claims)
+		if unsafe {
+			decisions[oldKey] = staleRWOOwnerDecision{
+				Reason: fmt.Sprintf("UnsafeStaleRWOPVCOwner:%s:%s", oldNode, strings.Join(claims, ",")),
+				Unsafe: true,
+			}
+			continue
+		}
+		decisions[oldKey] = staleRWOOwnerDecision{
+			Reason:      fmt.Sprintf("SidecarOnlyStaleRWOPVCOwner:%s:%s->%s", oldNode, strings.Join(claims, ","), replacementKey),
+			ForceDelete: true,
+		}
+	}
+	return decisions, nil
+}
+
+// queryPVCs runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) queryPVCs(ctx context.Context) (map[string]persistentVolumeClaimResource, error).
+// Why: stale-owner recovery must confirm single-writer PVC semantics before it
+// considers force-deleting a terminating pod object.
+func (o *Orchestrator) queryPVCs(ctx context.Context) (map[string]persistentVolumeClaimResource, error) {
+	out, err := o.kubectl(ctx, 30*time.Second, "get", "pvc", "-A", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query pvcs for stale RWO owner scan: %w", err)
+	}
+	pvcs := map[string]persistentVolumeClaimResource{}
+	if strings.TrimSpace(out) == "" {
+		return pvcs, nil
+	}
+	var list persistentVolumeClaimList
+	if err := json.Unmarshal([]byte(out), &list); err != nil {
+		return nil, fmt.Errorf("decode pvcs for stale RWO owner scan: %w", err)
+	}
+	for _, pvc := range list.Items {
+		key := strings.TrimSpace(pvc.Metadata.Namespace) + "/" + strings.TrimSpace(pvc.Metadata.Name)
+		if strings.TrimSpace(pvc.Metadata.Namespace) != "" && strings.TrimSpace(pvc.Metadata.Name) != "" {
+			pvcs[key] = pvc
+		}
+	}
+	return pvcs, nil
+}
+
+// podRWOPVCVolumeNames runs one orchestration or CLI step.
+// Signature: podRWOPVCVolumeNames(pod podResource, pvcs map[string]persistentVolumeClaimResource) map[string]string.
+// Why: container mount safety checks need the pod volume names that correspond
+// to ReadWriteOnce PVC claims.
+func podRWOPVCVolumeNames(pod podResource, pvcs map[string]persistentVolumeClaimResource) map[string]string {
+	volumes := map[string]string{}
+	for _, volume := range pod.Spec.Volumes {
+		if volume.PersistentVolumeClaim == nil {
+			continue
+		}
+		claim := strings.TrimSpace(volume.PersistentVolumeClaim.ClaimName)
+		if claim == "" {
+			continue
+		}
+		pvc, ok := pvcs[strings.TrimSpace(pod.Metadata.Namespace)+"/"+claim]
+		if !ok || !pvcSingleWriter(pvc) {
+			continue
+		}
+		volumes[strings.TrimSpace(volume.Name)] = claim
+	}
+	return volumes
+}
+
+// pvcSingleWriter runs one orchestration or CLI step.
+// Signature: pvcSingleWriter(pvc persistentVolumeClaimResource) bool.
+// Why: the stale-owner path is limited to exclusive-writer PVC modes and should
+// leave shared volumes to normal Kubernetes cleanup.
+func pvcSingleWriter(pvc persistentVolumeClaimResource) bool {
+	for _, mode := range pvc.Spec.AccessModes {
+		normalized := strings.ToLower(strings.TrimSpace(mode))
+		if normalized == "readwriteonce" || normalized == "readwriteoncepod" {
+			return true
+		}
+	}
+	return false
+}
+
+// podControllerKey runs one orchestration or CLI step.
+// Signature: podControllerKey(pod podResource) string.
+// Why: stale owner and replacement pods should be grouped by controller without
+// hard-coding ReplicaSet, StatefulSet, or application names.
+func podControllerKey(pod podResource) string {
+	ns := strings.TrimSpace(pod.Metadata.Namespace)
+	for _, owner := range pod.Metadata.OwnerReferences {
+		kind := strings.TrimSpace(owner.Kind)
+		name := strings.TrimSpace(owner.Name)
+		if ns != "" && kind != "" && name != "" {
+			return ns + "/" + kind + "/" + name
+		}
+	}
+	return ""
+}
+
+// podKey runs one orchestration or CLI step.
+// Signature: podKey(pod podResource) string.
+// Why: recovery incidents need stable namespace/name keys for pod maps and
+// operator summaries.
+func podKey(pod podResource) string {
+	ns := strings.TrimSpace(pod.Metadata.Namespace)
+	name := strings.TrimSpace(pod.Metadata.Name)
+	if ns == "" || name == "" {
+		return ""
+	}
+	return ns + "/" + name
+}
+
+// replacementPodPendingForAttach runs one orchestration or CLI step.
+// Signature: replacementPodPendingForAttach(pod podResource) bool.
+// Why: stale RWO recovery should only act when a replacement is actually blocked
+// during scheduling or initialization.
+func replacementPodPendingForAttach(pod podResource) bool {
+	phase := strings.TrimSpace(pod.Status.Phase)
+	if strings.EqualFold(phase, "Pending") {
+		return true
+	}
+	for _, st := range append(append([]podContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) {
+		if st.State.Waiting == nil {
+			continue
+		}
+		reason := strings.TrimSpace(st.State.Waiting.Reason)
+		if strings.EqualFold(reason, "ContainerCreating") || strings.EqualFold(reason, "PodInitializing") {
+			return true
+		}
+	}
+	return false
+}
+
+// podHasBlockedAttachEvent runs one orchestration or CLI step.
+// Signature: podHasBlockedAttachEvent(pod podResource, events eventList) bool.
+// Why: force deletion should be tied to concrete Multi-Attach or exclusive-use
+// evidence rather than any old terminating PVC pod.
+func podHasBlockedAttachEvent(pod podResource, events eventList) bool {
+	key := podKey(pod)
+	for _, event := range events.Items {
+		if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
+			continue
+		}
+		eventKey := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name)
+		if eventKey != key {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
+			continue
+		}
+		message := strings.ToLower(strings.TrimSpace(event.Message))
+		reason := strings.ToLower(strings.TrimSpace(event.Reason))
+		if reason == "failedattachvolume" ||
+			strings.Contains(message, "multi-attach") ||
+			strings.Contains(message, "volume is already used by pod") ||
+			strings.Contains(message, "already exclusively attached") {
+			return true
+		}
+	}
+	return false
+}
+
+// podsShareAnyClaim runs one orchestration or CLI step.
+// Signature: podsShareAnyClaim(oldPod podResource, replacement podResource, oldClaims map[string]string) bool.
+// Why: the old stale pod and replacement must be competing for the same PVC
+// before Ananke treats them as one storage handoff incident.
+func podsShareAnyClaim(oldPod podResource, replacement podResource, oldClaims map[string]string) bool {
+	replacementClaims := map[string]struct{}{}
+	for _, volume := range replacement.Spec.Volumes {
+		if volume.PersistentVolumeClaim == nil {
+			continue
+		}
+		claim := strings.TrimSpace(volume.PersistentVolumeClaim.ClaimName)
+		if claim != "" {
+			replacementClaims[claim] = struct{}{}
+		}
+	}
+	for _, claim := range oldClaims {
+		if _, ok := replacementClaims[claim]; ok {
+			return true
+		}
+	}
+	return false
+}
+
+// runningContainersMountAnyVolume runs one orchestration or CLI step.
+// Signature: runningContainersMountAnyVolume(pod podResource, volumeNames map[string]string) bool.
+// Why: a live app container mounting the blocked PVC is unsafe to clear, while
+// sidecars without that mount can be handled as stale API ownership.
+func runningContainersMountAnyVolume(pod podResource, volumeNames map[string]string) bool {
+	if len(volumeNames) == 0 {
+		return false
+	}
+	specByName := map[string]podContainer{}
+	for _, c := range pod.Spec.Containers {
+		specByName[strings.TrimSpace(c.Name)] = c
+	}
+	for _, status := range pod.Status.ContainerStatuses {
+		if status.State.Running == nil {
+			continue
+		}
+		spec, ok := specByName[strings.TrimSpace(status.Name)]
+		if !ok {
+			return true
+		}
+		for _, mount := range spec.VolumeMounts {
+			if _, ok := volumeNames[strings.TrimSpace(mount.Name)]; ok {
+				return true
+			}
+		}
+	}
+	return false
+}

internal/cluster/orchestrator_storage_types.go

@@ -244,6 +244,7 @@ type podResource struct {
 	Metadata struct {
 		Namespace         string            `json:"namespace"`
 		Name              string            `json:"name"`
+		Labels            map[string]string `json:"labels"`
 		Annotations       map[string]string `json:"annotations"`
 		CreationTimestamp time.Time         `json:"creationTimestamp"`
 		DeletionTimestamp *time.Time        `json:"deletionTimestamp"`
@@ -287,6 +288,8 @@ type podSpec struct {
 	NodeSelector   map[string]string `json:"nodeSelector"`
 	Affinity       *podAffinity      `json:"affinity"`
 	Volumes        []podVolume       `json:"volumes"`
+	Containers     []podContainer    `json:"containers"`
+	InitContainers []podContainer    `json:"initContainers"`
 }
 
 type podVolume struct {
@@ -294,6 +297,17 @@ type podVolume struct {
 	PersistentVolumeClaim *podPersistentVolumeClaim `json:"persistentVolumeClaim"`
 }
 
+type podContainer struct {
+	Name         string           `json:"name"`
+	Image        string           `json:"image"`
+	VolumeMounts []podVolumeMount `json:"volumeMounts"`
+}
+
+type podVolumeMount struct {
+	Name      string `json:"name"`
+	MountPath string `json:"mountPath"`
+}
+
 type podPersistentVolumeClaim struct {
 	ClaimName string `json:"claimName"`
 }

internal/cluster/orchestrator_tcp_service_check.go

@@ -0,0 +1,164 @@
+package cluster
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"sort"
+	"strings"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// tcpServiceChecklistReady runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) tcpServiceChecklistReady(ctx context.Context) (bool, string).
+// Why: SMTP, IMAP, and other non-HTTP services need protocol checks alongside
+// HTTP ingress checks before startup can call the cluster healthy.
+func (o *Orchestrator) tcpServiceChecklistReady(ctx context.Context) (bool, string) {
+	checks := o.cfg.Startup.TCPServiceChecklist
+	if len(checks) == 0 {
+		return true, "no tcp checklist items configured"
+	}
+	for _, check := range checks {
+		ok, detail := o.tcpServiceCheckReady(ctx, check)
+		if !ok {
+			name := strings.TrimSpace(check.Name)
+			if name == "" {
+				name = fmt.Sprintf("%s:%d", strings.TrimSpace(check.Host), check.Port)
+			}
+			return false, fmt.Sprintf("%s: %s", name, detail)
+		}
+	}
+	return true, fmt.Sprintf("tcp-checks=%d", len(checks))
+}
+
+// tcpServiceCheckReady runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) tcpServiceCheckReady(ctx context.Context, check config.TCPServiceChecklistCheck) (bool, string).
+// Why: a single mail protocol endpoint should report whether connect, TLS, and
+// banner expectations passed without exposing any configured secret material.
+func (o *Orchestrator) tcpServiceCheckReady(ctx context.Context, check config.TCPServiceChecklistCheck) (bool, string) {
+	banner, err := tcpServiceProbe(ctx, check)
+	if err != nil {
+		return false, err.Error()
+	}
+	expected := strings.TrimSpace(check.ExpectContains)
+	if expected != "" && !checklistContains(banner, expected) {
+		return false, fmt.Sprintf("banner missing expected marker %q", expected)
+	}
+	return true, "connected"
+}
+
+// tcpServiceProbe runs one orchestration or CLI step.
+// Signature: tcpServiceProbe(ctx context.Context, check config.TCPServiceChecklistCheck) (string, error).
+// Why: protocol probes need a small direct TCP/TLS implementation so Ananke can
+// validate Mailu without shelling out to nc or openssl.
+func tcpServiceProbe(ctx context.Context, check config.TCPServiceChecklistCheck) (string, error) {
+	timeout := time.Duration(check.TimeoutSeconds) * time.Second
+	if timeout <= 0 {
+		timeout = 8 * time.Second
+	}
+	address := net.JoinHostPort(strings.TrimSpace(check.Host), fmt.Sprintf("%d", check.Port))
+	dialer := net.Dialer{Timeout: timeout}
+	conn, err := dialer.DialContext(ctx, "tcp", address)
+	if err != nil {
+		return "", fmt.Errorf("connect %s failed: %w", address, err)
+	}
+	defer conn.Close()
+	deadline := time.Now().Add(timeout)
+	_ = conn.SetDeadline(deadline)
+
+	if check.TLS {
+		tlsConn := tls.Client(conn, &tls.Config{
+			ServerName:         strings.TrimSpace(check.Host),
+			InsecureSkipVerify: check.InsecureSkipTLS,
+		})
+		if err := tlsConn.HandshakeContext(ctx); err != nil {
+			return "", fmt.Errorf("tls handshake %s failed: %w", address, err)
+		}
+		conn = tlsConn
+		_ = conn.SetDeadline(deadline)
+	}
+
+	reader := bufio.NewReader(conn)
+	bannerParts := []string{}
+	line, _ := reader.ReadString('\n')
+	if strings.TrimSpace(line) != "" {
+		bannerParts = append(bannerParts, strings.TrimSpace(line))
+	}
+	if send := check.Send; send != "" {
+		if _, err := conn.Write([]byte(send)); err != nil {
+			return strings.Join(bannerParts, "\n"), fmt.Errorf("write probe command failed: %w", err)
+		}
+		line, _ = reader.ReadString('\n')
+		if strings.TrimSpace(line) != "" {
+			bannerParts = append(bannerParts, strings.TrimSpace(line))
+		}
+	}
+	return strings.Join(bannerParts, "\n"), nil
+}
+
+// maybeAutoHealTCPServiceBackends runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) maybeAutoHealTCPServiceBackends(ctx context.Context, lastAttempt *time.Time).
+// Why: failed Mailu protocol probes should drive the same backend repair loop as
+// HTTP services when a Kubernetes service hint is configured.
+func (o *Orchestrator) maybeAutoHealTCPServiceBackends(ctx context.Context, lastAttempt *time.Time) {
+	if o.runner.DryRun || len(o.cfg.Startup.TCPServiceChecklist) == 0 {
+		return
+	}
+	now := time.Now()
+	if lastAttempt != nil && !lastAttempt.IsZero() && now.Sub(*lastAttempt) < 45*time.Second {
+		return
+	}
+	if lastAttempt != nil {
+		*lastAttempt = now
+	}
+	healed, err := o.healFailedTCPServiceBackends(ctx)
+	if err != nil {
+		o.log.Printf("warning: tcp service backend auto-heal failed: %v", err)
+		return
+	}
+	if len(healed) == 0 {
+		return
+	}
+	sort.Strings(healed)
+	detail := fmt.Sprintf("restored tcp service backends: %s", joinLimited(healed, 8))
+	o.log.Printf("%s", detail)
+	o.noteStartupAutoHeal(detail)
+}
+
+// healFailedTCPServiceBackends runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) healFailedTCPServiceBackends(ctx context.Context) ([]string, error).
+// Why: failed TCP protocol checks need a direct backend-heal primitive that can
+// be reused by startup waits and post-start daemon repair.
+func (o *Orchestrator) healFailedTCPServiceBackends(ctx context.Context) ([]string, error) {
+	if o.runner.DryRun || len(o.cfg.Startup.TCPServiceChecklist) == 0 {
+		return nil, nil
+	}
+	healed := []string{}
+	attempted := map[string]struct{}{}
+	for _, check := range o.cfg.Startup.TCPServiceChecklist {
+		namespace := strings.TrimSpace(check.Namespace)
+		service := strings.TrimSpace(check.Service)
+		if namespace == "" || service == "" {
+			continue
+		}
+		key := namespace + "/" + service
+		if _, ok := attempted[key]; ok {
+			continue
+		}
+		ok, _ := o.tcpServiceCheckReady(ctx, check)
+		if ok {
+			continue
+		}
+		attempted[key] = struct{}{}
+		items, err := o.maybeHealCriticalEndpointBackends(ctx, namespace, service)
+		if err != nil {
+			return healed, fmt.Errorf("%s/%s: %w", namespace, service, err)
+		}
+		healed = append(healed, items...)
+	}
+	return healed, nil
+}

internal/cluster/orchestrator_vault_health.go

@@ -0,0 +1,74 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// vaultSealed runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error).
+// Why: treats exec and HTTP health as independent Vault signals so one flaky
+// kubectl exec does not block startup when HTTP health proves Vault is usable.
+func (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error) {
+	sealed, err := o.vaultSealedViaExec(ctx)
+	if err == nil {
+		return sealed, nil
+	}
+	httpSealed, httpErr := o.vaultSealedViaHTTP(ctx)
+	if httpErr == nil {
+		o.log.Printf("warning: vault exec status probe failed but HTTP health succeeded; continuing with HTTP health result: %v", err)
+		return httpSealed, nil
+	}
+	return false, fmt.Errorf("vault status check failed: exec=%v http=%v", err, httpErr)
+}
+
+// vaultSealedViaExec runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) vaultSealedViaExec(ctx context.Context) (bool, error).
+// Why: preserves the existing direct Vault CLI probe as the primary status
+// signal when kubectl exec is healthy.
+func (o *Orchestrator) vaultSealedViaExec(ctx context.Context) (bool, error) {
+	out, err := o.kubectl(
+		ctx,
+		25*time.Second,
+		"-n", "vault",
+		"exec", "vault-0", "--",
+		"sh", "-lc",
+		"VAULT_ADDR=http://127.0.0.1:8200 vault status -format=json 2>/dev/null || true",
+	)
+	if err != nil {
+		return false, fmt.Errorf("vault status check failed: %w", err)
+	}
+	sealed, err := parseVaultSealed(out)
+	if err != nil {
+		return false, fmt.Errorf("parse vault status: %w", err)
+	}
+	return sealed, nil
+}
+
+// vaultSealedViaHTTP runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) vaultSealedViaHTTP(ctx context.Context) (bool, error).
+// Why: gives startup a second authoritative Vault health path when exec is
+// transiently killed or the apiserver exec tunnel is degraded.
+func (o *Orchestrator) vaultSealedViaHTTP(ctx context.Context) (bool, error) {
+	out, err := o.kubectl(
+		ctx,
+		15*time.Second,
+		"-n", "vault",
+		"get",
+		"--raw",
+		"/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health",
+	)
+	if err != nil && strings.TrimSpace(out) == "" {
+		return false, fmt.Errorf("vault HTTP health check failed: %w", err)
+	}
+	sealed, parseErr := parseVaultSealed(out)
+	if parseErr != nil {
+		if err != nil {
+			return false, fmt.Errorf("vault HTTP health check failed: %w; parse: %v", err, parseErr)
+		}
+		return false, fmt.Errorf("parse vault HTTP health: %w", parseErr)
+	}
+	return sealed, nil
+}

internal/cluster/orchestrator_workload_convergence.go

@@ -188,6 +188,18 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error {
 	} else {
 		stalePhaseReasons = reasons
 	}
+	staleRWODecisions := map[string]staleRWOOwnerDecision{}
+	if decisions, scanErr := o.staleRWOPVCOwnerDecisions(ctx, list, grace); scanErr != nil {
+		o.log.Printf("warning: stale RWO PVC owner scan failed: %v", scanErr)
+	} else {
+		staleRWODecisions = decisions
+	}
+	imagePullDNSReasons := map[string]string{}
+	if reasons, scanErr := o.imagePullDNSBlockerReasons(ctx); scanErr != nil {
+		o.log.Printf("warning: image-pull DNS blocker scan failed: %v", scanErr)
+	} else {
+		imagePullDNSReasons = reasons
+	}
 	containerRuntimeWedgeReasons := map[string]string{}
 	if reasons, scanErr := o.containerRuntimeWedgePodReasons(ctx, list, grace); scanErr != nil {
 		o.log.Printf("warning: container runtime wedge scan failed: %v", scanErr)
@@ -219,6 +231,10 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error {
 			continue
 		}
 		reason := stuckContainerReason(pod, stuckReasons)
+		if (reason == "ImagePullBackOff" || reason == "ErrImagePull") && imagePullDNSReasons[ns+"/"+name] != "" {
+			o.log.Printf("warning: not recycling pod %s/%s because image pull is blocked by DNS/registry lookup: %s", ns, name, imagePullDNSReasons[ns+"/"+name])
+			continue
+		}
 		if reason == "" {
 			reason = stuckVaultInitReason(pod, grace)
 		}
@@ -231,6 +247,15 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error {
 		if reason == "" {
 			reason = stalePhaseReasons[ns+"/"+name]
 		}
+		if decision, ok := staleRWODecisions[ns+"/"+name]; ok && decision.Unsafe {
+			o.log.Printf("warning: stale RWO PVC owner is unsafe to force delete pod=%s/%s reason=%s", ns, name, decision.Reason)
+			continue
+		}
+		if reason == "" {
+			if decision, ok := staleRWODecisions[ns+"/"+name]; ok && decision.ForceDelete {
+				reason = decision.Reason
+			}
+		}
 		if runtimeReason := containerRuntimeWedgeReasons[ns+"/"+name]; runtimeReason != "" {
 			reason = runtimeReason
 		}
@@ -242,6 +267,9 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error {
 		}
 		deleteArgs := []string{"-n", ns, "delete", "pod", name, "--wait=false"}
 		forceDelete := staleControllerPodForceDeleteSafe(pod, grace)
+		if decision, ok := staleRWODecisions[ns+"/"+name]; ok && decision.ForceDelete {
+			forceDelete = true
+		}
 		if forceDelete {
 			deleteArgs = append(deleteArgs, "--grace-period=0", "--force")
 		}

internal/config/apply_defaults.go

@@ -129,11 +129,21 @@ func (c *Config) applyDefaults() {
 	} else {
 		c.Startup.ServiceChecklist = mergeServiceChecklistDefaults(c.Startup.ServiceChecklist, defaultServiceChecklist())
 	}
+	if c.Startup.TCPServiceChecklistExplicitOnly {
+		c.Startup.TCPServiceChecklist = mergeTCPServiceChecklistDefaults(c.Startup.TCPServiceChecklist, []TCPServiceChecklistCheck{})
+	} else {
+		c.Startup.TCPServiceChecklist = mergeTCPServiceChecklistDefaults(c.Startup.TCPServiceChecklist, defaultTCPServiceChecklist())
+	}
 	for i := range c.Startup.ServiceChecklist {
 		if c.Startup.ServiceChecklist[i].TimeoutSeconds <= 0 {
 			c.Startup.ServiceChecklist[i].TimeoutSeconds = 12
 		}
 	}
+	for i := range c.Startup.TCPServiceChecklist {
+		if c.Startup.TCPServiceChecklist[i].TimeoutSeconds <= 0 {
+			c.Startup.TCPServiceChecklist[i].TimeoutSeconds = 8
+		}
+	}
 	if c.Startup.CriticalServiceEndpointWaitSec <= 0 {
 		c.Startup.CriticalServiceEndpointWaitSec = 420
 	}
@@ -141,6 +151,9 @@ func (c *Config) applyDefaults() {
 		c.Startup.CriticalServiceEndpointPollSec = 5
 	}
 	c.Startup.CriticalServiceEndpoints = mergeStringDefaults(c.Startup.CriticalServiceEndpoints, defaultCriticalServiceEndpoints())
+	if c.Startup.CriticalServiceStartupProbeThreshold <= 0 {
+		c.Startup.CriticalServiceStartupProbeThreshold = 30
+	}
 	if c.Startup.IngressChecklistWaitSeconds <= 0 {
 		c.Startup.IngressChecklistWaitSeconds = 420
 	}
@@ -207,6 +220,18 @@ func (c *Config) applyDefaults() {
 	if c.Startup.DeadNodeCleanupGraceSeconds <= 0 {
 		c.Startup.DeadNodeCleanupGraceSeconds = 300
 	}
+	if strings.TrimSpace(c.Startup.HostSudoSecretPasswordKey) == "" {
+		c.Startup.HostSudoSecretPasswordKey = "password"
+	}
+	if c.Startup.HostPrivilegedCommandTimeoutSec <= 0 {
+		c.Startup.HostPrivilegedCommandTimeoutSec = 90
+	}
+	if c.Startup.NodeRuntimeRestartWaitSeconds <= 0 {
+		c.Startup.NodeRuntimeRestartWaitSeconds = 140
+	}
+	if c.Startup.NodeRuntimeRebootWaitSeconds <= 0 {
+		c.Startup.NodeRuntimeRebootWaitSeconds = 420
+	}
 	if strings.TrimSpace(c.Startup.VaultUnsealKeyFile) == "" {
 		c.Startup.VaultUnsealKeyFile = "/var/lib/ananke/vault-unseal.key"
 	}

internal/config/config_test.go

@@ -132,6 +132,71 @@ state:
 	}
 }
 
+// TestLoadDefaultsCriticalStartupProbeRepairWhenOmitted runs one orchestration or CLI step.
+// Signature: TestLoadDefaultsCriticalStartupProbeRepairWhenOmitted(t *testing.T).
+// Why: existing configs with a startup block should inherit the Mailu hardening
+// repair instead of silently decoding the new boolean as false.
+func TestLoadDefaultsCriticalStartupProbeRepairWhenOmitted(t *testing.T) {
+	tmp := t.TempDir()
+	cfgPath := filepath.Join(tmp, "ananke.yaml")
+	raw := `
+control_planes: [titan-0a, titan-0b, titan-0c]
+expected_flux_branch: main
+iac_repo_path: /opt/titan-iac
+startup:
+  api_wait_seconds: 900
+ups:
+  enabled: false
+state:
+  run_history_path: /tmp/runs.json
+  lock_path: /tmp/ananke.lock
+`
+	if err := os.WriteFile(cfgPath, []byte(strings.TrimSpace(raw)), 0o644); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+	cfg, err := Load(cfgPath)
+	if err != nil {
+		t.Fatalf("load config: %v", err)
+	}
+	if !cfg.Startup.CriticalServiceStartupProbeRepair {
+		t.Fatalf("expected omitted startup-probe repair setting to default true")
+	}
+	if cfg.Startup.CriticalServiceStartupProbeThreshold != 30 {
+		t.Fatalf("expected startup-probe threshold default 30, got %d", cfg.Startup.CriticalServiceStartupProbeThreshold)
+	}
+}
+
+// TestLoadHonorsExplicitCriticalStartupProbeRepairFalse runs one orchestration or CLI step.
+// Signature: TestLoadHonorsExplicitCriticalStartupProbeRepairFalse(t *testing.T).
+// Why: operators should still be able to disable the repair explicitly even
+// though omitted values inherit the hardening default.
+func TestLoadHonorsExplicitCriticalStartupProbeRepairFalse(t *testing.T) {
+	tmp := t.TempDir()
+	cfgPath := filepath.Join(tmp, "ananke.yaml")
+	raw := `
+control_planes: [titan-0a, titan-0b, titan-0c]
+expected_flux_branch: main
+iac_repo_path: /opt/titan-iac
+startup:
+  critical_service_startup_probe_repair: false
+ups:
+  enabled: false
+state:
+  run_history_path: /tmp/runs.json
+  lock_path: /tmp/ananke.lock
+`
+	if err := os.WriteFile(cfgPath, []byte(strings.TrimSpace(raw)), 0o644); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+	cfg, err := Load(cfgPath)
+	if err != nil {
+		t.Fatalf("load config: %v", err)
+	}
+	if cfg.Startup.CriticalServiceStartupProbeRepair {
+		t.Fatalf("expected explicit startup-probe repair false to be honored")
+	}
+}
+
 // TestValidateRejectsInvalidStartupShutdownCooldown runs one orchestration or CLI step.
 // Signature: TestValidateRejectsInvalidStartupShutdownCooldown(t *testing.T).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

internal/config/defaults.go

@@ -94,10 +94,13 @@ func defaults() Config {
 				AdminSecretPasswordKey: "password",
 			},
 			ServiceChecklist:                      defaultServiceChecklist(),
+			TCPServiceChecklist:                   defaultTCPServiceChecklist(),
 			RequireCriticalServiceEndpoints:       true,
 			CriticalServiceEndpointWaitSec:        420,
 			CriticalServiceEndpointPollSec:        5,
 			CriticalServiceEndpoints:              defaultCriticalServiceEndpoints(),
+			CriticalServiceStartupProbeRepair:     true,
+			CriticalServiceStartupProbeThreshold:  30,
 			RequireIngressChecklist:               true,
 			IngressChecklistWaitSeconds:           420,
 			IngressChecklistPollSeconds:           5,
@@ -122,6 +125,10 @@ func defaults() Config {
 			AutoRecycleStuckPods:                  true,
 			StuckPodGraceSeconds:                  180,
 			RecoveryCordonMaxSeconds:              3600,
+			HostSudoSecretPasswordKey:             "password",
+			HostPrivilegedCommandTimeoutSec:       90,
+			NodeRuntimeRestartWaitSeconds:         140,
+			NodeRuntimeRebootWaitSeconds:          420,
 			VaultUnsealKeyFile:                    "/var/lib/ananke/vault-unseal.key",
 			VaultUnsealBreakglassTimeout:          15,
 		},

internal/config/load.go

@@ -17,13 +17,52 @@ func Load(path string) (Config, error) {
 	if err != nil {
 		return Config{}, fmt.Errorf("read config %s: %w", path, err)
 	}
+	startupProbeRepairConfigured, err := yamlPathExists(b, "startup", "critical_service_startup_probe_repair")
+	if err != nil {
+		return Config{}, fmt.Errorf("decode config %s: %w", path, err)
+	}
 	if err := yaml.Unmarshal(b, &cfg); err != nil {
 		return Config{}, fmt.Errorf("decode config %s: %w", path, err)
 	}
 
 	cfg.applyDefaults()
+	if !startupProbeRepairConfigured {
+		cfg.Startup.CriticalServiceStartupProbeRepair = defaults().Startup.CriticalServiceStartupProbeRepair
+	}
 	if err := cfg.Validate(); err != nil {
 		return Config{}, err
 	}
 	return cfg, nil
 }
+
+// yamlPathExists runs one orchestration or CLI step.
+// Signature: yamlPathExists(raw []byte, path ...string) (bool, error).
+// Why: selected boolean defaults need to distinguish an omitted YAML key from an
+// explicit false value after nested startup mappings are decoded.
+func yamlPathExists(raw []byte, path ...string) (bool, error) {
+	var root yaml.Node
+	if err := yaml.Unmarshal(raw, &root); err != nil {
+		return false, err
+	}
+	node := &root
+	if node.Kind == yaml.DocumentNode && len(node.Content) > 0 {
+		node = node.Content[0]
+	}
+	for _, key := range path {
+		if node.Kind != yaml.MappingNode {
+			return false, nil
+		}
+		found := false
+		for i := 0; i+1 < len(node.Content); i += 2 {
+			if node.Content[i].Value == key {
+				node = node.Content[i+1]
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false, nil
+		}
+	}
+	return true, nil
+}

internal/config/startup_service_catalog.go

@@ -251,6 +251,57 @@ func defaultServiceChecklist() []ServiceChecklistCheck {
 	}
 }
 
+// defaultTCPServiceChecklist runs one orchestration or CLI step.
+// Signature: defaultTCPServiceChecklist() []TCPServiceChecklistCheck.
+// Why: mail and other non-HTTP edge services need protocol-level startup checks
+// so Ananke does not declare success while SMTP/IMAP ports are broken.
+func defaultTCPServiceChecklist() []TCPServiceChecklistCheck {
+	return []TCPServiceChecklistCheck{
+		{
+			Name:           "mail-smtp",
+			Host:           "mail.bstein.dev",
+			Port:           25,
+			Send:           "QUIT\r\n",
+			ExpectContains: "ESMTP",
+			TimeoutSeconds: 8,
+			Namespace:      "mailu-mailserver",
+			Service:        "mailu-front",
+		},
+		{
+			Name:           "mail-smtps",
+			Host:           "mail.bstein.dev",
+			Port:           465,
+			TLS:            true,
+			Send:           "QUIT\r\n",
+			ExpectContains: "ready",
+			TimeoutSeconds: 8,
+			Namespace:      "mailu-mailserver",
+			Service:        "mailu-front",
+		},
+		{
+			Name:           "mail-submission",
+			Host:           "mail.bstein.dev",
+			Port:           587,
+			Send:           "QUIT\r\n",
+			ExpectContains: "ready",
+			TimeoutSeconds: 8,
+			Namespace:      "mailu-mailserver",
+			Service:        "mailu-front",
+		},
+		{
+			Name:           "mail-imaps",
+			Host:           "mail.bstein.dev",
+			Port:           993,
+			TLS:            true,
+			Send:           "a001 LOGOUT\r\n",
+			ExpectContains: "OK",
+			TimeoutSeconds: 8,
+			Namespace:      "mailu-mailserver",
+			Service:        "mailu-front",
+		},
+	}
+}
+
 // defaultCriticalServiceEndpoints runs one orchestration or CLI step.
 // Signature: defaultCriticalServiceEndpoints() []string.
 // Why: service edge checks are insufficient for protected stacks; endpoint
@@ -263,6 +314,11 @@ func defaultCriticalServiceEndpoints() []string {
 		"logging/oauth2-proxy-logs",
 		"logging/opensearch-dashboards",
 		"logging/opensearch-master",
+		"mailu-mailserver/mailu-admin",
+		"mailu-mailserver/mailu-dovecot",
+		"mailu-mailserver/mailu-front",
+		"mailu-mailserver/mailu-postfix",
+		"mailu-mailserver/mailu-rspamd",
 	}
 }
 
@@ -301,6 +357,41 @@ func mergeServiceChecklistDefaults(existing, defaults []ServiceChecklistCheck) [
 	return out
 }
 
+// mergeTCPServiceChecklistDefaults runs one orchestration or CLI step.
+// Signature: mergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck.
+// Why: host configs should inherit new mail/TCP health checks while preserving
+// site-specific protocol probes.
+func mergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck {
+	if len(existing) == 0 {
+		out := make([]TCPServiceChecklistCheck, 0, len(defaults))
+		out = append(out, defaults...)
+		return out
+	}
+
+	defaultByName := map[string]struct{}{}
+	for _, check := range defaults {
+		name := strings.TrimSpace(check.Name)
+		if name == "" {
+			continue
+		}
+		defaultByName[name] = struct{}{}
+	}
+
+	out := make([]TCPServiceChecklistCheck, 0, len(defaults)+len(existing))
+	out = append(out, defaults...)
+	for _, check := range existing {
+		name := strings.TrimSpace(check.Name)
+		if name == "" {
+			continue
+		}
+		if _, exists := defaultByName[name]; exists {
+			continue
+		}
+		out = append(out, check)
+	}
+	return out
+}
+
 // mergeStringDefaults runs one orchestration or CLI step.
 // Signature: mergeStringDefaults(existing, defaults []string) []string.
 // Why: keeps baseline startup guards applied while preserving site-specific

internal/config/testing_hooks.go

@@ -18,6 +18,15 @@ func TestHookDefaultCriticalServiceEndpoints() []string {
 	return out
 }
 
+// TestHookDefaultTCPServiceChecklist runs one orchestration or CLI step.
+// Signature: TestHookDefaultTCPServiceChecklist() []TCPServiceChecklistCheck.
+// Why: exposes default TCP/mail service checks to top-level tests.
+func TestHookDefaultTCPServiceChecklist() []TCPServiceChecklistCheck {
+	out := make([]TCPServiceChecklistCheck, 0, len(defaultTCPServiceChecklist()))
+	out = append(out, defaultTCPServiceChecklist()...)
+	return out
+}
+
 // TestHookMergeServiceChecklistDefaults runs one orchestration or CLI step.
 // Signature: TestHookMergeServiceChecklistDefaults(existing, defaults []ServiceChecklistCheck) []ServiceChecklistCheck.
 // Why: exposes checklist merge helper to top-level tests.
@@ -31,3 +40,10 @@ func TestHookMergeServiceChecklistDefaults(existing, defaults []ServiceChecklist
 func TestHookMergeStringDefaults(existing, defaults []string) []string {
 	return mergeStringDefaults(existing, defaults)
 }
+
+// TestHookMergeTCPServiceChecklistDefaults runs one orchestration or CLI step.
+// Signature: TestHookMergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck.
+// Why: exposes TCP checklist merge helper to top-level tests.
+func TestHookMergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck {
+	return mergeTCPServiceChecklistDefaults(existing, defaults)
+}

internal/config/types.go

@@ -62,10 +62,14 @@ type Startup struct {
 	ServiceChecklistAuth                  ServiceChecklistAuthSettings `yaml:"service_checklist_auth"`
 	ServiceChecklistExplicitOnly          bool                         `yaml:"service_checklist_explicit_only"`
 	ServiceChecklist                      []ServiceChecklistCheck      `yaml:"service_checklist"`
+	TCPServiceChecklistExplicitOnly       bool                         `yaml:"tcp_service_checklist_explicit_only"`
+	TCPServiceChecklist                   []TCPServiceChecklistCheck   `yaml:"tcp_service_checklist"`
 	RequireCriticalServiceEndpoints       bool                         `yaml:"require_critical_service_endpoints"`
 	CriticalServiceEndpointWaitSec        int                          `yaml:"critical_service_endpoint_wait_seconds"`
 	CriticalServiceEndpointPollSec        int                          `yaml:"critical_service_endpoint_poll_seconds"`
 	CriticalServiceEndpoints              []string                     `yaml:"critical_service_endpoints"`
+	CriticalServiceStartupProbeRepair     bool                         `yaml:"critical_service_startup_probe_repair"`
+	CriticalServiceStartupProbeThreshold  int                          `yaml:"critical_service_startup_probe_failure_threshold"`
 	RequireIngressChecklist               bool                         `yaml:"require_ingress_checklist"`
 	IngressChecklistWaitSeconds           int                          `yaml:"ingress_checklist_wait_seconds"`
 	IngressChecklistPollSeconds           int                          `yaml:"ingress_checklist_poll_seconds"`
@@ -96,6 +100,13 @@ type Startup struct {
 	PostStartAutoHealSeconds              int                          `yaml:"post_start_auto_heal_seconds"`
 	RecoveryCordonMaxSeconds              int                          `yaml:"recovery_cordon_max_seconds"`
 	DeadNodeCleanupGraceSeconds           int                          `yaml:"dead_node_cleanup_grace_seconds"`
+	HostSudoSecretNamespace               string                       `yaml:"host_sudo_secret_namespace"`
+	HostSudoSecretNameTemplate            string                       `yaml:"host_sudo_secret_name_template"`
+	HostSudoSecretPasswordKey             string                       `yaml:"host_sudo_secret_password_key"`
+	HostRepairAllowReboot                 bool                         `yaml:"host_repair_allow_reboot"`
+	HostPrivilegedCommandTimeoutSec       int                          `yaml:"host_privileged_command_timeout_seconds"`
+	NodeRuntimeRestartWaitSeconds         int                          `yaml:"node_runtime_restart_wait_seconds"`
+	NodeRuntimeRebootWaitSeconds          int                          `yaml:"node_runtime_reboot_wait_seconds"`
 	VaultUnsealKeyFile                    string                       `yaml:"vault_unseal_key_file"`
 	VaultUnsealBreakglassCommand          string                       `yaml:"vault_unseal_breakglass_command"`
 	VaultUnsealBreakglassTimeout          int                          `yaml:"vault_unseal_breakglass_timeout_seconds"`
@@ -117,6 +128,19 @@ type ServiceChecklistCheck struct {
 	InsecureSkipTLS     bool   `yaml:"insecure_skip_tls"`
 }
 
+type TCPServiceChecklistCheck struct {
+	Name            string `yaml:"name"`
+	Host            string `yaml:"host"`
+	Port            int    `yaml:"port"`
+	TLS             bool   `yaml:"tls"`
+	Send            string `yaml:"send"`
+	ExpectContains  string `yaml:"expect_contains"`
+	TimeoutSeconds  int    `yaml:"timeout_seconds"`
+	InsecureSkipTLS bool   `yaml:"insecure_skip_tls"`
+	Namespace       string `yaml:"namespace"`
+	Service         string `yaml:"service"`
+}
+
 type ServiceChecklistAuthSettings struct {
 	Mode                   string `yaml:"mode"`
 	KeycloakBaseURL        string `yaml:"keycloak_base_url"`

internal/config/validate.go

@@ -203,6 +203,28 @@ func (c Config) Validate() error {
 			}
 		}
 	}
+	for i, check := range c.Startup.TCPServiceChecklist {
+		if strings.TrimSpace(check.Name) == "" {
+			return fmt.Errorf("config.startup.tcp_service_checklist[%d].name must not be empty", i)
+		}
+		if strings.TrimSpace(check.Host) == "" {
+			return fmt.Errorf("config.startup.tcp_service_checklist[%d].host must not be empty", i)
+		}
+		if strings.ContainsAny(check.Host, " \t\r\n/") {
+			return fmt.Errorf("config.startup.tcp_service_checklist[%d].host is invalid: %q", i, check.Host)
+		}
+		if check.Port <= 0 || check.Port > 65535 {
+			return fmt.Errorf("config.startup.tcp_service_checklist[%d].port must be in range 1-65535", i)
+		}
+		if check.TimeoutSeconds <= 0 {
+			return fmt.Errorf("config.startup.tcp_service_checklist[%d].timeout_seconds must be > 0", i)
+		}
+		if strings.TrimSpace(check.Namespace) != "" || strings.TrimSpace(check.Service) != "" {
+			if strings.TrimSpace(check.Namespace) == "" || strings.TrimSpace(check.Service) == "" {
+				return fmt.Errorf("config.startup.tcp_service_checklist[%d].namespace and service must be set together", i)
+			}
+		}
+	}
 	if c.Startup.CriticalServiceEndpointWaitSec <= 0 {
 		return fmt.Errorf("config.startup.critical_service_endpoint_wait_seconds must be > 0")
 	}
@@ -221,6 +243,9 @@ func (c Config) Validate() error {
 			return fmt.Errorf("config.startup.critical_service_endpoints entries must be namespace/service, got %q", entry)
 		}
 	}
+	if c.Startup.CriticalServiceStartupProbeRepair && c.Startup.CriticalServiceStartupProbeThreshold <= 0 {
+		return fmt.Errorf("config.startup.critical_service_startup_probe_failure_threshold must be > 0 when critical_service_startup_probe_repair is true")
+	}
 	if c.Startup.IngressChecklistWaitSeconds <= 0 {
 		return fmt.Errorf("config.startup.ingress_checklist_wait_seconds must be > 0")
 	}
@@ -286,6 +311,26 @@ func (c Config) Validate() error {
 	if c.Startup.DeadNodeCleanupGraceSeconds <= 0 {
 		return fmt.Errorf("config.startup.dead_node_cleanup_grace_seconds must be > 0")
 	}
+	if strings.TrimSpace(c.Startup.HostSudoSecretNamespace) != "" || strings.TrimSpace(c.Startup.HostSudoSecretNameTemplate) != "" {
+		if strings.TrimSpace(c.Startup.HostSudoSecretNamespace) == "" {
+			return fmt.Errorf("config.startup.host_sudo_secret_namespace must not be empty when host sudo secret lookup is configured")
+		}
+		if strings.TrimSpace(c.Startup.HostSudoSecretNameTemplate) == "" {
+			return fmt.Errorf("config.startup.host_sudo_secret_name_template must not be empty when host sudo secret lookup is configured")
+		}
+		if strings.TrimSpace(c.Startup.HostSudoSecretPasswordKey) == "" {
+			return fmt.Errorf("config.startup.host_sudo_secret_password_key must not be empty when host sudo secret lookup is configured")
+		}
+	}
+	if c.Startup.HostPrivilegedCommandTimeoutSec <= 0 {
+		return fmt.Errorf("config.startup.host_privileged_command_timeout_seconds must be > 0")
+	}
+	if c.Startup.NodeRuntimeRestartWaitSeconds <= 0 {
+		return fmt.Errorf("config.startup.node_runtime_restart_wait_seconds must be > 0")
+	}
+	if c.Startup.NodeRuntimeRebootWaitSeconds <= 0 {
+		return fmt.Errorf("config.startup.node_runtime_reboot_wait_seconds must be > 0")
+	}
 	for _, probe := range c.Startup.PostStartProbes {
 		if strings.TrimSpace(probe) == "" {
 			return fmt.Errorf("config.startup.post_start_probes entries must not be empty")

internal/config/validate_matrix_test.go

@@ -52,6 +52,21 @@ func TestValidateRejectsInvalidFieldsMatrix(t *testing.T) {
 		{"bad_service_checklist_http_code", func(c *Config) {
 			c.Startup.ServiceChecklist = []ServiceChecklistCheck{{Name: "x", URL: "https://ok", TimeoutSeconds: 5, AcceptedStatuses: []int{999}}}
 		}},
+		{"bad_tcp_service_checklist_name", func(c *Config) {
+			c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Host: "mail.bstein.dev", Port: 25, TimeoutSeconds: 5}}
+		}},
+		{"bad_tcp_service_checklist_host", func(c *Config) {
+			c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail/bad", Port: 25, TimeoutSeconds: 5}}
+		}},
+		{"bad_tcp_service_checklist_port", func(c *Config) {
+			c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail.bstein.dev", Port: 70000, TimeoutSeconds: 5}}
+		}},
+		{"bad_tcp_service_checklist_timeout", func(c *Config) {
+			c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail.bstein.dev", Port: 25}}
+		}},
+		{"bad_tcp_service_checklist_partial_backend", func(c *Config) {
+			c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail.bstein.dev", Port: 25, TimeoutSeconds: 5, Namespace: "mailu-mailserver"}}
+		}},
 		{"bad_critical_endpoint_wait", func(c *Config) { c.Startup.CriticalServiceEndpointWaitSec = 0 }},
 		{"bad_critical_endpoint_poll", func(c *Config) { c.Startup.CriticalServiceEndpointPollSec = 0 }},
 		{"bad_empty_critical_endpoints_when_required", func(c *Config) {
@@ -159,6 +174,9 @@ func TestApplyDefaultsPopulatesZeroConfig(t *testing.T) {
 	if len(cfg.Startup.CriticalServiceEndpoints) == 0 {
 		t.Fatalf("expected critical service endpoint defaults to be set")
 	}
+	if len(cfg.Startup.TCPServiceChecklist) == 0 {
+		t.Fatalf("expected TCP service checklist defaults to be set")
+	}
 	if cfg.Shutdown.SSHParallelism <= 0 || cfg.Shutdown.ScaleParallelism <= 0 || cfg.Shutdown.DrainParallelism <= 0 {
 		t.Fatalf("expected shutdown parallelism defaults to be set")
 	}

testing/config/config_quality_matrix_test.go

@@ -51,15 +51,33 @@ func TestHookServiceCatalogAndMergeContracts(t *testing.T) {
 		t.Fatalf("expected critical endpoint defaults")
 	}
 	foundMonitoring := false
+	foundMailAdmin := false
 	for _, entry := range critical {
 		if entry == "monitoring/grafana" {
 			foundMonitoring = true
-			break
+		}
+		if entry == "mailu-mailserver/mailu-admin" {
+			foundMailAdmin = true
 		}
 	}
 	if !foundMonitoring {
 		t.Fatalf("expected monitoring/grafana critical endpoint default")
 	}
+	if !foundMailAdmin {
+		t.Fatalf("expected mailu-mailserver/mailu-admin critical endpoint default")
+	}
+
+	tcpChecks := icfg.TestHookDefaultTCPServiceChecklist()
+	seenTCP := map[string]icfg.TCPServiceChecklistCheck{}
+	for _, check := range tcpChecks {
+		seenTCP[strings.TrimSpace(check.Name)] = check
+	}
+	if seenTCP["mail-smtp"].Port != 25 || seenTCP["mail-smtp"].Service != "mailu-front" {
+		t.Fatalf("expected mail-smtp TCP default to target Mailu front")
+	}
+	if !seenTCP["mail-imaps"].TLS || seenTCP["mail-imaps"].Port != 993 {
+		t.Fatalf("expected mail-imaps TCP default with TLS on 993")
+	}
 
 	mergedChecks := icfg.TestHookMergeServiceChecklistDefaults(
 		[]icfg.ServiceChecklistCheck{
@@ -75,6 +93,20 @@ func TestHookServiceCatalogAndMergeContracts(t *testing.T) {
 		t.Fatalf("expected 3 merged checks with dedupe, got %d", len(mergedChecks))
 	}
 
+	mergedTCP := icfg.TestHookMergeTCPServiceChecklistDefaults(
+		[]icfg.TCPServiceChecklistCheck{
+			{Name: "custom-tcp", Host: "custom.bstein.dev", Port: 1234, TimeoutSeconds: 5},
+			{Name: "mail-smtp", Host: "override.invalid", Port: 25, TimeoutSeconds: 5},
+		},
+		[]icfg.TCPServiceChecklistCheck{
+			{Name: "mail-smtp", Host: "mail.bstein.dev", Port: 25, TimeoutSeconds: 5},
+			{Name: "mail-imaps", Host: "mail.bstein.dev", Port: 993, TimeoutSeconds: 5},
+		},
+	)
+	if len(mergedTCP) != 3 {
+		t.Fatalf("expected 3 merged TCP checks with dedupe, got %d", len(mergedTCP))
+	}
+
 	mergedStrings := icfg.TestHookMergeStringDefaults(
 		[]string{" one ", "one", "", "two"},
 		[]string{"two", "three", " "},
@@ -84,6 +116,33 @@ func TestHookServiceCatalogAndMergeContracts(t *testing.T) {
 	}
 }
 
+// TestTitanCoordinatorConfigInheritsMailHardeningDefaults runs one orchestration or CLI step.
+// Signature: TestTitanCoordinatorConfigInheritsMailHardeningDefaults(t *testing.T).
+// Why: the deployed coordinator config has a startup block, so new Mailu recovery
+// defaults must still be present after YAML decoding and default merging.
+func TestTitanCoordinatorConfigInheritsMailHardeningDefaults(t *testing.T) {
+	cfg, err := icfg.Load(filepath.Join("..", "..", "configs", "ananke.titan-db.yaml"))
+	if err != nil {
+		t.Fatalf("load titan coordinator config: %v", err)
+	}
+	if !cfg.Startup.CriticalServiceStartupProbeRepair {
+		t.Fatalf("expected titan config to inherit critical startup-probe repair")
+	}
+	seenMailAdmin := false
+	seenMailRspamd := false
+	for _, entry := range cfg.Startup.CriticalServiceEndpoints {
+		if entry == "mailu-mailserver/mailu-admin" {
+			seenMailAdmin = true
+		}
+		if entry == "mailu-mailserver/mailu-rspamd" {
+			seenMailRspamd = true
+		}
+	}
+	if !seenMailAdmin || !seenMailRspamd {
+		t.Fatalf("expected titan config to include Mailu critical endpoints, admin=%v rspamd=%v", seenMailAdmin, seenMailRspamd)
+	}
+}
+
 // TestValidateServiceChecklistAuthContracts runs one orchestration or CLI step.
 // Signature: TestValidateServiceChecklistAuthContracts(t *testing.T).
 // Why: covers service-checklist auth and final-url validation branches that are

testing/hygiene/in_tree_test_allowlist.txt

@@ -4,6 +4,7 @@ cmd/ananke/bootstrap_handoff_additional_test.go
 cmd/ananke/bootstrap_handoff_test.go
 cmd/ananke/builder_test.go
 cmd/ananke/command_additional_test.go
+cmd/ananke/command_handlers_autoheal_test.go
 cmd/ananke/command_handlers_injection_test.go
 cmd/ananke/command_handlers_status_error_test.go
 cmd/ananke/command_handlers_test.go
@@ -16,9 +17,11 @@ internal/cluster/orchestrator_report_test.go
 internal/cluster/orchestrator_autorepair_test.go
 internal/cluster/orchestrator_autorepair_cleanup_test.go
 internal/cluster/orchestrator_autorepair_proxy_test.go
+internal/cluster/orchestrator_autorepair_service_test.go
 internal/cluster/orchestrator_critical_endpoint_additional_test.go
 internal/cluster/orchestrator_cordon_lease_test.go
 internal/cluster/orchestrator_test.go
+internal/cluster/orchestrator_hardening_test.go
 internal/cluster/orchestrator_unit_additional_test.go
 internal/cluster/orchestrator_workload_recovery_test.go
 internal/cluster/orchestrator_vault_test.go

testing/orchestrator/drill_lifecycle_test.go

@@ -147,9 +147,9 @@ func lifecycleDispatcher(recorder *commandRecorder) func(context.Context, time.D
 			case strings.Contains(command, "jsonpath={.spec.ref.branch}"):
 				return "main", nil
 			case strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
-				return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false,"timeout":"30s"},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}}]}`, nil
+				return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false,"timeout":"30s"},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false,"timeout":"30s"},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}}]}`, nil
 			case strings.Contains(command, "-n flux-system get kustomizations.kustomize.toolkit.fluxcd.io -o jsonpath="):
-				return "services\n", nil
+				return "flux-system\nservices\n", nil
 			case strings.Contains(command, "get helmreleases.helm.toolkit.fluxcd.io -A -o jsonpath="):
 				return "monitoring/grafana\n", nil
 			case strings.Contains(command, "annotate kustomizations.kustomize.toolkit.fluxcd.io"):

testing/orchestrator/drill_recovery_test.go

@@ -155,6 +155,7 @@ func TestLifecycleStartupFluxImmutableJobSelfHeal(t *testing.T) {
 	cfg := lifecycleConfig(t)
 	cfg.Startup.RequireCriticalServiceEndpoints = false
 	cfg.Startup.RequireWorkloadConvergence = false
+	cfg.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/services"}
 	cfg.Startup.FluxHealthWaitSeconds = 4
 	cfg.Startup.FluxHealthPollSeconds = 1
 
@@ -186,15 +187,15 @@ func TestLifecycleStartupFluxImmutableJobSelfHeal(t *testing.T) {
 			case strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
 				fluxCalls++
 				if fluxCalls <= 2 {
-					return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","message":"Job update failed: field is immutable"}]}}]}`, nil
+					return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","message":"Job update failed: field is immutable"}]}}]}`, nil
 				}
-				return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil
+				return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil
 			case strings.Contains(command, "get jobs -A -o json"):
 				return `{"items":[{"metadata":{"namespace":"flux-system","name":"reconcile-services","labels":{"kustomize.toolkit.fluxcd.io/name":"services"}},"status":{"failed":1,"conditions":[{"type":"Failed","status":"True"}]}}]}`, nil
 			case strings.Contains(command, "delete job reconcile-services"):
 				return "", nil
 			case strings.Contains(command, "-n flux-system get kustomizations.kustomize.toolkit.fluxcd.io -o jsonpath="):
-				return "services\n", nil
+				return "flux-system\nservices\n", nil
 			case strings.Contains(command, "get helmreleases.helm.toolkit.fluxcd.io -A -o jsonpath="):
 				return "", nil
 			case strings.Contains(command, "patch "), strings.Contains(command, "annotate "):

testing/orchestrator/hooks_access_deep_matrix_test.go

@@ -32,7 +32,7 @@ func TestHookAccessDeepMatrixReconcileAndSSHAuth(t *testing.T) {
 		}
 		orch, _ := newHookOrchestrator(t, cfg, run, run)
 		err := orch.TestHookReconcileNodeAccess(context.Background(), []string{"titan-db", "titan-23"})
-		if err == nil || !strings.Contains(err.Error(), "access validation had") || !strings.Contains(err.Error(), "missing sudo access") {
+		if err == nil || !strings.Contains(err.Error(), "access validation had 2 errors") || !strings.Contains(err.Error(), "host-privilege-unavailable") {
 			t.Fatalf("expected aggregated reconcile error, got %v", err)
 		}
 	})

testing/orchestrator/hooks_access_vault_lifecycle_matrix_test.go

@@ -241,7 +241,7 @@ func TestHookAccessVaultLifecycleMatrix(t *testing.T) {
 			case name == "kubectl" && strings.Contains(command, "get ingress -A -o json"):
 				return `{"items":[]}`, nil
 			case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
-				return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil
+				return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil
 			case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"):
 				return `{"items":[{"kind":"Deployment","metadata":{"namespace":"monitoring","name":"grafana"},"spec":{"replicas":1,"template":{"spec":{}}},"status":{"readyReplicas":1}}]}`, nil
 			case name == "kubectl" && strings.Contains(command, "get endpoints victoria-metrics-single-server"):

testing/orchestrator/hooks_branch_saturation_test.go

@@ -323,7 +323,7 @@ func TestHookStartupConvergenceAndStabilitySaturation(t *testing.T) {
 			case name == "kubectl" && strings.Contains(command, "get ingress -A -o json"):
 				return `{"items":[]}`, nil
 			case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
-				return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"Ready","message":"ok"}]}}]}`, nil
+				return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"Ready","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"Ready","message":"ok"}]}}]}`, nil
 			case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"):
 				return `{"items":[]}`, nil
 			case name == "kubectl" && strings.Contains(command, "get pods -A -o json"):

testing/orchestrator/hooks_health_failure_matrix_test.go

@@ -253,9 +253,17 @@ func TestHookFluxHealthFailureBranches(t *testing.T) {
 	t.Run("flux-health-ready-and-immutable-job-heal-branches", func(t *testing.T) {
 		cfg := lifecycleConfig(t)
 		cfg.Startup.IgnoreFluxKustomizations = []string{"infra/ignored"}
+		cfg.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/services"}
 
 		fluxItems := map[string]any{
 			"items": []map[string]any{
+				{
+					"metadata": map[string]any{"namespace": "flux-system", "name": "flux-system"},
+					"spec":     map[string]any{"suspend": false, "timeout": "30s"},
+					"status": map[string]any{"conditions": []map[string]any{
+						{"type": "Ready", "status": "True", "reason": "ReconciliationSucceeded", "message": "ok"},
+					}},
+				},
 				{
 					"metadata": map[string]any{"namespace": "infra", "name": "ignored"},
 					"spec":     map[string]any{"suspend": false, "timeout": "30s"},

testing/orchestrator/hooks_health_storage_test.go

@@ -23,6 +23,7 @@ func TestHookFluxHealthAndStorageBranches(t *testing.T) {
 	cfg.Startup.FluxHealthWaitSeconds = 2
 	cfg.Startup.FluxHealthPollSeconds = 1
 	cfg.Startup.IgnoreFluxKustomizations = []string{"flux-system/skip-me"}
+	cfg.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/services"}
 	cfg.Startup.StorageReadyWaitSeconds = 2
 	cfg.Startup.StorageReadyPollSeconds = 1
 	cfg.Startup.StorageMinReadyNodes = 1
@@ -38,9 +39,9 @@ func TestHookFluxHealthAndStorageBranches(t *testing.T) {
 			recorder.record(name, args)
 			fluxCalls++
 			if fluxCalls <= 1 {
-				return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","reason":"Unknown","message":"waiting"}]}}]}`, nil
+				return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","reason":"Unknown","message":"waiting"}]}}]}`, nil
 			}
-			return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}}]}`, nil
+			return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ReconciliationSucceeded","message":"ok"}]}}]}`, nil
 		case name == "kubectl" && strings.Contains(command, "get jobs -A -o json"):
 			recorder.record(name, args)
 			return `{"items":[]}`, nil

testing/orchestrator/hooks_lowfile_coverage_test.go

@@ -54,8 +54,8 @@ func TestHookLowFileCoverageBoost(t *testing.T) {
 		if !cluster.TestHookPodControllerOwned([]string{"DaemonSet"}) {
 			t.Fatalf("expected DaemonSet owner to be controller-owned")
 		}
-		if cluster.TestHookPodControllerOwned([]string{"Job"}) {
+		if !cluster.TestHookPodControllerOwned([]string{"Job"}) {
-			t.Fatalf("expected Job owner to be non controller-owned")
+			t.Fatalf("expected Job owner to be controller-owned")
 		}
 
 		if got := cluster.TestHookStuckContainerReason([]string{"ImagePullBackOff"}, nil, []string{"ImagePullBackOff"}); got != "ImagePullBackOff" {
@@ -370,7 +370,7 @@ func TestHookLowFileCoverageBoost(t *testing.T) {
 			command := name + " " + strings.Join(args, " ")
 			switch {
 			case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
-				return `{"items":[{"metadata":{"namespace":"flux-system","name":"ready"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil
+				return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"ready"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil
 			default:
 				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
 			}
@@ -382,7 +382,7 @@ func TestHookLowFileCoverageBoost(t *testing.T) {
 		if ready, detail, err := orchFlux.TestHookFluxHealthReady(context.Background()); err != nil || ready || !strings.Contains(detail, "not ready") {
 			t.Fatalf("expected flux health not-ready result, ready=%v detail=%q err=%v", ready, detail, err)
 		}
-		if ready, detail, err := orchNoTimeout.TestHookFluxHealthReady(context.Background()); err != nil || !ready || !strings.Contains(detail, "all kustomizations ready=") {
+		if ready, detail, err := orchNoTimeout.TestHookFluxHealthReady(context.Background()); err != nil || !ready || !strings.Contains(detail, "ready=") {
 			t.Fatalf("expected flux health ready result, ready=%v detail=%q err=%v", ready, detail, err)
 		}
 		if !cluster.TestHookLooksLikeImmutableJobError("Job update failed: field is immutable") {

testing/orchestrator/hooks_lowfunc_matrix_test.go

@@ -124,7 +124,7 @@ func TestHookServiceStabilityLowFunctionMatrix(t *testing.T) {
 		command := name + " " + strings.Join(args, " ")
 		switch {
 		case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
-			return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil
+			return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","message":"ok"}]}}]}`, nil
 		case name == "kubectl" && strings.Contains(command, "get deploy,statefulset,daemonset -A -o json"):
 			return `{"items":[]}`, nil
 		case name == "kubectl" && strings.Contains(command, "get pods -A -o json"):

testing/orchestrator/hooks_vault_lifecycle_branch_matrix_test.go

@@ -283,11 +283,12 @@ func TestHookVaultLifecycleBranchMatrix(t *testing.T) {
 	t.Run("flux-ingress-service-and-inventory-low-branches", func(t *testing.T) {
 		t.Run("flux-health-ready-reason-fallback-and-heal-query-error", func(t *testing.T) {
 			cfg := lifecycleConfig(t)
+			cfg.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/apps"}
 			runReady := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
 				command := name + " " + strings.Join(args, " ")
 				switch {
 				case name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json"):
-					return `{"items":[{"metadata":{"namespace":"","name":"skip"},"spec":{"suspend":false},"status":{"conditions":[]}},{"metadata":{"namespace":"flux-system","name":"apps"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","reason":"","message":""}]}}]}`, nil
+					return `{"items":[{"metadata":{"namespace":"","name":"skip"},"spec":{"suspend":false},"status":{"conditions":[]}},{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"Ready","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"apps"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"False","reason":"","message":""}]}}]}`, nil
 				case name == "kubectl" && strings.Contains(command, "get jobs -A -o json"):
 					return "", errors.New("jobs query failed")
 				default:

testing/orchestrator/hooks_vault_poststart_branch_matrix_test.go

@@ -226,10 +226,11 @@ func TestHookVaultPostStartBranchMatrix(t *testing.T) {
 			runMissingReady := func(ctx context.Context, timeout time.Duration, name string, args ...string) (string, error) {
 				command := name + " " + strings.Join(args, " ")
 				if name == "kubectl" && strings.Contains(command, "get kustomizations.kustomize.toolkit.fluxcd.io -A -o json") {
-					return `{"items":[{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Reconciling","status":"True","reason":"Progressing"}]}}]}`, nil
+					return `{"items":[{"metadata":{"namespace":"flux-system","name":"flux-system"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Ready","status":"True","reason":"Ready","message":"ok"}]}},{"metadata":{"namespace":"flux-system","name":"services"},"spec":{"suspend":false},"status":{"conditions":[{"type":"Reconciling","status":"True","reason":"Progressing"}]}}]}`, nil
 				}
 				return lifecycleDispatcher(&commandRecorder{})(ctx, timeout, name, args...)
 			}
+			cfg.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/services"}
 			orchMissingReady, _ := newHookOrchestrator(t, cfg, runMissingReady, runMissingReady)
 			ready, detail, err := orchMissingReady.TestHookFluxHealthReady(context.Background())
 			if err != nil || ready || !strings.Contains(detail, "ready condition missing") {