ci: omit null pod volumes before workspace injection

# Conflicts:
#	Jenkinsfile
#	scripts/publish_quality_metrics.py

.gitignore

@@ -1,4 +1,6 @@
 /bin/
+/build/
 /dist/
+internal/state/.corrupt-*
 *.log
 *.tmp

Jenkinsfile

@@ -1,41 +1,89 @@
 pipeline {
   agent {
     kubernetes {
-      label 'ananke-quality'
+      // Keep build I/O off the node's runtime USB drive.
+      workspaceVolume dynamicPVC(accessModes: 'ReadWriteOnce', requestsSize: '20Gi', storageClassName: 'ci-scratch')
       defaultContainer 'go-tester'
       yaml """
 apiVersion: v1
 kind: Pod
 spec:
+  securityContext:
+    fsGroup: 1000
+    fsGroupChangePolicy: OnRootMismatch
   nodeSelector:
+    hardware: rpi5
     kubernetes.io/arch: arm64
     node-role.kubernetes.io/worker: "true"
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+          - matchExpressions:
+              - {key: hardware, operator: In, values: [rpi5, rpi4]}
+              - {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
+              - {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-05, titan-06, titan-08, titan-11, titan-12, titan-13, titan-14, titan-15, titan-17, titan-18, titan-19]}
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 100
+          preference:
+            matchExpressions:
+              - key: kubernetes.io/hostname
+                operator: NotIn
+                values:
+                  - titan-13
+                  - titan-15
+                  - titan-17
+                  - titan-19
+  topologySpreadConstraints:
+    - maxSkew: 1
+      topologyKey: kubernetes.io/hostname
+      whenUnsatisfiable: ScheduleAnyway
+      labelSelector:
+        matchLabels:
+          jenkins/jenkins-jenkins-agent: "true"
   containers:
     - name: go-tester
-      image: golang:1.25-bookworm
+      image: registry.bstein.dev/bstein/golang:1.25-bookworm
       command: ["cat"]
       tty: true
       volumeMounts:
         - name: workspace-volume
           mountPath: /home/jenkins/agent
     - name: publisher
-      image: python:3.12-slim
+      image: registry.bstein.dev/bstein/python:3.12-slim
+      command: ["cat"]
+      tty: true
+      volumeMounts:
+        - name: workspace-volume
+          mountPath: /home/jenkins/agent
+    - name: quality-tools
+      image: registry.bstein.dev/bstein/quality-tools:sonar8.0.1-trivy0.70.0-db20260422-arm64
       command: ["cat"]
       tty: true
       volumeMounts:
         - name: workspace-volume
           mountPath: /home/jenkins/agent
-  volumes:
-    - name: workspace-volume
-      emptyDir: {}
 """
     }
   }
 
   environment {
+    PIP_CACHE_DIR = '/home/jenkins/agent/.cache/pip'
+    NPM_CONFIG_CACHE = '/home/jenkins/agent/.cache/npm'
+    SONAR_USER_HOME = '/home/jenkins/agent/.cache/sonar'
+    TMPDIR = '/home/jenkins/agent/.cache/tmp'
+    GOCACHE = '/home/jenkins/agent/.cache/go-build'
+    GOMODCACHE = '/home/jenkins/agent/.cache/go-mod'
+    GOTMPDIR = '/home/jenkins/agent/.cache/go-tmp'
     SUITE_NAME = 'ananke'
     PUSHGATEWAY_URL = 'http://platform-quality-gateway.monitoring.svc.cluster.local:9091'
+    SONARQUBE_HOST_URL = 'http://sonarqube.quality.svc.cluster.local:9000'
+    SONARQUBE_PROJECT_KEY = 'ananke'
+    SONARQUBE_TOKEN = credentials('sonarqube-token')
+    QUALITY_GATE_SONARQUBE_ENFORCE = '1'
     QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
+    QUALITY_GATE_IRONBANK_ENFORCE = '1'
+    QUALITY_GATE_IRONBANK_REQUIRED = '0'
     QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
   }
 
@@ -49,6 +97,11 @@ spec:
   }
 
   stages {
+    stage('Prepare build scratch') {
+      steps {
+        sh 'mkdir -p /home/jenkins/agent/.cache/tmp /home/jenkins/agent/.cache/go-tmp'
+      }
+    }
     stage('Checkout') {
       steps {
         checkout scm
@@ -57,6 +110,61 @@ spec:
 
     stage('Collect SonarQube evidence') {
       steps {
+        container('go-tester') {
+          sh '''#!/usr/bin/env bash
+            set -euo pipefail
+            mkdir -p build
+            export GOFLAGS='-buildvcs=false'
+            set +e
+            go test ./... -coverprofile=build/sonar-root.cover.out
+            root_rc=$?
+            (
+              cd testing
+              go test ./config -coverpkg=scm.bstein.dev/bstein/ananke/... -coverprofile=../build/sonar-testing-config.cover.out
+            )
+            config_rc=$?
+            (
+              cd testing
+              go test ./execx ./metrics ./hygiene ./orchestrator ./service ./state ./sshutil ./ups -coverpkg=scm.bstein.dev/bstein/ananke/... -coverprofile=../build/sonar-testing.cover.out
+            )
+            testing_rc=$?
+            set -e
+            coverage_rc=0
+            if [ "${root_rc}" -ne 0 ] || [ "${config_rc}" -ne 0 ] || [ "${testing_rc}" -ne 0 ]; then
+              coverage_rc=1
+            fi
+            if [ -s build/sonar-root.cover.out ] && [ -s build/sonar-testing-config.cover.out ] && [ -s build/sonar-testing.cover.out ]; then
+              {
+                head -n 1 build/sonar-root.cover.out
+                tail -n +2 build/sonar-root.cover.out
+                tail -n +2 build/sonar-testing-config.cover.out
+                tail -n +2 build/sonar-testing.cover.out
+              } > build/coverage.out
+            fi
+            printf '%s\n' "${coverage_rc}" > build/sonarqube-coverage.rc
+          '''
+        }
+        container('quality-tools') {
+          sh '''#!/usr/bin/env bash
+            set -euo pipefail
+            mkdir -p build
+            args=(
+              "-Dsonar.host.url=${SONARQUBE_HOST_URL}"
+              "-Dsonar.login=${SONARQUBE_TOKEN}"
+              "-Dsonar.projectKey=${SONARQUBE_PROJECT_KEY}"
+              "-Dsonar.projectName=${SONARQUBE_PROJECT_KEY}"
+              "-Dsonar.sources=."
+              "-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**,**/coverage/**,**/test-results/**,**/playwright-report/**"
+              "-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
+            )
+            [ -f build/coverage.out ] && args+=("-Dsonar.go.coverage.reportPaths=build/coverage.out")
+            set +e
+            sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
+            rc=${PIPESTATUS[0]}
+            set -e
+            printf '%s\n' "${rc}" > build/sonarqube-analysis.rc
+          '''
+        }
         container('publisher') {
           sh '''
             set -eu
@@ -65,23 +173,55 @@ spec:
 import base64
 import json
 import os
+import time
 import urllib.parse
 import urllib.request
+from pathlib import Path
 
 host = os.getenv('SONARQUBE_HOST_URL', '').strip().rstrip('/')
 project_key = os.getenv('SONARQUBE_PROJECT_KEY', '').strip()
 token = os.getenv('SONARQUBE_TOKEN', '').strip()
 report_path = os.getenv('QUALITY_GATE_SONARQUBE_REPORT', 'build/sonarqube-quality-gate.json')
 payload = {"status": "ERROR", "note": "missing SONARQUBE_HOST_URL and/or SONARQUBE_PROJECT_KEY"}
-if host and project_key:
+
-    query = urllib.parse.urlencode({"projectKey": project_key})
+def fetch_json(url):
-    request = urllib.request.Request(f"{host}/api/qualitygates/project_status?{query}", method="GET")
+    request = urllib.request.Request(url, method="GET")
     if token:
         encoded = base64.b64encode(f"{token}:".encode("utf-8")).decode("utf-8")
         request.add_header("Authorization", f"Basic {encoded}")
+    with urllib.request.urlopen(request, timeout=12) as response:
+        return json.loads(response.read().decode("utf-8"))
+
+if host and project_key:
+    analysis_id = ""
+    task_id = ""
+    task_file = Path(".scannerwork/report-task.txt")
+    if task_file.exists():
+        for line in task_file.read_text(encoding="utf-8", errors="ignore").splitlines():
+            key, _, value = line.partition("=")
+            if key == "ceTaskId":
+                task_id = value.strip()
+                break
     try:
-        with urllib.request.urlopen(request, timeout=12) as response:
+        if task_id:
-            payload = json.loads(response.read().decode("utf-8"))
+            for _ in range(24):
+                task_payload = fetch_json(
+                    f"{host}/api/ce/task?{urllib.parse.urlencode({'id': task_id})}"
+                )
+                task = task_payload.get("task", {})
+                task_status = str(task.get("status", "")).upper()
+                if task_status in {"SUCCESS", "FAILED", "CANCELED"}:
+                    analysis_id = str(task.get("analysisId", "")).strip()
+                    if task_status != "SUCCESS":
+                        payload = {"status": "ERROR", "error": f"sonarqube task {task_status.lower()}"}
+                    break
+                time.sleep(5)
+            else:
+                payload = {"status": "ERROR", "error": "timed out waiting for sonarqube task"}
+        if "error" not in payload:
+            params = {"analysisId": analysis_id} if analysis_id else {"projectKey": project_key}
+            query = urllib.parse.urlencode(params)
+            payload = fetch_json(f"{host}/api/qualitygates/project_status?{query}")
     except Exception as exc:  # noqa: BLE001
         payload = {"status": "ERROR", "error": str(exc)}
 with open(report_path, "w", encoding="utf-8") as handle:
@@ -95,6 +235,34 @@ PY
 
     stage('Collect Supply Chain evidence') {
       steps {
+        container('quality-tools') {
+          sh '''#!/usr/bin/env bash
+            set -euo pipefail
+            mkdir -p build
+            set +e
+            trivy fs --cache-dir "${TRIVY_CACHE_DIR}" --skip-db-update --timeout 5m --no-progress --format json --output build/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
+            trivy_rc=$?
+            set -e
+            if [ ! -s build/trivy-fs.json ]; then
+              cat > build/ironbank-compliance.json <<EOF
+{"status":"failed","compliant":false,"scanner":"trivy","scan_type":"filesystem","error":"trivy did not produce JSON output","trivy_rc":${trivy_rc}}
+EOF
+              exit 0
+            fi
+            critical="$(jq '[.Results[]? | .Vulnerabilities[]? | select(.Severity=="CRITICAL")] | length' build/trivy-fs.json)"
+            high="$(jq '[.Results[]? | .Vulnerabilities[]? | select(.Severity=="HIGH")] | length' build/trivy-fs.json)"
+            secrets="$(jq '[.Results[]? | .Secrets[]?] | length' build/trivy-fs.json)"
+            misconfigs="$(jq '[.Results[]? | .Misconfigurations[]? | select(.Status=="FAIL" and (.Severity=="CRITICAL" or .Severity=="HIGH"))] | length' build/trivy-fs.json)"
+            status=ok
+            compliant=true
+            if [ "${critical}" -gt 0 ] || [ "${secrets}" -gt 0 ] || [ "${misconfigs}" -gt 0 ]; then
+              status=failed
+              compliant=false
+            fi
+            jq -n --arg status "${status}" --argjson compliant "${compliant}" --argjson critical "${critical}" --argjson high "${high}" --argjson secrets "${secrets}" --argjson misconfigs "${misconfigs}" --argjson trivy_rc "${trivy_rc}" \
+              '{status:$status, compliant:$compliant, category:"artifact_security", scan_type:"filesystem", scanner:"trivy", critical_vulnerabilities:$critical, high_vulnerabilities:$high, secrets:$secrets, high_or_critical_misconfigurations:$misconfigs, trivy_rc:$trivy_rc, high_vulnerability_policy:"observe"}' > build/ironbank-compliance.json
+          '''
+        }
         container('publisher') {
           sh '''
             set -eu
@@ -152,13 +320,29 @@ PY
             failed_runs="$(awk -F= '$1=="failed"{print $2}' build/quality-gate.state 2>/dev/null | tail -n1)"
             [ -n "${ok_runs}" ] || ok_runs=0
             [ -n "${failed_runs}" ] || failed_runs=0
+            coverage_percent="$(python3 - <<'PY'
+import re
+from pathlib import Path
+
+coverage_path = Path("build/coverage-percent.txt")
+if coverage_path.exists():
+    print(coverage_path.read_text(encoding="utf-8", errors="ignore").strip() or "0.0")
+    raise SystemExit(0)
+log_path = Path("build/quality-gate.out")
+text = log_path.read_text(encoding="utf-8", errors="ignore") if log_path.exists() else ""
+values = [float(match.group(1)) for match in re.finditer(r"([0-9]+(?:\\.[0-9]+)?)%", text)]
+print(values[-1] if values else 0.0)
+PY
+)"
+            printf '%s\n' "${coverage_percent}" > build/coverage-percent.txt
             python3 scripts/publish_quality_metrics.py \
               --pushgateway-url "${PUSHGATEWAY_URL}" \
               --job-name platform-quality-ci \
               --suite "${SUITE_NAME}" \
               --trigger jenkins \
               --local-ok "${ok_runs}" \
-              --local-failed "${failed_runs}"
+              --local-failed "${failed_runs}" \
+              --coverage-percent-file build/coverage-percent.txt
           '''
         }
       }
@@ -169,7 +353,95 @@ PY
         container('publisher') {
           sh '''
             set -eu
-            test "$(cat build/quality-gate.rc 2>/dev/null || echo 1)" -eq 0
+            gate_rc="$(cat build/quality-gate.rc 2>/dev/null || echo 1)"
+            fail=0
+            if [ "${gate_rc}" -ne 0 ]; then
+              echo "quality gate failed with rc=${gate_rc}" >&2
+              fail=1
+            fi
+
+            enabled() {
+              case "$(printf '%s' "${1:-}" | tr '[:upper:]' '[:lower:]')" in
+                1|true|yes|on) return 0 ;;
+                *) return 1 ;;
+              esac
+            }
+
+            if enabled "${QUALITY_GATE_SONARQUBE_ENFORCE:-1}"; then
+              sonar_status="$(python3 - <<'PY'
+import json
+from pathlib import Path
+
+path = Path("build/sonarqube-quality-gate.json")
+if not path.exists():
+    print("missing")
+    raise SystemExit(0)
+try:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+except Exception:  # noqa: BLE001
+    print("error")
+    raise SystemExit(0)
+status = (payload.get("status") or payload.get("projectStatus", {}).get("status") or payload.get("qualityGate", {}).get("status") or "").strip().lower()
+print(status or "missing")
+PY
+)"
+              case "${sonar_status}" in
+                ok|pass|passed|success) ;;
+                *)
+                  echo "sonarqube gate failed: ${sonar_status}" >&2
+                  fail=1
+                  ;;
+              esac
+            fi
+
+            ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
+            if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
+              ironbank_required=1
+            fi
+            if enabled "${QUALITY_GATE_IRONBANK_ENFORCE:-1}"; then
+              supply_status="$(python3 - <<'PY'
+import json
+from pathlib import Path
+
+path = Path("build/ironbank-compliance.json")
+if not path.exists():
+    print("missing")
+    raise SystemExit(0)
+try:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+except Exception:  # noqa: BLE001
+    print("error")
+    raise SystemExit(0)
+compliant = payload.get("compliant")
+if compliant is True:
+    print("ok")
+elif compliant is False:
+    print("failed")
+else:
+    status = str(payload.get("status") or payload.get("result") or payload.get("compliance") or "").strip().lower()
+    print(status or "missing")
+PY
+)"
+              case "${supply_status}" in
+                ok|pass|passed|success|compliant) ;;
+                not_applicable|na|n/a)
+                  if enabled "${ironbank_required}"; then
+                    echo "supply chain gate required but status=${supply_status}" >&2
+                    fail=1
+                  fi
+                  ;;
+                *)
+                  if enabled "${ironbank_required}"; then
+                    echo "supply chain gate failed: ${supply_status}" >&2
+                    fail=1
+                  else
+                    echo "supply chain gate not passing (${supply_status}) but not required for this run" >&2
+                  fi
+                  ;;
+              esac
+            fi
+
+            exit "${fail}"
           '''
         }
       }
@@ -178,7 +450,7 @@ PY
 
   post {
     always {
-      archiveArtifacts artifacts: 'build/quality-gate.out,build/quality-gate.rc', allowEmptyArchive: true, fingerprint: true
+      archiveArtifacts artifacts: 'build/*.json,build/*.out,build/*.rc,build/*.txt,build/*.xml', allowEmptyArchive: true, fingerprint: true
     }
   }
 }

README.md

@@ -1,71 +1,41 @@
 # ananke
 
-Ananke is the host-side recovery orchestrator for Titan power events.
+Ananke gets Atlas back on its feet after power trouble.
 
-It runs outside Kubernetes (systemd on host), so it can:
+It runs on both tethys (in cluster - titan-24) and titan-db (out of cluster), outside Kubernetes as the host level, because some failures start before the cluster is healthy enough to fix itself. Its job is to bring nodes, Flux, core workloads, ingresses, and service checks back into a known-good state.
-- shut the cluster down gracefully before runtime gets dangerous
-- bootstrap the cluster after power is restored
-- break known startup deadlocks (including Flux + in-cluster Gitea coupling)
-- verify real service availability before declaring startup complete
 
-The goal is not clever automation. The goal is boring, repeatable recovery.
+It aspires to be boring software: do the checks, repair the known deadlocks, and stomp loudly when it exhausts is remedy library.
 
-## Why `ananke`
+## How it works
 
-In Greek myth, **Ananke** is inevitability and necessity.
+Ananke walks the cluster through startup or shutdown gates:
-That is the exact constraint we operate under during outages and drills.
 
-Power-domain names in this lab align with that naming:
+- confirm the expected nodes and SSH access
-- `Statera` UPS: `titan-23`, `titan-24`, `titan-jh`
+- check that Flux is looking at the right repo and branch
-- `Pyrphoros` UPS: all other nodes
+- wait for required Flux kustomizations and namespaces
+- repair known startup traps, including Harbor/Gitea/Flux coupling
+- run ingress, service, endpoint, and soak checks before calling startup done
 
-## Operating model (non-negotiable)
+Recovery cordons are given short 1hr leases. If Ananke cordons a node to repair something, it must either clear the cordon within the configured window or mark the node for manual action.
 
-- Ananke does **cluster orchestration**, not host power control.
+The following are notes for future Brad.
-- Shutdown defaults to `cluster-only` and should remain that way for normal drills.
-- Physical outages can cut host power themselves; Ananke’s job is clean state transitions.
 
-Flux source of truth remains `titan-iac.git`.
+## Environmental Dependencies
-Ananke’s own repo (`ananke.git`) is software only; it is not the desired-state cluster config repo.
 
-## Breakglass reminder
+Ananke should be one of the first things working. It does not need Harbor, Gitea, Longhorn, Grafana, or the apps to be healthy before it starts; their mess is what Ananke was born to sort out.
 
-Vault breakglass is available through a remote Magic Mirror path.
+Ananke needs:
-If standard unseal retrieval fails, use `startup.vault_unseal_breakglass_command`.
 
-## What "startup complete" means
+- an Ananke host that came up on its own: usually `titan-db` (for the arm64 ups) and `tethys`/`titan-24` (for amd64 ups)
+- `/etc/ananke/ananke.yaml`, the Ananke SSH key, and enough host ssh config to reach nodes on the Atlas SSH port
+- Kubernetes API access once the control plane is answering; before that it can only do host-side checks
+- Flux CRDs/controllers and the `titan-iac` source once the API is up, because most startup gates are Flux-shaped
+- basic node hygiene that Ananke cannot fake forever: SSH, sudo for managed repairs, sane clocks, and Longhorn host packages like `cryptsetup`, `open-iscsi`, `dmsetup`, and `nfs-common`
+- NUT/UPS access if this is making real shutdown decisions instead of just doing startup recovery
 
-Startup is complete only after all required gates pass:
+If this is a total bring-up, start Ananke after the host boots and before waiting on applications. If Ananke is not running, Atlas is missing the thing that knows the order of operations.
-- inventory mapping is valid
-- expected SSH nodes are reachable/authenticated (minus explicit ignores)
-- Flux source drift guard passes (expected URL + branch)
-- required Flux kustomizations are healthy
-- workload convergence is healthy
-- ingress checklist passes
-- service checklist passes (internal + externally exposed)
-- critical endpoint checks pass
-- stability soak passes with no regressions
 
-If a manual intervention is needed during a drill, that is treated as an Ananke gap and must be encoded back into Ananke logic.
+## Daily commands
-
-## Status and reports
-
-Live status:
-- `ananke status --config /etc/ananke/ananke.yaml`
-- `ananke status --config /etc/ananke/ananke.yaml --json`
-
-Artifacts:
-- `/var/lib/ananke/startup-progress.json` (live run progress)
-- `/var/lib/ananke/last-startup-report.json`
-- `/var/lib/ananke/last-shutdown-report.json`
-- `/var/lib/ananke/reports/*.json` (historical per-run reports)
-- `/var/lib/ananke/runs.json` (timing history)
-- `/var/lib/ananke/update-last.env` (latest self-update result)
-- `/var/log/ananke/update.log` (self-update execution log)
-
-## Quick commands
-
-From `titan-db`:
 
 ```bash
 sudo /usr/local/bin/ananke status --config /etc/ananke/ananke.yaml
@@ -73,70 +43,52 @@ sudo /usr/local/bin/ananke startup --config /etc/ananke/ananke.yaml --execute --
 sudo /usr/local/bin/ananke shutdown --config /etc/ananke/ananke.yaml --execute --reason graceful-maintenance --mode cluster-only
 ```
 
-From `titan-24` (`tethys` peer):
+Host files:
+
+- `/var/lib/ananke/startup-progress.json`
+- `/var/lib/ananke/last-startup-report.json`
+- `/var/lib/ananke/last-shutdown-report.json`
+- `/var/log/ananke/update.log`
+
+## Terraform-Generated Inventory
+
+Ananke can load a generated inventory fragment from `/etc/ananke/ananke.inventory.yaml` alongside `/etc/ananke/ananke.yaml`. The installer also accepts `ANANKE_INVENTORY_FRAGMENT=/path/to/ananke.inventory.yaml` and copies it into place.
+
+That fragment is for non-secret declarative inputs only: node hosts, managed nodes, control planes, workers, required node labels, and ignored unavailable nodes. When Terraform owns node labels, the fragment should set `startup.required_node_labels_mode: validate`; Ananke will report label drift but will not apply `kubectl label` during normal startup or post-start auto-heal.
+
+Keep recovery behavior in Ananke: UPS shutdown, SSH repair, k3s restart/reboot, Longhorn/runtime recovery, cordon/uncordon, pod recycling, and Flux suspend/resume remain imperative Ananke actions.
+
+## Development
+
+Local testing check before installing:
 
 ```bash
-sudo /usr/local/bin/ananke shutdown --config /etc/ananke/ananke.yaml --execute --reason graceful-maintenance --mode cluster-only
+./scripts/quality_gate.sh
 ```
 
-Systemd control:
+Emergency installs can bypass the gate with `ANANKE_ENFORCE_QUALITY_GATE=0` - try to avoid this. You should be treating failures as an instructive opportunity to improve Ananke.
+
+## Credential recovery and targeted updates
+
+Registry credential recovery requires a warning observed within ten minutes for
+an existing pod UID whose container or init container is still in ErrImagePull
+or ImagePullBackOff. Completed and deleting pods are ignored. A helper restart
+is reserved in the existing run history before execution and may occur at most
+once per helper in 30 minutes, including failed rollouts. History must remain
+writable; otherwise this repair fails closed. Only the coordinator daemon runs periodic shared-cluster repairs; the peer
+continues UPS monitoring and shutdown forwarding. Do not configure two hosts as
+coordinators for one cluster.
+
+A targeted daemon fix can use the existing installer without rewriting host
+configuration, UPS settings, or systemd/bootstrap units:
 
 ```bash
-sudo systemctl status ananke.service
+sudo ./scripts/install.sh --binary-only --skip-deps
-sudo systemctl start ananke-bootstrap.service
-sudo systemctl start ananke-update.service
-sudo cat /var/lib/ananke/update-last.env
-sudo tail -n 200 /var/log/ananke/update.log
 ```
 
-## Config
+The quality gate still runs. The previous binary is retained at
-
+`/usr/local/lib/ananke/rollback/ananke.previous`; a failed service start restores
-Primary config path:
+it. Verify service health and journals after installation. Automated self-updates
-- `/etc/ananke/ananke.yaml`
+now default to keeping the current binary when their quality gate fails. The
-
+service unit and updater script both set that default; existing hosts need those
-Keep these fields accurate:
+two files updated to adopt it.
-- `expected_flux_source_url`
-- `expected_flux_branch`
-- `startup.service_checklist`
-- `startup.critical_service_endpoints`
-- `startup.require_ingress_checklist`
-- `startup.require_node_inventory_reachability`
-- `startup.ignore_unavailable_nodes`
-- `coordination.role`
-- `coordination.peer_hosts`
-
-## Quality gate
-
-Top-level quality/testing module:
-- `testing/`
-
-Deployment gate script:
-- `scripts/quality_gate.sh`
-
-Gate order:
-1. docs contract checks
-2. split test-module contract (`cmd/` + `internal/` cannot grow new in-tree `_test.go` files)
-3. naming + LOC hygiene checks
-4. pedantic lint
-5. per-file coverage gate (95% minimum)
-
-Current migration rule:
-- keep new tests in the top-level `testing/` module
-- legacy in-tree `_test.go` files are temporarily grandfathered through `testing/hygiene/in_tree_test_allowlist.txt` until they are migrated safely
-
-Installer behavior:
-- `scripts/install.sh` runs the quality gate by default
-- override only for emergency break/fix: `ANANKE_ENFORCE_QUALITY_GATE=0`
-- host quality runs keep writing local `ananke_quality_gate_*` metrics and also publish `platform_quality_gate_runs_total{suite="ananke",status=*}` to Pushgateway for shared Grafana panels
-- override the Pushgateway target when running outside cluster DNS: `ANANKE_QUALITY_PUSHGATEWAY_URL=http://... ./scripts/quality_gate.sh`
-
-## Growing with the lab
-
-When adding nodes or services:
-1. Update inventory and node mapping in config.
-2. Add/adjust service checklist entries for anything user-facing or critical.
-3. Add/adjust ingress expectations for exposed services.
-4. Use temporary ignores only when truly intentional, then remove them.
-5. Run `scripts/quality_gate.sh` before host deployment.
-
-Recovery quality should improve over time: every drill should reduce manual work in the next drill.

cmd/ananke/builder.go

@@ -2,6 +2,9 @@ package main
 
 import (
 	"log"
+	"os"
+	"path/filepath"
+	"strings"
 
 	"scm.bstein.dev/bstein/ananke/internal/cluster"
 	"scm.bstein.dev/bstein/ananke/internal/config"
@@ -13,7 +16,7 @@ import (
 // Signature: buildOrchestrator(logger *log.Logger, cfgPath string, dryRun bool) (config.Config, *cluster.Orchestrator, error).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
 func buildOrchestrator(logger *log.Logger, cfgPath string, dryRun bool) (config.Config, *cluster.Orchestrator, error) {
-	cfg, err := config.Load(cfgPath)
+	cfg, err := config.LoadWithFragments(cfgPath, configFragmentPaths(cfgPath)...)
 	if err != nil {
 		return config.Config{}, nil, err
 	}
@@ -32,3 +35,44 @@ func buildOrchestrator(logger *log.Logger, cfgPath string, dryRun bool) (config.
 	orch := cluster.New(cfg, runner, store, logger)
 	return cfg, orch, nil
 }
+
+// configFragmentPaths runs one orchestration or CLI step.
+// Signature: configFragmentPaths(cfgPath string) []string.
+// Why: keeps generated inventory overlays discoverable without changing every
+// systemd command line or recovery script invocation.
+func configFragmentPaths(cfgPath string) []string {
+	paths := []string{}
+	if env := strings.TrimSpace(os.Getenv("ANANKE_CONFIG_FRAGMENTS")); env != "" {
+		for _, path := range filepath.SplitList(env) {
+			path = strings.TrimSpace(path)
+			if path != "" {
+				paths = append(paths, path)
+			}
+		}
+	}
+	defaultPath := defaultInventoryFragmentPath(cfgPath)
+	if defaultPath != "" {
+		if _, err := os.Stat(defaultPath); err == nil {
+			paths = append(paths, defaultPath)
+		}
+	}
+	return paths
+}
+
+// defaultInventoryFragmentPath runs one orchestration or CLI step.
+// Signature: defaultInventoryFragmentPath(cfgPath string) string.
+// Why: pairs ananke.yaml with an adjacent Terraform-generated inventory
+// fragment using a deterministic host-local convention.
+func defaultInventoryFragmentPath(cfgPath string) string {
+	dir := filepath.Dir(cfgPath)
+	base := filepath.Base(cfgPath)
+	ext := filepath.Ext(base)
+	name := strings.TrimSuffix(base, ext)
+	if name == "" {
+		return ""
+	}
+	if ext == "" {
+		return filepath.Join(dir, name+".inventory.yaml")
+	}
+	return filepath.Join(dir, name+".inventory"+ext)
+}

cmd/ananke/builder_test.go

@@ -33,6 +33,57 @@ func TestBuildOrchestratorCreatesStateDirs(t *testing.T) {
 	}
 }
 
+// TestBuildOrchestratorLoadsAdjacentInventoryFragment runs one orchestration or CLI step.
+// Signature: TestBuildOrchestratorLoadsAdjacentInventoryFragment(t *testing.T).
+// Why: the systemd default path must consume Terraform-generated inventory
+// without requiring extra command-line flags.
+func TestBuildOrchestratorLoadsAdjacentInventoryFragment(t *testing.T) {
+	cfgPath := writeTestConfig(t)
+	fragmentPath := filepath.Join(filepath.Dir(cfgPath), "ananke.inventory.yaml")
+	raw := `
+ssh_node_hosts:
+  titan-0a: 192.168.22.11
+  titan-05: 192.168.22.31
+ssh_managed_nodes: [titan-0a, titan-05]
+control_planes: [titan-0a]
+workers: [titan-05]
+startup:
+  required_node_labels_mode: validate
+  required_node_labels: {}
+  ignore_unavailable_nodes: []
+`
+	if err := os.WriteFile(fragmentPath, []byte(raw), 0o644); err != nil {
+		t.Fatalf("write inventory fragment: %v", err)
+	}
+
+	cfg, orch, err := buildOrchestrator(log.New(io.Discard, "", 0), cfgPath, true)
+	if err != nil {
+		t.Fatalf("buildOrchestrator failed: %v", err)
+	}
+	if orch == nil {
+		t.Fatalf("expected orchestrator")
+	}
+	if len(cfg.ControlPlanes) != 1 || cfg.ControlPlanes[0] != "titan-0a" {
+		t.Fatalf("expected adjacent fragment control plane, got %v", cfg.ControlPlanes)
+	}
+	if len(cfg.Workers) != 1 || cfg.Workers[0] != "titan-05" {
+		t.Fatalf("expected adjacent fragment workers, got %v", cfg.Workers)
+	}
+	if cfg.Startup.RequiredNodeLabelsMode != "validate" {
+		t.Fatalf("expected adjacent fragment label mode validate, got %q", cfg.Startup.RequiredNodeLabelsMode)
+	}
+}
+
+// TestBuildOrchestratorPropagatesLoadError runs one orchestration or CLI step.
+// Signature: TestBuildOrchestratorPropagatesLoadError(t *testing.T).
+// Why: missing host configs must fail before any recovery runner is built.
+func TestBuildOrchestratorPropagatesLoadError(t *testing.T) {
+	missing := filepath.Join(t.TempDir(), "missing.yaml")
+	if _, orch, err := buildOrchestrator(log.New(io.Discard, "", 0), missing, true); err == nil || orch != nil {
+		t.Fatalf("expected missing config to fail without orchestrator, orch=%v err=%v", orch, err)
+	}
+}
+
 // TestBuildOrchestratorFailsWhenStateDirIsFile runs one orchestration or CLI step.
 // Signature: TestBuildOrchestratorFailsWhenStateDirIsFile(t *testing.T).
 // Why: covers EnsureDir error path in orchestrator builder.
@@ -116,3 +167,59 @@ state:
 		t.Fatalf("expected buildOrchestrator failure when reports dir is a file")
 	}
 }
+
+// TestConfigFragmentPathsUsesEnvAndAdjacentInventory runs one orchestration or CLI step.
+// Signature: TestConfigFragmentPathsUsesEnvAndAdjacentInventory(t *testing.T).
+// Why: host installs should be able to consume Terraform-generated inventory
+// without changing the systemd command line.
+func TestConfigFragmentPathsUsesEnvAndAdjacentInventory(t *testing.T) {
+	tmp := t.TempDir()
+	cfgPath := filepath.Join(tmp, "ananke.yaml")
+	defaultFragment := filepath.Join(tmp, "ananke.inventory.yaml")
+	envFragment := filepath.Join(tmp, "env-fragment.yaml")
+	if err := os.WriteFile(defaultFragment, []byte("ssh_managed_nodes: []\n"), 0o644); err != nil {
+		t.Fatalf("write default fragment: %v", err)
+	}
+	t.Setenv("ANANKE_CONFIG_FRAGMENTS", envFragment)
+
+	paths := configFragmentPaths(cfgPath)
+	if len(paths) != 2 {
+		t.Fatalf("expected env and adjacent fragments, got %v", paths)
+	}
+	if paths[0] != envFragment || paths[1] != defaultFragment {
+		t.Fatalf("unexpected fragment order: %v", paths)
+	}
+}
+
+// TestConfigFragmentPathsSkipsMissingAndBlankEntries runs one orchestration or CLI step.
+// Signature: TestConfigFragmentPathsSkipsMissingAndBlankEntries(t *testing.T).
+// Why: optional generated fragments should be discovered deterministically without
+// turning an absent adjacent file into a startup blocker.
+func TestConfigFragmentPathsSkipsMissingAndBlankEntries(t *testing.T) {
+	tmp := t.TempDir()
+	envFragment := filepath.Join(tmp, "env.yaml")
+	t.Setenv("ANANKE_CONFIG_FRAGMENTS", string(os.PathListSeparator)+envFragment+string(os.PathListSeparator)+" ")
+
+	paths := configFragmentPaths(filepath.Join(tmp, "ananke.yaml"))
+	if len(paths) != 1 || paths[0] != envFragment {
+		t.Fatalf("expected only env fragment, got %v", paths)
+	}
+}
+
+// TestDefaultInventoryFragmentPathVariants runs one orchestration or CLI step.
+// Signature: TestDefaultInventoryFragmentPathVariants(t *testing.T).
+// Why: generated inventory naming should stay stable for config paths with and
+// without a YAML extension.
+func TestDefaultInventoryFragmentPathVariants(t *testing.T) {
+	tmp := t.TempDir()
+	cases := map[string]string{
+		filepath.Join(tmp, "ananke.yaml"): filepath.Join(tmp, "ananke.inventory.yaml"),
+		filepath.Join(tmp, "ananke"):      filepath.Join(tmp, "ananke.inventory.yaml"),
+		".":                               "",
+	}
+	for in, want := range cases {
+		if got := defaultInventoryFragmentPath(in); got != want {
+			t.Fatalf("defaultInventoryFragmentPath(%q)=%q want %q", in, got, want)
+		}
+	}
+}

cmd/ananke/command_handlers.go

@@ -56,6 +56,9 @@ var (
 	etcdRestoreOrchestratorCommand = func(ctx context.Context, orch *cluster.Orchestrator, opts cluster.EtcdRestoreOptions) error {
 		return orch.EtcdRestore(ctx, opts)
 	}
+	autoHealOrchestratorCommand = func(ctx context.Context, orch *cluster.Orchestrator) error {
+		return orch.RunPostStartAutoHeal(ctx)
+	}
 	daemonRunCommand   = func(ctx context.Context, daemon *service.Daemon) error { return daemon.Run(ctx) }
 	readIntentCommand  = state.ReadIntent
 	writeIntentCommand = state.MustWriteIntent
@@ -132,6 +135,25 @@ func runStartup(logger *log.Logger, args []string) error {
 	})
 }
 
+// runAutoHeal runs one orchestration or CLI step.
+// Signature: runAutoHeal(logger *log.Logger, args []string) error.
+// Why: operators need a narrow way to run Ananke's daemon repair pass once
+// without invoking a full startup or depending on the UPS daemon loop.
+func runAutoHeal(logger *log.Logger, args []string) error {
+	fs := flag.NewFlagSet("auto-heal", flag.ExitOnError)
+	configPath := fs.String("config", "/etc/ananke/ananke.yaml", "Path to config file")
+	execute := fs.Bool("execute", false, "Actually execute changes (default dry-run)")
+	_ = fs.Parse(args)
+
+	_, orch, err := buildOrchestratorCommand(logger, *configPath, !*execute)
+	if err != nil {
+		return err
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	return autoHealOrchestratorCommand(ctx, orch)
+}
+
 // runShutdown runs one orchestration or CLI step.
 // Signature: runShutdown(logger *log.Logger, args []string) error.
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

cmd/ananke/command_handlers_autoheal_test.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log"
+	"strings"
+	"testing"
+
+	"scm.bstein.dev/bstein/ananke/internal/cluster"
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestRunAutoHealInvokesPostStartRepair runs one orchestration or CLI step.
+// Signature: TestRunAutoHealInvokesPostStartRepair(t *testing.T).
+// Why: the one-shot auto-heal command must call the same repair entrypoint as
+// the daemon loop so operators can run Ananke's post-start repairs directly.
+func TestRunAutoHealInvokesPostStartRepair(t *testing.T) {
+	restore := stubCommandHandlerHooks()
+	defer restore()
+
+	called := false
+	buildOrchestratorCommand = func(_ *log.Logger, _ string, dryRun bool) (config.Config, *cluster.Orchestrator, error) {
+		cfg := minimalHandlerConfig(t)
+		return cfg, newTestOrchestrator(cfg, dryRun), nil
+	}
+	autoHealOrchestratorCommand = func(_ context.Context, orch *cluster.Orchestrator) error {
+		called = true
+		if orch == nil {
+			t.Fatalf("expected orchestrator")
+		}
+		return nil
+	}
+	logger := log.New(io.Discard, "", 0)
+	if err := runAutoHeal(logger, []string{"--config", "/ignored.yaml", "--execute"}); err != nil {
+		t.Fatalf("expected auto-heal success, got %v", err)
+	}
+	if !called {
+		t.Fatalf("expected auto-heal orchestrator hook to be called")
+	}
+}
+
+// TestDefaultAutoHealWrapperHonorsDryRunOrchestrator runs one orchestration or CLI step.
+// Signature: TestDefaultAutoHealWrapperHonorsDryRunOrchestrator(t *testing.T).
+// Why: the default command hook should keep pointing at the orchestrator method,
+// not only at test-injected replacements.
+func TestDefaultAutoHealWrapperHonorsDryRunOrchestrator(t *testing.T) {
+	cfg := minimalHandlerConfig(t)
+	orch := newTestOrchestrator(cfg, true)
+	if err := autoHealOrchestratorCommand(context.Background(), orch); err != nil {
+		t.Fatalf("default auto-heal wrapper failed: %v", err)
+	}
+}
+
+// TestRunAutoHealBuildError runs one orchestration or CLI step.
+// Signature: TestRunAutoHealBuildError(t *testing.T).
+// Why: command handlers should return config/build failures before invoking
+// repair actions.
+func TestRunAutoHealBuildError(t *testing.T) {
+	restore := stubCommandHandlerHooks()
+	defer restore()
+	buildOrchestratorCommand = func(_ *log.Logger, _ string, _ bool) (config.Config, *cluster.Orchestrator, error) {
+		return config.Config{}, nil, errors.New("build failed")
+	}
+	err := runAutoHeal(log.New(io.Discard, "", 0), []string{"--config", "/bad.yaml"})
+	if err == nil || !strings.Contains(err.Error(), "build failed") {
+		t.Fatalf("expected build failure, got %v", err)
+	}
+}

cmd/ananke/command_handlers_injection_test.go

@@ -403,6 +403,7 @@ func stubCommandHandlerHooks() func() {
 	prevStartup := startupOrchestratorCommand
 	prevShutdown := shutdownOrchestratorCommand
 	prevRestore := etcdRestoreOrchestratorCommand
+	prevAutoHeal := autoHealOrchestratorCommand
 	prevDaemon := daemonRunCommand
 	prevReadIntent := readIntentCommand
 	prevWriteIntent := writeIntentCommand
@@ -416,6 +417,7 @@ func stubCommandHandlerHooks() func() {
 		startupOrchestratorCommand = prevStartup
 		shutdownOrchestratorCommand = prevShutdown
 		etcdRestoreOrchestratorCommand = prevRestore
+		autoHealOrchestratorCommand = prevAutoHeal
 		daemonRunCommand = prevDaemon
 		readIntentCommand = prevReadIntent
 		writeIntentCommand = prevWriteIntent

cmd/ananke/main_dispatch.go

@@ -11,6 +11,7 @@ var (
 	shutdownCommand    = runShutdown
 	etcdRestoreCommand = runEtcdRestore
 	daemonCommand      = runDaemon
+	autoHealCommand    = runAutoHeal
 	statusCommand      = runStatus
 	intentCommand      = runIntent
 )
@@ -54,6 +55,11 @@ func runMain(logger *log.Logger, args []string) int {
 			logger.Printf("daemon failed: %v", err)
 			return 1
 		}
+	case "auto-heal":
+		if err := autoHealCommand(logger, args[2:]); err != nil {
+			logger.Printf("auto-heal failed: %v", err)
+			return 1
+		}
 	case "status":
 		if err := statusCommand(logger, args[2:]); err != nil {
 			logger.Printf("status failed: %v", err)
@@ -88,6 +94,7 @@ Commands:
   shutdown  Perform graceful cluster shutdown
   etcd-restore  Restore etcd from snapshot on a control plane
   daemon    Monitor UPS and auto-trigger shutdown
+  auto-heal Run one post-start auto-heal pass
   status    Print current ananke status and estimates
   intent    Read or manually set intent state
 
@@ -96,6 +103,7 @@ Examples:
   ananke shutdown --config /etc/ananke/ananke.yaml --execute --mode cluster-only --reason "manual-maintenance"
   ananke etcd-restore --config /etc/ananke/ananke.yaml --execute
   ananke daemon --config /etc/ananke/ananke.yaml
+  ananke auto-heal --config /etc/ananke/ananke.yaml --execute
   ananke status --config /etc/ananke/ananke.yaml
   ananke intent --config /etc/ananke/ananke.yaml --set normal --reason "manual-clear" --execute
 

cmd/ananke/main_dispatch_additional_test.go

@@ -71,10 +71,11 @@ func TestRunMainDispatchSuccessPaths(t *testing.T) {
 		func(_ *log.Logger, _ []string) error { return nil },
 		func(_ *log.Logger, _ []string) error { return nil },
 		func(_ *log.Logger, _ []string) error { return nil },
+		func(_ *log.Logger, _ []string) error { return nil },
 	)
 	defer restore()
 
-	commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "status", "intent", "help", "-h", "--help"}
+	commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "auto-heal", "status", "intent", "help", "-h", "--help"}
 	for _, command := range commands {
 		code := runMain(logger, []string{"ananke", command})
 		if code != 0 {
@@ -96,10 +97,11 @@ func TestRunMainDispatchErrorPaths(t *testing.T) {
 		func(_ *log.Logger, _ []string) error { return fail },
 		func(_ *log.Logger, _ []string) error { return fail },
 		func(_ *log.Logger, _ []string) error { return fail },
+		func(_ *log.Logger, _ []string) error { return fail },
 	)
 	defer restore()
 
-	commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "status", "intent"}
+	commands := []string{"startup", "shutdown", "etcd-restore", "daemon", "auto-heal", "status", "intent"}
 	for _, command := range commands {
 		code := runMain(logger, []string{"ananke", command})
 		if code != 1 {
@@ -122,13 +124,14 @@ func TestRunMainUsageAndUnknownPaths(t *testing.T) {
 }
 
 // stubDispatchCommands runs one orchestration or CLI step.
-// Signature: stubDispatchCommands(startup, shutdown, etcdRestore, daemon, status, intent func(*log.Logger, []string) error) func().
+// Signature: stubDispatchCommands(startup, shutdown, etcdRestore, daemon, autoHeal, status, intent func(*log.Logger, []string) error) func().
 // Why: keeps command-var overrides scoped and safely restorable across tests.
 func stubDispatchCommands(
 	startup,
 	shutdown,
 	etcdRestore,
 	daemon,
+	autoHeal,
 	status,
 	intent func(*log.Logger, []string) error,
 ) func() {
@@ -136,6 +139,7 @@ func stubDispatchCommands(
 	prevShutdown := shutdownCommand
 	prevEtcdRestore := etcdRestoreCommand
 	prevDaemon := daemonCommand
+	prevAutoHeal := autoHealCommand
 	prevStatus := statusCommand
 	prevIntent := intentCommand
 
@@ -143,6 +147,7 @@ func stubDispatchCommands(
 	shutdownCommand = shutdown
 	etcdRestoreCommand = etcdRestore
 	daemonCommand = daemon
+	autoHealCommand = autoHeal
 	statusCommand = status
 	intentCommand = intent
 
@@ -151,6 +156,7 @@ func stubDispatchCommands(
 		shutdownCommand = prevShutdown
 		etcdRestoreCommand = prevEtcdRestore
 		daemonCommand = prevDaemon
+		autoHealCommand = prevAutoHeal
 		statusCommand = prevStatus
 		intentCommand = prevIntent
 	}

configs/ananke.example.yaml

@@ -51,9 +51,12 @@ startup:
   require_node_inventory_reachability: true
   node_inventory_reachability_wait_seconds: 300
   node_inventory_reachability_poll_seconds: 5
+  node_inventory_reachability_required_nodes: []
+  longhorn_cryptsetup_exempt_nodes: []
   required_node_labels:
     titan-09:
       ananke.bstein.dev/harbor-bootstrap: "true"
+  required_node_labels_mode: enforce
   require_time_sync: true
   time_sync_wait_seconds: 240
   time_sync_poll_seconds: 5
@@ -90,6 +93,7 @@ startup:
     admin_secret_name: keycloak-admin
     admin_secret_username_key: username
     admin_secret_password_key: password
+  service_checklist_explicit_only: false
   service_checklist:
     - name: gitea-api
       url: https://scm.bstein.dev/api/healthz
@@ -134,18 +138,27 @@ startup:
   require_node_ssh_auth: true
   node_ssh_auth_wait_seconds: 240
   node_ssh_auth_poll_seconds: 5
+  node_ssh_auth_required_nodes: []
   require_flux_health: true
   flux_health_wait_seconds: 900
   flux_health_poll_seconds: 5
+  flux_health_required_kustomizations: []
   ignore_flux_kustomizations: []
   require_workload_convergence: true
   workload_convergence_wait_seconds: 900
   workload_convergence_poll_seconds: 5
+  workload_convergence_required_namespaces: []
   ignore_workload_namespaces: []
   ignore_workloads: []
   ignore_unavailable_nodes: []
   auto_recycle_stuck_pods: true
+  auto_quarantine_scheduling_storms: false
+  scheduling_storm_event_threshold: 30
+  scheduling_storm_window_seconds: 180
   stuck_pod_grace_seconds: 180
+  post_start_auto_heal_seconds: 60
+  recovery_cordon_max_seconds: 3600
+  dead_node_cleanup_grace_seconds: 300
   vault_unseal_key_file: /var/lib/ananke/vault-unseal.key
   vault_unseal_breakglass_command: ""
   vault_unseal_breakglass_timeout_seconds: 15
@@ -170,6 +183,7 @@ ups:
       target: pyrphoros@localhost
   poll_seconds: 5
   runtime_safety_factor: 1.25
+  on_battery_grace_seconds: 90
   debounce_count: 3
   telemetry_timeout_seconds: 90
 coordination:
@@ -186,6 +200,9 @@ metrics:
   enabled: true
   bind_addr: 0.0.0.0:9560
   path: /metrics
+gitops:
+  enabled: true
+  poll_seconds: 60
 state:
   dir: /var/lib/ananke
   reports_dir: /var/lib/ananke/reports

configs/ananke.tethys.yaml

@@ -29,7 +29,7 @@ ssh_node_hosts:
   titan-22: 192.168.22.22
   titan-24: 192.168.22.26
 ssh_node_users:
-  titan-24: atlas
+  titan-24: tethys
 ssh_managed_nodes:
   - titan-db
   - titan-0a
@@ -110,6 +110,10 @@ excluded_namespaces:
   - postgres
   - maintenance
 startup:
+  # Existing Vault CSI synchronization supplies password-backed host actions.
+  host_sudo_secret_namespace: maintenance
+  host_sudo_secret_name_template: "ananke-sudo-{node}"
+  host_sudo_secret_password_key: password
   api_wait_seconds: 1200
   api_poll_seconds: 2
   shutdown_cooldown_seconds: 45
@@ -117,9 +121,56 @@ startup:
   require_node_inventory_reachability: true
   node_inventory_reachability_wait_seconds: 300
   node_inventory_reachability_poll_seconds: 5
+  node_inventory_reachability_required_nodes:
+    - titan-0a
+    - titan-0b
+    - titan-0c
+  longhorn_cryptsetup_exempt_nodes:
+    - titan-23
   required_node_labels:
+    titan-04:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-05:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-06:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-07:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-08:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-11:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-12:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-13:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-14:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-15:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-17:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-18:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-19:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
     titan-09:
+      node-role.kubernetes.io/worker: "true"
       ananke.bstein.dev/harbor-bootstrap: "true"
+  required_node_labels_mode: enforce
   require_time_sync: true
   time_sync_wait_seconds: 240
   time_sync_poll_seconds: 5
@@ -156,6 +207,7 @@ startup:
     admin_secret_name: keycloak-admin
     admin_secret_username_key: username
     admin_secret_password_key: password
+  service_checklist_explicit_only: true
   service_checklist:
     - name: gitea-api
       url: https://scm.bstein.dev/api/healthz
@@ -200,18 +252,52 @@ startup:
   require_node_ssh_auth: true
   node_ssh_auth_wait_seconds: 240
   node_ssh_auth_poll_seconds: 5
+  node_ssh_auth_required_nodes:
+    - titan-0a
+    - titan-0b
+    - titan-0c
   require_flux_health: true
   flux_health_wait_seconds: 900
   flux_health_poll_seconds: 5
+  flux_health_required_kustomizations:
+    - flux-system/core
+    - flux-system/helm
+    - flux-system/traefik
+    - flux-system/cert-manager
+    - flux-system/longhorn
+    - flux-system/vault-csi
+    - flux-system/vault-injector
+    - flux-system/postgres
+    - flux-system/vault
+    - flux-system/keycloak
+    - flux-system/oauth2-proxy
+    - flux-system/gitea
+    - flux-system/monitoring
+    - flux-system/harbor
   ignore_flux_kustomizations: []
   require_workload_convergence: true
   workload_convergence_wait_seconds: 900
   workload_convergence_poll_seconds: 5
+  workload_convergence_required_namespaces:
+    - vault
+    - postgres
+    - sso
+    - gitea
+    - monitoring
+    - harbor
   ignore_workload_namespaces: []
   ignore_workloads: []
-  ignore_unavailable_nodes: []
+  ignore_unavailable_nodes:
+    - titan-09
+    - titan-10
   auto_recycle_stuck_pods: true
+  auto_quarantine_scheduling_storms: true
+  scheduling_storm_event_threshold: 30
+  scheduling_storm_window_seconds: 180
   stuck_pod_grace_seconds: 180
+  post_start_auto_heal_seconds: 60
+  recovery_cordon_max_seconds: 3600
+  dead_node_cleanup_grace_seconds: 300
   vault_unseal_key_file: /var/lib/ananke/vault-unseal.key
   vault_unseal_breakglass_command: "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -i /home/tethys/.ssh/id_ed25519 -p 1122 brad@99.183.132.163 'cat ~/.ananke-breakglass/vault-unseal.key'"
   vault_unseal_breakglass_timeout_seconds: 15
@@ -235,6 +321,7 @@ ups:
       target: statera@localhost
   poll_seconds: 5
   runtime_safety_factor: 1.25
+  on_battery_grace_seconds: 90
   debounce_count: 3
   telemetry_timeout_seconds: 90
 coordination:
@@ -252,6 +339,9 @@ metrics:
   enabled: true
   bind_addr: 0.0.0.0:9560
   path: /metrics
+gitops:
+  enabled: true
+  poll_seconds: 60
 state:
   dir: /var/lib/ananke
   reports_dir: /var/lib/ananke/reports

configs/ananke.titan-db.yaml

@@ -29,7 +29,7 @@ ssh_node_hosts:
   titan-22: 192.168.22.22
   titan-24: 192.168.22.26
 ssh_node_users:
-  titan-24: atlas
+  titan-24: tethys
 ssh_managed_nodes:
   - titan-db
   - titan-0a
@@ -110,6 +110,10 @@ excluded_namespaces:
   - postgres
   - maintenance
 startup:
+  # Existing Vault CSI synchronization supplies password-backed host actions.
+  host_sudo_secret_namespace: maintenance
+  host_sudo_secret_name_template: "ananke-sudo-{node}"
+  host_sudo_secret_password_key: password
   api_wait_seconds: 1200
   api_poll_seconds: 2
   shutdown_cooldown_seconds: 45
@@ -117,9 +121,56 @@ startup:
   require_node_inventory_reachability: true
   node_inventory_reachability_wait_seconds: 300
   node_inventory_reachability_poll_seconds: 5
+  node_inventory_reachability_required_nodes:
+    - titan-0a
+    - titan-0b
+    - titan-0c
+  longhorn_cryptsetup_exempt_nodes:
+    - titan-23
   required_node_labels:
+    titan-04:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-05:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-06:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-07:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-08:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-11:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-12:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-13:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-14:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-15:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-17:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-18:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
+    titan-19:
+      node-role.kubernetes.io/worker: "true"
+      longhorn-host: "true"
     titan-09:
+      node-role.kubernetes.io/worker: "true"
       ananke.bstein.dev/harbor-bootstrap: "true"
+  required_node_labels_mode: enforce
   require_time_sync: true
   time_sync_wait_seconds: 240
   time_sync_poll_seconds: 5
@@ -156,6 +207,7 @@ startup:
     admin_secret_name: keycloak-admin
     admin_secret_username_key: username
     admin_secret_password_key: password
+  service_checklist_explicit_only: true
   service_checklist:
     - name: gitea-api
       url: https://scm.bstein.dev/api/healthz
@@ -200,18 +252,52 @@ startup:
   require_node_ssh_auth: true
   node_ssh_auth_wait_seconds: 240
   node_ssh_auth_poll_seconds: 5
+  node_ssh_auth_required_nodes:
+    - titan-0a
+    - titan-0b
+    - titan-0c
   require_flux_health: true
   flux_health_wait_seconds: 900
   flux_health_poll_seconds: 5
+  flux_health_required_kustomizations:
+    - flux-system/core
+    - flux-system/helm
+    - flux-system/traefik
+    - flux-system/cert-manager
+    - flux-system/longhorn
+    - flux-system/vault-csi
+    - flux-system/vault-injector
+    - flux-system/postgres
+    - flux-system/vault
+    - flux-system/keycloak
+    - flux-system/oauth2-proxy
+    - flux-system/gitea
+    - flux-system/monitoring
+    - flux-system/harbor
   ignore_flux_kustomizations: []
   require_workload_convergence: true
   workload_convergence_wait_seconds: 900
   workload_convergence_poll_seconds: 5
+  workload_convergence_required_namespaces:
+    - vault
+    - postgres
+    - sso
+    - gitea
+    - monitoring
+    - harbor
   ignore_workload_namespaces: []
   ignore_workloads: []
-  ignore_unavailable_nodes: []
+  ignore_unavailable_nodes:
+    - titan-09
+    - titan-10
   auto_recycle_stuck_pods: true
+  auto_quarantine_scheduling_storms: true
+  scheduling_storm_event_threshold: 30
+  scheduling_storm_window_seconds: 180
   stuck_pod_grace_seconds: 180
+  post_start_auto_heal_seconds: 60
+  recovery_cordon_max_seconds: 3600
+  dead_node_cleanup_grace_seconds: 300
   vault_unseal_key_file: /var/lib/ananke/vault-unseal.key
   vault_unseal_breakglass_command: "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -i /home/atlas/.ssh/id_ed25519 -p 1122 brad@99.183.132.163 'cat ~/.ananke-breakglass/vault-unseal.key'"
   vault_unseal_breakglass_timeout_seconds: 15
@@ -235,6 +321,7 @@ ups:
       target: pyrphoros@localhost
   poll_seconds: 5
   runtime_safety_factor: 1.25
+  on_battery_grace_seconds: 90
   debounce_count: 3
   telemetry_timeout_seconds: 90
 coordination:
@@ -252,6 +339,9 @@ metrics:
   enabled: true
   bind_addr: 0.0.0.0:9560
   path: /metrics
+gitops:
+  enabled: true
+  poll_seconds: 60
 state:
   dir: /var/lib/ananke
   reports_dir: /var/lib/ananke/reports

deploy/systemd/ananke-update.service

@@ -10,7 +10,7 @@ User=root
 Group=root
 Environment=ANANKE_UPDATE_LOG_FILE=/var/log/ananke/update.log
 Environment=ANANKE_UPDATE_STATE_FILE=/var/lib/ananke/update-last.env
-Environment=ANANKE_UPDATE_ALLOW_QUALITY_FALLBACK=1
+Environment=ANANKE_UPDATE_ALLOW_QUALITY_FALLBACK=0
 ExecStart=/usr/local/lib/ananke/ananke-self-update.sh
 TimeoutStartSec=1800
 StandardOutput=journal

docs/ananke-hardening-prompt-2026-07-07.md

@@ -0,0 +1,465 @@
+# Prompt For Ananke Recovery Hardening
+
+You are a fresh, stateless Codex session working in the Ananke repository. Your goal is to implement, test, and validate generalized recovery improvements in Ananke based on a real Titan cluster power-loss incident on 2026-07-07. Treat this as production reliability work: read the repo first, follow existing code style and architecture, keep changes cohesive, and solve failure classes categorically rather than hard-coding node names, namespaces, or one-off incidents.
+
+## Mission
+
+Make Ananke resilient and automatic during post-power-loss cluster recovery. After your changes, one Ananke startup/recovery process should be able to:
+
+- Bring Vault and critical dependencies back safely.
+- Repair or escalate wedged k3s/containerd nodes using existing Ananke SSH/host access.
+- Use sudo credentials from Vault, securely and noninteractively, for host repair and diagnostics.
+- Avoid leaving Longhorn storage nodes cordoned indefinitely.
+- Detect Kubernetes/Longhorn readiness mismatches and repair label/manager/VolumeAttachment drift.
+- Resolve stale pod/PVC ownership safely, with different behavior for sidecar-only stale owners versus still-running application containers that mount the PVC.
+- Continue post-success convergence monitoring long enough to catch late node/runtime/storage wedges.
+- Report concise, actionable blockers without noisy repeated pod recycling.
+
+Do not implement special cases for `titan-04`, `titan-05`, `titan-15`, `titan-17`, `titan-22`, Firefly, Jellyfin, or any specific app. Use those names only as incident fixtures/test names. The code should operate on generic managed nodes, generic Longhorn nodes, generic Kubernetes workloads, and generic RWO PVCs.
+
+## Incident Context
+
+The Titan cluster lost power for a few minutes. Ariadne may have started shutdown behavior. The operator asked for a single Ananke process to bring the cluster back up and learn from any manual help required.
+
+Ananke did a lot correctly:
+
+- It recovered Vault and unsealed it.
+- It recovered critical workloads, Vault injector, Postgres, Keycloak, OpenSearch, oauth2 logs, CoreDNS, metrics-server, and many app pods.
+- It ran a single startup workflow and eventually completed successfully.
+- It recycled many stuck pods and ran several checks.
+
+But Ananke needed manual assistance in multiple reusable failure classes:
+
+- Node runtime wedges where Kubernetes readiness did not fully reflect kubelet/containerd health.
+- Host repairs blocked by sudo privilege gaps.
+- Longhorn nodes cordoned by Kubernetes remained unavailable for storage scheduling.
+- Longhorn readiness diverged from Kubernetes readiness.
+- Missing node labels caused Longhorn manager pods to disappear.
+- Stale terminating pods held RWO volumes and blocked replacement pods.
+- Repeated pod deletes did not resolve storage ownership or runtime reservation issues.
+- Ananke declared startup success before all late runtime/storage fallout was resolved.
+
+## Live Incident Facts To Encode As Tests
+
+Use these as fixtures for tests and documentation, not as hard-coded behavior:
+
+- `titan-05`: k3s-agent restart hung. Host SSH worked. `k3s-agent.service` entered `deactivating`, `final-sigkill`, `activating/start`, and `Result=timeout`. Kubelet port `10250` stayed closed. Kubernetes heartbeat was stale with `node.kubernetes.io/unreachable` taints. In-cluster helper pod could not start. Manual controlled reboot recovered it.
+- `titan-04`: after Ananke reported startup success, the node was Kubernetes `Ready` and kubelet `10250` initially open, but new pods failed with `FailedCreatePodSandBox`, `failed to reserve sandbox name`, and `context deadline exceeded`. A non-blocking `k3s-agent` restart then stuck in `deactivating/stop-sigkill`, kubelet `10250` closed, Kubernetes moved to `Ready=Unknown`, and a controlled reboot recovered it.
+- `titan-07`: a pod looked running in the API, but `kubectl exec` failed with `cannot exec in a deleted state`; restarting k3s-agent and recreating the pod fixed it.
+- `titan-14`: a replacement pod hit `failed to reserve container name` / `CreateContainerError`; k3s-agent restart and rerunning the in-cluster helper fixed it. Later another pod on the same node showed the same sandbox reservation pattern, but then progressed after enough time. Tests should distinguish transient progress from a stall.
+- `titan-15` and `titan-17`: Ananke cordoned them after encrypted Longhorn mounts exposed missing `cryptsetup`. Longhorn itself had `allowScheduling=true` and ready disks, but Longhorn `Schedulable=False` because Kubernetes had cordoned the nodes. Manual `kubectl uncordon titan-15 titan-17` restored Kubernetes scheduling and Longhorn schedulability. Ananke had attempted `cryptsetup-bin` repair but failed because `atlas` sudo required a password.
+- `titan-22`: Kubernetes `Ready=True`, CSI plugin and engine images were running, but Longhorn node `Ready=False` because `longhorn-manager` pod was missing. The root cause was missing node label `longhorn-host=true`, while the `longhorn-manager` DaemonSet selected `longhorn-host=true`. Manual `kubectl label node titan-22 longhorn-host=true --overwrite` started `longhorn-manager` and Longhorn node Ready became true. A pending wallet pod still carried an old failed VolumeAttachment until the pod was deleted/recreated.
+- Firefly stale owner, safe-to-force variant: `finance/firefly-...-q8rtq` was terminating for more than 25 minutes, had no finalizers, and only a Vault sidecar remained running. Replacement pod was blocked by `Multi-Attach` for an RWO PVC. Force-deleting the stale pod cleared ownership and allowed Longhorn attach to the replacement.
+- Firefly stale owner, unsafe-to-force variant: a later terminating Firefly pod still had the main application container running and mounting the RWO PVC. It was not safe to blindly force-delete. Waiting allowed kubelet to finish termination, then Longhorn attached the volume to the replacement.
+- Manual runtime inspection attempt on `titan-06`: `sudo -n crictl ...` failed because `atlas` required a sudo password. Ananke is expected to have access to sudo passwords in Vault, so this must be designed as a first-class Ananke capability rather than manual operator shell access.
+- Vault health check robustness: at one point `kubectl exec vault-0 -- vault status` was killed, but HTTP health via Kubernetes pod proxy showed `initialized=true`, `sealed=false`, `standby=false`. Startup should not fail solely on transient `kubectl exec` failure when endpoint and HTTP health prove Vault is available.
+- Image/DNS issues: several image pulls failed with `lookup registry-1.docker.io`, `lookup auth.docker.io`, or `lookup production.cloudfront.docker.com: Try again`, then later succeeded. Ananke repeatedly recycling unchanged ImagePullBackOff pods created noise.
+
+## Required Implementation Areas
+
+### 1. Secure Sudo Capability From Vault
+
+Ananke should be able to run bounded host diagnostics and repairs that require sudo.
+
+Requirements:
+
+- Discover and use the existing Vault integration and secret naming conventions already present in the repo/config. Do not invent a new hard-coded secret path if the repo has one.
+- Support noninteractive sudo for configured managed nodes.
+- Never log sudo passwords, Vault tokens, rendered secrets, command input containing secrets, or full environment dumps.
+- Provide a preflight check: for each managed node, can Ananke retrieve host sudo material and can it run a harmless command such as `sudo -S -p '' true` with a timeout?
+- If credentials are missing, wrong, or Vault is unavailable, Ananke should report `host-privilege-unavailable` with node, intended action, and secret lookup class, not spin.
+- Use strict command allowlists for privileged commands. The recovery path should not become arbitrary remote root execution.
+- Use bounded timeouts for all host commands and classify timeout versus auth failure versus command failure.
+- Prefer safe wrappers for:
+  - `systemctl show/is-active/restart --no-block k3s-agent`
+  - `systemctl reboot` when configured and escalated
+  - `crictl ps/pods/inspect/stop` for recovery diagnostics and controlled container stop
+  - package checks/install for `cryptsetup-bin`, `open-iscsi`, `nfs-common`, `dmsetup`
+  - `modprobe dm_crypt` or equivalent kernel-module validation where appropriate
+
+Tests:
+
+- Vault lookup succeeds and sudo command succeeds.
+- Vault lookup succeeds but sudo password is wrong.
+- Vault lookup missing.
+- Vault unavailable while Vault HTTP health is also degraded.
+- Vault unavailable but action does not require sudo.
+- Command times out.
+- Command returns nonzero.
+- Logs are scanned to ensure no password appears.
+
+### 2. Managed Node Runtime Recovery Primitive
+
+Implement a generic state machine for node runtime recovery.
+
+Detection signals:
+
+- Kubernetes node `Ready=Unknown` or stale heartbeat.
+- Kubernetes node `Ready=True` but pods on that node show repeated:
+  - `FailedCreatePodSandBox`
+  - `failed to reserve sandbox name`
+  - `failed to reserve container name`
+  - `CreateContainerError`
+  - `FailedKillPod`
+  - long-lived `ContainerCreating` or `PodInitializing` with no progress
+  - `kubectl exec`/logs failures indicating deleted runtime state
+- Host SSH works but kubelet port `127.0.0.1:10250` is closed or slow.
+- `systemctl show k3s-agent` reports `deactivating`, `stop-sigterm`, `stop-sigkill`, `final-sigterm`, `final-sigkill`, `activating/start`, or `Result=timeout`.
+- Existing in-cluster maintenance helper pod for the node cannot start, stays `ContainerCreating`, or cannot be scheduled because the node runtime is already unhealthy.
+
+Escalation ladder:
+
+- Open a single node recovery incident and suppress noisy repeated pod recycling for pods on that node while host repair is in progress.
+- Cordon the node before host-level intervention unless it is already unreachable/tainted.
+- Try in-cluster maintenance helper if kubelet/runtime can create pods.
+- If helper cannot run, use Ananke SSH route and sudo from Vault.
+- Prefer `systemctl --no-block restart k3s-agent`; never use a blocking restart over SSH as the primary recovery action.
+- Wait a bounded interval and require both:
+  - Kubernetes Ready=True with fresh heartbeat
+  - local kubelet `10250` open
+- If k3s-agent remains stuck in stop/kill/activating states or heartbeats remain stale, escalate to configured controlled reboot.
+- After reboot, wait for SSH, k3s-agent active/running, kubelet port open, Kubernetes Ready=True, and unreachable taints cleared.
+- Uncordon only after the node is healthy and after storage-specific checks pass.
+- Continue this monitoring after startup success for a configurable post-success convergence window.
+
+Tests:
+
+- Ready node with repeated sandbox reservation events triggers recovery.
+- Ready node with one transient sandbox event and then progress does not trigger restart.
+- Unknown node with SSH unavailable reports external blocker without repeated action.
+- Unknown node with SSH available, kubelet closed, k3s-agent stuck, helper unavailable escalates to reboot.
+- Non-blocking restart recovers node.
+- Non-blocking restart moves service to `deactivating/stop-sigkill`; reboot escalation occurs.
+- Reboot command accepted but SSH does not return within timeout.
+- Reboot returns and all readiness conditions clear.
+- False positive prevention: node Ready=True, kubelet open, events old/stale only, no action.
+- Idempotency: repeated Ananke runs do not issue repeated restarts/reboots while an incident is active.
+
+### 3. Longhorn And Kubernetes Readiness Reconciliation
+
+Ananke must not treat Kubernetes node Ready as sufficient for storage workloads.
+
+Implement a Longhorn readiness reconciler:
+
+- Read Kubernetes Nodes, Longhorn Node CRs, Longhorn manager DaemonSet, Longhorn manager pods, CSI pods, engine-image pods, Volume CRs, VolumeAttachments, and relevant PVC-bound pods.
+- Compare:
+  - Kubernetes Ready/SchedulingDisabled/taints
+  - Node labels required by Longhorn manager DaemonSet, especially `longhorn-host=true`
+  - Longhorn Node `Ready` and `Schedulable` conditions
+  - Longhorn Node `spec.allowScheduling`
+  - Longhorn manager pod presence on nodes where Longhorn Node exists or where PVC workloads are allowed
+  - CSI plugin readiness
+  - VolumeAttachment errors that reference stale Longhorn node readiness
+- Detect label drift: Longhorn node exists and CSI/engine pods exist, but manager pod is missing because the node no longer matches the DaemonSet selector.
+- Repair label drift only if it is safe:
+  - There is a known desired label source in repo/config/state, or
+  - Longhorn Node CR exists for that Kubernetes node and existing Longhorn metadata indicates the node is intended to be a Longhorn node.
+  - Emit exactly what label will be restored and why.
+- If safe repair cannot be proven, cordon or avoid scheduling PVC workloads to the node and report a blocker.
+- After restoring labels or manager pods, wait for Longhorn Node Ready=True.
+- For VolumeAttachments that failed because Longhorn thought a now-ready node was not ready, trigger a bounded retry:
+  - Prefer deleting/recreating the controller-owned Pending pod if that is the established pattern.
+  - Delete stale failed VolumeAttachment only if Kubernetes controller semantics and repo policy allow it.
+  - Never delete PV/PVC/Longhorn Volume data.
+
+Specific expectations from incident:
+
+- `titan-22` class: Kubernetes Ready, missing `longhorn-host=true`, no manager pod, Longhorn Node Ready=False ManagerPodMissing. Ananke should restore label or report exact unsafe reason, then wait for manager pod and Longhorn Ready.
+- `titan-15`/`titan-17` class: Kubernetes cordon makes Longhorn Schedulable=False despite Longhorn `allowScheduling=true` and ready disks. Ananke should not leave them cordoned after the blocking repair condition is resolved or after operator-approved override.
+
+Tests:
+
+- Longhorn manager selector changes are handled generically.
+- Missing Longhorn label restored from desired state.
+- Missing Longhorn label not restored if node lacks Longhorn Node CR and no desired-state evidence exists.
+- Longhorn Node Ready=False ManagerPodMissing becomes Ready after label restoration and manager pod startup.
+- VolumeAttachment error due stale Longhorn node readiness is retried after Longhorn Ready.
+- Kubernetes cordoned node causes Longhorn Schedulable=False and is uncordoned after repair.
+- Kubernetes cordoned node remains cordoned if required package/kernel preflight still fails and no override is configured.
+- `allowScheduling=false` is respected; Ananke should not enable Longhorn disk scheduling unless explicitly configured, but it may still ensure manager/CSI readiness for attachment if workloads are allowed there.
+
+### 4. Cordon/Uncordon And Host Package Repair Policy
+
+Ananke cordoned `titan-15` and `titan-17` because encrypted Longhorn mounts exposed missing `cryptsetup`; it failed to install because sudo required a password. This must become a closed-loop repair.
+
+Requirements:
+
+- Distinguish:
+  - Kubernetes scheduling cordon
+  - Longhorn disk scheduling (`allowScheduling`)
+  - Longhorn node readiness/schedulability
+  - ordinary `PreferNoSchedule` taints used as soft placement hints
+- If Ananke cordons a node, it must record why, when, what remediation is required, and what condition will uncordon it.
+- If the remediation is package/module installation, use Vault-backed sudo and verify the result.
+- If package repair succeeds, uncordon automatically and verify Longhorn Schedulable=True.
+- If package repair fails due missing sudo secret, wrong sudo password, package manager lock, apt/dpkg failure, or network outage, report the exact class and do not leave the node silently disabled.
+- Support an operator override policy for "uncordon for Longhorn availability even if encrypted workload preflight is not perfect", but this must be explicit/configured and logged.
+- Do not remove existing soft taints such as `atlas.bstein.dev/spillover=true:PreferNoSchedule` or `longhorn=true:PreferNoSchedule` unless the repo’s desired-state policy says to.
+
+Tests:
+
+- Missing cryptsetup with valid sudo installs package, loads module if needed, uncordons node.
+- Missing cryptsetup with missing sudo secret reports privilege blocker and leaves cordon with reason.
+- Node already has package but missing kernel module: module load path tested.
+- Node has Longhorn ready disks but is Kubernetes cordoned: Longhorn schedulability blocker is reported.
+- Uncordon preserves unrelated soft taints.
+- Re-running recovery is idempotent.
+
+### 5. Stale RWO PVC Owner Recovery
+
+Implement a generic stale-owner recovery flow for controller-owned pods and RWO PVCs.
+
+Detection:
+
+- Old pod has `metadata.deletionTimestamp` older than threshold.
+- Replacement pod from same controller exists and is pending/initializing.
+- Replacement events include `Multi-Attach`, `FailedAttachVolume`, `Volume is already used by pod(s)`, or `Volume is already exclusively attached to one node`.
+- PVC is `ReadWriteOnce` or equivalent single-writer mode.
+- Old and new pods are on different nodes.
+- Longhorn Volume/VolumeAttachment shows old node still owns attachment or new attach blocked.
+- Ordinary pod deletes have been attempted repeatedly without changing ownership.
+
+Recovery grouping:
+
+- Group old pod, replacement pod, controller, PVC, PV, VolumeAttachment(s), Longhorn Volume, old node, new node, and relevant events into one incident.
+- Emit one concise operator record.
+- Stop repeated blind pod deletes while the incident is active.
+
+Safety decisions:
+
+- Inspect old pod container statuses and volume mounts.
+- Classify containers:
+  - PVC-writing application containers: mount the blocked PVC.
+  - Sidecars that do not mount the blocked PVC: Vault agent, projected service account, emptyDir-only, etc.
+  - Unknown containers: treat as unsafe.
+- If only sidecars remain and no container mounts the PVC, force-delete the stale pod after a bounded wait.
+- If an application container still runs and mounts the PVC, do not blindly force-delete. Instead:
+  - Wait for kubelet termination if it is making progress.
+  - If stalled and host runtime inspection is available through Vault-backed sudo, inspect and optionally stop the specific container through a controlled allowlisted path.
+  - If Ananke cannot prove the PVC-writing container is stopped, report `unsafe-stale-owner` and do not clear the API object.
+- If old node kubelet/runtime is unhealthy, route through managed node recovery before stale-owner cleanup.
+
+Tests:
+
+- Sidecar-only stale pod force-clears and replacement attaches.
+- Live app container with blocked PVC waits and does not force-delete.
+- Live app container eventually exits; replacement attaches without force-delete.
+- Host runtime inspection unavailable due sudo privilege reports unsafe blocker.
+- Host runtime inspection available and controlled container stop succeeds.
+- Longhorn detach/attach follows cleanup.
+- Replacement readiness is rechecked after attach.
+- Multiple PVCs in one pod are handled.
+- Non-RWO PVC is not processed by this recovery path.
+
+### 6. Vault Health Robustness
+
+Startup should not fail because one Vault probe path is flaky if other authoritative health signals prove Vault is usable.
+
+Requirements:
+
+- Prefer multiple health signals:
+  - pod Running/Ready
+  - service endpoints populated
+  - HTTP `/v1/sys/health` through service or pod proxy
+  - `vault status` through exec as one signal, not the only signal
+- If `kubectl exec vault status` is killed/transiently fails but HTTP health reports initialized/unsealed/active or standby-acceptable per config, classify as transient exec failure and continue with warning.
+- If HTTP health and exec disagree, report clearly and retry with bounded backoff.
+- Never log tokens.
+
+Tests:
+
+- exec killed but HTTP health unsealed => startup continues with warning.
+- exec says sealed and HTTP says sealed => startup blocks/unseals.
+- endpoints missing => startup waits.
+- HTTP unavailable but exec succeeds => startup may proceed if policy allows.
+
+### 7. Image Pull And DNS Failure Classification
+
+Ananke should not repeatedly recycle pods whose only issue is unchanged image pull or DNS failure.
+
+Requirements:
+
+- Detect ImagePullBackOff/ErrImagePull events caused by DNS/registry lookup.
+- Run separate DNS checks:
+  - CoreDNS pods ready
+  - in-cluster DNS query from a known diagnostic pod if available
+  - node host DNS or kubelet image pull DNS failures grouped by node
+  - registry reachability by hostname class, not one app
+- If the image pull failure is unchanged and recent, do not repeatedly delete the pod.
+- Emit a compact blocker: image, registry host, node, last error, and whether DNS is cluster-wide or node-specific.
+- Retry only after DNS/registry health changes or after bounded interval.
+
+Tests:
+
+- Docker Hub auth DNS failure is classified.
+- Private registry DNS failure is classified.
+- One-off pull failure that later succeeds is not over-escalated.
+- Repeated pod recycle is suppressed.
+
+### 8. Startup Completion And Post-Success Monitoring
+
+Ananke declared `startup_status=success` while late runtime/storage issues still surfaced. Make completion smarter.
+
+Requirements:
+
+- Startup can reach "cluster core recovered" while still monitoring app convergence.
+- Do not mark total recovery complete until:
+  - critical services pass
+  - all managed nodes are either healthy, intentionally excluded, or explicitly blocked
+  - Longhorn node readiness/schedulability matches desired policy
+  - no active node-runtime incidents exist
+  - no unsafe stale-owner incidents exist
+- Add a post-success watch window for late events: sandbox reservation, FailedKillPod, kubelet metrics scrape failures, Longhorn manager missing, VolumeAttachment errors.
+- Distinguish statuses:
+  - `core_recovered`
+  - `converging`
+  - `success`
+  - `success_with_noncritical_blockers`
+  - `blocked_requires_operator`
+- Report app-level blockers separately from cluster-level blockers.
+
+Tests:
+
+- Critical services pass but node runtime wedge appears during post-success window => not final success until resolved.
+- Noncritical app image pull blocker can be `success_with_noncritical_blockers`.
+- Unsafe stale RWO owner blocks full success.
+- All clear => success.
+
+## Status And Operator Reporting
+
+Improve operator output. The user should not need to read journals to know what is happening.
+
+Status should include:
+
+- Whether a startup/recovery workflow is already running.
+- Current phase and elapsed time.
+- Current incidents grouped by type:
+  - node runtime recovery
+  - Longhorn/Kubernetes drift
+  - stale RWO owner
+  - Vault health
+  - image/DNS pull
+  - host privilege/preflight
+- For each incident: resource, trigger signals, action taken, next action, elapsed time, and whether operator input is needed.
+- Counts should be accurate and stable.
+- Repeated noisy details should be summarized with "same blocker still present for X minutes" instead of repeated lines.
+
+## Security Constraints
+
+- Do not log sudo passwords, Vault tokens, unsealed keys, or rendered secret contents.
+- Redact secrets in command arguments and environment.
+- Use command allowlists for privileged host actions.
+- Make destructive operations explicit and guarded:
+  - reboot only after escalation criteria
+  - force-delete pod only after stale-owner safety checks
+  - deleting VolumeAttachments only if repo policy allows and after safeguards
+- Keep audit logs of actions: node/resource, reason, command class, result, elapsed time, but not secrets.
+
+## Implementation Guidance
+
+Before editing:
+
+- Read Ananke’s existing startup/recovery code, Kubernetes client abstractions, Vault access layer, SSH/host command layer, Longhorn helpers if any, status/progress writer, and tests.
+- Reuse existing abstractions. Do not invent a parallel framework unless necessary.
+- Use fake clients and table-driven tests where the repo already uses them.
+- Keep changes modular:
+  - host privilege provider
+  - node runtime incident detector/reconciler
+  - Longhorn readiness reconciler
+  - stale RWO owner reconciler
+  - image/DNS classifier
+  - status/progress summarizer
+- Ensure all reconcilers are idempotent and bounded.
+
+Suggested model:
+
+- Define incident structs with stable keys, timestamps, observed signals, action history, next action, and terminal status.
+- Every recovery action should require:
+  - detection evidence
+  - policy permission
+  - safety gate
+  - timeout
+  - postcondition check
+- Avoid "delete pod until healthy" loops. Replace with evidence-driven remediation.
+
+## Validation Plan
+
+Run the repository’s standard checks first, then add targeted tests.
+
+Required validation:
+
+- Unit tests for each new classifier/reconciler.
+- Tests for redaction/no secret leakage.
+- Tests for idempotency on repeated reconcile calls.
+- Tests for progress/status output.
+- Integration-style tests with fake Kubernetes/Longhorn objects:
+  - nodes, pods, events, PVCs, PVs, VolumeAttachments, Longhorn Node CRs, Longhorn Volume CRs, DaemonSets, and pods.
+- Host command fake tests:
+  - SSH unavailable
+  - sudo unavailable
+  - sudo succeeds
+  - command timeout
+  - k3s-agent service states
+  - reboot accepted/not accepted
+- Vault fake tests:
+  - secret found
+  - secret missing
+  - Vault sealed/unavailable
+  - HTTP health fallback
+- End-to-end scenario tests composed from the incident:
+  - `titan-05` hard runtime wedge -> reboot -> Ready
+  - `titan-04` post-success sandbox wedge -> restart fails -> reboot -> Ready
+  - `titan-15`/`titan-17` cryptsetup/cordon flow -> sudo repair or explicit blocker -> uncordon when safe
+  - `titan-22` missing `longhorn-host` label -> restore label -> manager pod appears -> Longhorn Ready
+  - Firefly sidecar-only stale owner -> force delete -> attach replacement
+  - Firefly live app container stale owner -> wait/report unsafe, no force delete
+  - Vault exec killed but HTTP healthy -> continue
+  - DNS image pull errors -> classify without repeated recycle
+
+Run commands appropriate for the repo, likely including:
+
+```bash
+go test ./...
+go test -race ./...
+go vet ./...
+```
+
+If the repo uses other tooling, discover and run it. If any expensive/integration tests require cluster credentials, document how to run them and ensure fake-client tests cover the logic by default.
+
+## Acceptance Criteria
+
+The work is complete only when:
+
+- Ananke can retrieve and use Vault-backed sudo credentials securely or clearly reports why it cannot.
+- Node runtime wedges are detected and recovered by a bounded escalation ladder.
+- Ananke can handle Ready nodes that cannot create sandboxes.
+- Ananke can recover from k3s-agent stop-sigkill/timeout by controlled reboot when configured.
+- Ananke does not leave Longhorn nodes cordoned indefinitely after repair/override conditions are satisfied.
+- Ananke detects and fixes or reports Longhorn/Kubernetes drift, including missing manager-pod labels.
+- Ananke verifies Longhorn node readiness before trusting Kubernetes node readiness for PVC workloads.
+- Ananke handles stale RWO ownership safely, distinguishing sidecar-only stale pods from live PVC-writing app containers.
+- Ananke avoids noisy repeated deletion of unchanged image-pull failures.
+- Vault health checks tolerate transient exec failures when HTTP health is authoritative.
+- Startup success reflects real cluster recovery, with explicit `success_with_noncritical_blockers` or `blocked_requires_operator` states where appropriate.
+- All changes are covered by rigorous tests and pass the repo’s standard test suite.
+
+## What Not To Do
+
+- Do not hard-code Titan node names, app names, PVC names, or namespaces.
+- Do not log secrets.
+- Do not delete PVs, PVCs, or Longhorn Volume data.
+- Do not force-delete pods with live app containers mounting RWO PVCs unless a controlled, audited stop path has proven storage safety.
+- Do not repeatedly restart/reboot nodes without incident state and cooldowns.
+- Do not treat Kubernetes Ready as equivalent to Longhorn Ready.
+- Do not declare full success while node-runtime or unsafe stale-owner incidents are active.
+
+## Useful Terminology From The Incident
+
+- "Node runtime wedge": Kubernetes or host state where kubelet/containerd cannot create/kill sandboxes even if the node appears partially Ready.
+- "Stale RWO owner": a terminating pod or runtime survivor that still holds an exclusive PVC and blocks replacement attach.
+- "Sidecar-only stale owner": old pod still exists, but only non-PVC sidecars remain; force deletion may be safe after checks.
+- "Unsafe stale owner": app container still running and mounting the PVC; wait or controlled stop required.
+- "Longhorn/Kubernetes drift": Kubernetes node readiness/labels and Longhorn node readiness/manager state disagree.
+- "Host privilege unavailable": Ananke cannot run necessary sudo actions despite needing host-level repair.
+
+Use this prompt to implement production-grade, categorical recovery logic in Ananke, not one-off incident handling.

docs/ananke-recovery-2026-07-07.md

@@ -0,0 +1,207 @@
+# Ananke Recovery Notes - 2026-07-07
+
+## Context
+
+- Titan cluster lost power briefly; Ariadne may have started shutdown behavior.
+- User goal: let one Ananke recovery process bring the cluster back to health, monitor it, and record any manual help or automation gaps.
+- Operator view is from `/home/brad/Development/bstein_dev_home`; Kubernetes API is reachable through `/home/brad/titan/kubeconfig`.
+
+## Current Ananke Run
+
+- Coordinator host: `titan-db`.
+- `ananke.service` daemon is active.
+- `ananke-bootstrap.service` is active and running:
+  - command: `/usr/local/bin/ananke startup --config /etc/ananke/ananke.yaml --execute --force-flux-branch main --auto-peer-failover --peer-wait-seconds 180`
+  - current run started: `2026-07-07T17:19:50Z`
+  - current phase observed: `critical-workloads`
+  - checklist observed: `10 passed / 0 failed / 2 running`
+  - auto-heals observed: pod recycling for `VaultInitStuck`, `ContainerRuntimeWedge`, `CrashLoopBackOff`, stale terminating pods, and Longhorn attach blocked on unready nodes.
+  - previous run failed at `2026-07-07T17:19:19Z` waiting on `logging/deployment/oauth2-proxy-logs`; systemd restarted the bootstrap service.
+
+## Observed Recovery Progress
+
+- Latest snapshot at approximately `2026-07-07T17:58Z`:
+  - Ananke is still running in `convergence-checks` with `17/19` checks passed, `0` failed, and `2` running.
+  - Ananke has recorded `31` auto-heal actions, mostly repeated stuck-pod recycling.
+  - `titan-22` and `titan-24` both recovered to `Ready=True` with fresh kubelet transitions (`titan-22` at `2026-07-07T17:56:29Z`, `titan-24` at `2026-07-07T17:55:11Z`).
+  - Metrics server is running and `kubectl top nodes` works for recovered nodes; `titan-22` had just rejoined and metrics may lag briefly.
+  - Remaining convergence blocker is mostly app workload readiness, led by `finance/firefly`.
+- Ananke completed the startup workflow successfully at `2026-07-07T17:59:00Z`:
+  - `startup_status=success`
+  - `startup_phase=complete`
+  - `startup_checklist_total=20 passed=19 failed=0 running=1`
+  - `startup_auto_heals=33`
+  - `intent=normal`
+- Vault recovered:
+  - `vault/vault-0` is `1/1 Running` on `titan-18`.
+  - Vault status reports `initialized=true`, `sealed=false`, `ha_enabled=true`.
+  - Vault became active at `2026-07-07T17:05:46Z`.
+  - `vault-k8s-auth-config-autoheal-*` completed.
+- Vault injector recovered after `titan-04` kubelet recovery:
+  - `vault-injector-agent-injector-*` became `1/1 Running`.
+  - Vault service endpoints and injector webhook endpoints are populated.
+- Vault-backed workloads started moving again:
+  - Ariadne's Vault init authenticated successfully and rendered `/vault/secrets/ariadne-env.sh`.
+  - Ariadne is now `2/2 Running` on `titan-08`.
+- Flux is mostly reconciled, but several app kustomizations are still waiting on app readiness or dependencies.
+- Longhorn no longer shows the earlier broad faulted volume picture; observed state later was `41 attached`, `1 attaching`, `47 detached`.
+
+## Current Known Issues
+
+- `titan-22` and `titan-24` are still `NotReady` / `Unknown` from Kubernetes.
+- `titan-05` is `Unknown` after power-event recovery and k3s-agent restart attempts:
+  - Ananke SSH route on port `2277` reaches the host.
+  - `systemctl restart k3s-agent` timed out while stopping the old process.
+  - systemd reported `Processes still around after SIGKILL`, then spawned a new `/usr/local/bin/k3s agent`.
+  - `k3s-agent.service` remained `activating/start`; local `127.0.0.1:6444` was open but kubelet `127.0.0.1:10250` was closed.
+  - Kubernetes node heartbeat stayed stale at `2026-07-07T17:21:57Z` with `node.kubernetes.io/unreachable` taints.
+  - `maintenance/k3s-agent-restart-*` on `titan-05` stayed `ContainerCreating`, which confirms the in-cluster helper cannot repair this class once the node runtime is wedged.
+- `titan-15` and `titan-17` are `Ready,SchedulingDisabled`; Ananke cordoned them because encrypted Longhorn mounts exposed missing `cryptsetup`.
+- Ananke tried to install `cryptsetup-bin` on `titan-15` and `titan-17`, but the SSH repair failed because `sudo` required a password for `atlas`.
+- Several workloads are still unready while Ananke recycles stale `VaultInitStuck`, `CrashLoopBackOff`, and `ImagePullBackOff` pods.
+
+## Manual Actions Taken
+
+- Verified cluster health with `kubectl` and Flux commands.
+- Verified Vault directly with `vault status` inside `vault-0`.
+- Verified Ananke coordinator service state through `titan-db`.
+- Did not start a second Ananke recovery process after discovering the active `ananke-bootstrap.service`.
+- Used the existing `maintenance/k3s-agent-restart` DaemonSet as a targeted host-level nudge by deleting its pods on `titan-04`, `titan-05`, `titan-06`, `titan-08`, and `titan-18`; this reruns the DaemonSet's `k3s-agent` restart command on those nodes.
+- The in-cluster helper successfully reran on `titan-08`, but helper startup/termination was itself wedged on other affected nodes. Queued direct `systemctl --no-block restart k3s-agent` over Ananke SSH for `titan-05`, `titan-06`, and `titan-18`. A direct restart attempt against `titan-04` hung and was stopped.
+- After discovering Ananke's SSH profile uses `atlas@node:2277`, queued `systemctl --no-block restart k3s-agent` on `titan-04` and `titan-05` through the same route.
+- `titan-04` recovered to `Ready` after the non-blocking restart and its `maintenance/k3s-agent-restart-*` pod became `Running`.
+- `titan-05` did not recover: systemd timed out killing the old `k3s-agent`, left containerd shims behind, and started a new agent process without reopening kubelet port `10250` or restoring node heartbeats.
+- A controlled reboot was issued for `titan-05` through Ananke's SSH route at approximately `2026-07-07T17:32Z` because the node met the proposed escalation criteria: stale Kubernetes heartbeat, kubelet `10250` closed, `k3s-agent.service` stuck in `activating/start`, and the in-cluster helper stuck `ContainerCreating`.
+- After the reboot command returned, `titan-05` temporarily stopped responding to ping and SSH, then recovered to Kubernetes `Ready=True` with fresh heartbeats and no `unreachable` taints by approximately `2026-07-07T17:35Z`.
+- Ananke later detected and logged `cordoned container-runtime-wedged node(s): titan-07 pods=4`; direct host checks showed `titan-07` reachable, `k3s-agent` active/running, and kubelet `10250` open. This should become a false-positive/auto-uncordon test case if Kubernetes readiness remains healthy.
+- Later `titan-07` showed a real runtime/API-state split: `oauth2-proxy-logs` reported container state `running`, but `kubectl exec` failed with `cannot exec in a deleted state` and `kubectl logs --previous` could not retrieve the container logs. Manual assist: queued `systemctl --no-block restart k3s-agent` on `titan-07` and deleted the stale `logging/oauth2-proxy-logs-*` pod.
+- `titan-07` recovered to `Ready=True` with `k3s-agent` active and kubelet `10250` open. `logging/oauth2-proxy-logs` was recreated on `titan-12` and became `2/2 Running`.
+- Ananke repeatedly failed and restarted on `logging/deployment/oauth2-proxy-logs` because the replacement pod on `titan-14` hit the same `failed to reserve container name` / `CreateContainerError` runtime wedge signature.
+- Manual assist: queued `systemctl --no-block restart k3s-agent` on `titan-14` through Ananke SSH and deleted the stuck `logging/oauth2-proxy-logs-5b4d4f87db-2tdzd` pod with `--wait=false` so the controller can recreate it after the runtime reset.
+- Follow-up on `titan-14`: the direct SSH command timed out from the operator view, but a later status check showed `k3s-agent.service` had a new MainPID and was `activating/start`; the existing `maintenance/k3s-agent-restart` DaemonSet pod for `titan-14` was also deleted to rerun the in-cluster helper.
+- `titan-14` recovered to fresh `Ready=True` heartbeats and the replacement `maintenance/k3s-agent-restart-*` pod became `Running`.
+- After the `titan-05` reboot, `logging/opensearch-0` recovered to `1/1 Running`; `logging/oauth2-proxy-logs` was recreated on `titan-07` and was still initializing at the time of note.
+- Ananke also entered a short retry loop where `kubectl exec vault-0 ... vault status` was killed even though Vault HTTP health through the Kubernetes pod proxy reported `initialized=true`, `sealed=false`, `standby=false`. This should become a separate robustness fix: startup should not fail outright on a transient killed `kubectl exec` when service endpoints/HTTP health prove Vault is active.
+- After Keycloak completed its slow post-attach startup at approximately `2026-07-07T17:45:55Z`, OIDC-dependent OAuth proxies began recovering.
+- Ananke advanced past `critical-workloads` into `convergence-checks` by approximately `2026-07-07T17:50:50Z`.
+- Manual assist: force-deleted stale pod `finance/firefly-6669f46874-q8rtq` after Ananke repeatedly issued ordinary deletes but the pod remained `Terminating` with no finalizers and continued to block `firefly-storage` attachment. The Firefly Longhorn volume moved from `titan-04` to `titan-06` and reported `attached`/`healthy` on the replacement node.
+- Post-Ananke-completion manual assist: `titan-04` was still `Ready` with kubelet `10250` open, but several newly scheduled pods showed `FailedCreatePodSandBox`, `failed to reserve sandbox name`, and `context deadline exceeded`. Queued `systemctl --no-block restart k3s-agent` on `titan-04` at approximately `2026-07-07T18:02:55Z`.
+- `titan-04` did not recover from the non-blocking `k3s-agent` restart. It moved to Kubernetes `Ready=Unknown` with `node.kubernetes.io/unreachable` taints; host checks showed `k3s-agent.service` stuck in `deactivating/stop-sigkill`, `Result=timeout`, and kubelet `10250` closed. Manual assist: issued a controlled reboot for `titan-04` through Ananke's SSH route shortly after `2026-07-07T18:06Z`.
+- `titan-04` recovered after reboot by approximately `2026-07-07T18:10Z`: Kubernetes reported `Ready=True`, taints cleared, `k3s-agent` was `active/running`, and kubelet `10250` was open.
+- Firefly hit a second stale-owner handoff after the first cleanup. Replacement pod `firefly-6669f46874-pjhg6` on `titan-07` was blocked while terminating pod `firefly-6669f46874-lrz5d` on `titan-06` still had the main `firefly` container running and mounted `firefly-storage`. No force delete was issued for this second handoff because the application container was live and PVC-writing. The old pod later disappeared on its own; Longhorn then attached `pvc-358b4319-60cb-4322-b948-776c34a414a9` to `titan-07` as `attached`/`healthy`.
+- Attempted to inspect container runtime state on `titan-06` with `crictl`, but `atlas` required a sudo password. This should be part of the automation fix: Ananke either needs a noninteractive, bounded runtime-inspection primitive or must report that it cannot prove a PVC-writing container is stopped.
+- Manual assist: uncordoned `titan-15` and `titan-17` after confirming Kubernetes `Ready=True`, Longhorn `allowScheduling=true`, and Longhorn disks ready. The remaining blocker for Longhorn on both nodes was Kubernetes `SchedulingDisabled`; after uncordon, Longhorn `Schedulable=True` on both nodes.
+- Manual assist: resolved the `titan-22` Kubernetes/Longhorn readiness discrepancy. Kubernetes reported `Ready=True`, but Longhorn node `Ready=False` with `ManagerPodMissing`; the Longhorn manager DaemonSet selected `longhorn-host=true`, and `titan-22` was missing that label. Restored `longhorn-host=true`, which started `longhorn-manager-jtsjw` on `titan-22` and brought Longhorn node `Ready=True`.
+- Manual assist: deleted pending controller-owned pod `crypto/wallet-monero-temp-844bd949c7-28r4p` to force a fresh volume-attach retry after the `titan-22` Longhorn manager came up.
+- Manual assist: deleted stale unattached VolumeAttachment `csi-0bac99e18b687f6a50647dd959c26d6e2a1b2bf1a4f844de4f646cf3408d41c5` after it retained the pre-fix "node titan-22 is not ready" Longhorn error. Longhorn then attached `pvc-c6a4bf9b-17c7-4bdf-8a88-ca460ad0da5b` to `titan-22` as `attached`/`healthy`.
+
+## Automation Growth Opportunities
+
+- Ananke should expose from a remote operator machine whether the coordinator bootstrap is already running, so an operator does not risk starting a duplicate recovery.
+- The local Ananke config path assumes `/etc/ananke/ananke.yaml` and `/var/lib/ananke`; operator-side `status` failed without root. A read-only remote status wrapper would make this smoother.
+- Ananke should handle the `cryptsetup-bin` host repair on managed nodes without requiring interactive sudo, or clearly preflight and report the missing node privilege before a power event.
+- Ananke should separate `Vault pod/data volume unavailable` from `Vault sealed` in status. It did recover correctly, but early status said the unseal check passed while deferring because `vault-0` was pending.
+- Ananke should avoid repeatedly recycling the same ImagePullBackOff pods if the underlying registry/image problem is unchanged; this creates noisy recovery logs.
+- Ananke should surface current blocking workloads in a compact operator summary rather than requiring journal inspection.
+- Ananke should detect pods stuck in `PodInitializing` after init completion, especially when events show `FailedCreatePodSandbox`, `failed to reserve sandbox name`, `FailedKillPod`, or image pulls that never transition to `ImagePullBackOff`.
+- Ananke should be able to rerun the existing `k3s-agent-restart` maintenance primitive on affected nodes when containerd/kubelet sandbox state is wedged after a power event.
+- Ananke should fall back from the in-cluster restart helper to SSH `systemctl --no-block restart k3s-agent` when the helper pod cannot start because kubelet/containerd is already wedged. Blocking `systemctl restart` over SSH can hang the operator path.
+- Ananke should check host and CoreDNS upstream DNS health separately from node SSH reachability. During this incident, node IP reachability passed while hostname resolution and image pulls were intermittently failing.
+- Ananke should stop repeatedly issuing ordinary pod deletes when the same controller pod remains `Terminating` and continues to own exclusive resources. It should escalate to a stale-owner cleanup flow that proves the old workload is not writing, force-removes the orphaned API object when safe, then verifies Longhorn detach/attach and replacement pod startup.
+- Ananke should reconcile Kubernetes node labels against Longhorn manager scheduling requirements. During this incident `titan-22` was Kubernetes `Ready=True`, had CSI/engine Longhorn pods, and had a Longhorn Node CR, but no `longhorn-manager` pod because `longhorn-host=true` was missing. Kubernetes Ready alone was not sufficient for PVC workloads.
+- Ananke should understand that Kubernetes cordon directly affects Longhorn node schedulability. `titan-15` and `titan-17` had Longhorn-ready disks and `allowScheduling=true`, but Longhorn `Schedulable=False` solely because the Kubernetes nodes were cordoned.
+
+### Generalized Node Runtime Wedge Case
+
+This incident exposed a reusable failure mode that Ananke should treat as a node-runtime recovery problem, not just an app-pod recovery problem.
+
+Detection signals:
+
+- Node is `Ready=Unknown` or has a stale heartbeat while SSH on the managed Ananke route still works.
+- Host is pingable and local k3s load balancer port `127.0.0.1:6444` may be open, but kubelet port `127.0.0.1:10250` is closed or API proxy calls fail.
+- `systemctl restart k3s-agent` hangs, or `systemctl show k3s-agent` reports `deactivating`, `final-sigterm`, `final-sigkill`, `activating/start`, or `Result=timeout` for longer than a bounded recovery window.
+- `journalctl -u k3s-agent` includes `Processes still around after SIGKILL`, `State 'stop-sigterm' timed out`, or `State 'final-sigterm' timed out`.
+- Pods on the node show `FailedCreatePodSandBox`, `failed to reserve sandbox name`, `CreateContainerError`, `FailedKillPod`, or long-lived `ContainerCreating`/`PodInitializing`.
+- The `maintenance/k3s-agent-restart` DaemonSet pod for that node cannot start or stays `ContainerCreating`, which means Kubernetes-based repair is unavailable.
+
+Suggested automated escalation ladder:
+
+- Mark the node as in node-runtime recovery and pause noisy repeated pod recycling for pods pinned to that node.
+- Cordon the node or confirm it already has `unreachable` taints before host-level repair.
+- Try a bounded SSH health probe using Ananke's configured SSH user, port, config, and identity.
+- Prefer `systemctl --no-block restart k3s-agent` over blocking `systemctl restart k3s-agent`.
+- After a short wait, require both Kubernetes heartbeat freshness and local kubelet port `10250` to recover.
+- If the service remains `deactivating`, `final-sigkill`, or `activating/start` with stale heartbeats, escalate to a configured host reboot path.
+- After reboot, wait for SSH, k3s-agent active/running, kubelet `10250`, Kubernetes `Ready=True`, and removal of `unreachable` taints before resuming app-pod recycling for that node.
+
+Implementation notes:
+
+- This should be a generic managed-node repair primitive, not hard-coded to `titan-05`.
+- The primitive should emit one compact operator summary: node, phase, trigger signals, action taken, elapsed time, and next escalation.
+- Tests should cover a node with SSH available but kubelet closed, a blocking restart timeout, leftover shim evidence, a helper DaemonSet stuck in `ContainerCreating`, and successful recovery through reboot.
+- The post-completion `titan-04` recurrence confirms the primitive must continue monitoring after Ananke declares cluster startup success. A Ready node with fresh-enough status can still be unable to create new sandboxes, and a later restart can transition it into the same hard `deactivating/stop-sigkill` state seen on `titan-05`.
+
+### Generalized Stale Pod / Volume Ownership Case
+
+This incident also exposed a reusable app-convergence failure mode that Ananke should treat as stale Kubernetes/runtime ownership rather than simple pod unhealthiness.
+
+Concrete example:
+
+- `finance/firefly-6669f46874-q8rtq` entered deletion at `2026-07-07T17:43:55Z` and remained `Terminating` for more than 25 minutes.
+- The pod had no finalizers but still reported a running `vault-agent` sidecar on `titan-04`.
+- The replacement pod `finance/firefly-6669f46874-lrz5d` on `titan-06` stayed `Init:0/2`.
+- Events showed `FailedAttachVolume` / `Multi-Attach` for PVC `firefly-storage` because volume `pvc-358b4319-60cb-4322-b948-776c34a414a9` was still attached to `titan-04`.
+- Longhorn reported the volume `attached`, `healthy`, current node `titan-04`.
+- Ananke repeatedly logged ordinary recycling of the same stuck pod, but did not escalate to clearing the stale owner.
+- Manual force deletion of the stale pod cleared the stale API owner and allowed Longhorn to attach `firefly-storage` to the replacement pod's node.
+
+Detection signals:
+
+- A controller-owned pod has `metadata.deletionTimestamp` older than a bounded threshold and no finalizers.
+- A replacement pod from the same controller is pending or initializing on another node.
+- Events include `Multi-Attach`, `FailedAttachVolume`, `Volume is already used by pod(s)`, or `Volume is already exclusively attached to one node`.
+- The PVC is `ReadWriteOnce` and the old and new pod nodes differ.
+- Longhorn/CNI/runtime state still shows the old node as the attachment owner.
+- Ordinary `kubectl delete pod --wait=false` is repeated without changing the deletion timestamp or clearing the volume attachment.
+
+Suggested automated escalation ladder:
+
+- Group the old pod, replacement pod, PVC, VolumeAttachment, Longhorn volume, controller, and node into one recovery incident.
+- Verify whether any application container that mounts the PVC is still running. Sidecars that do not mount the PVC should not block stale-owner cleanup by themselves.
+- Check node health for the old owner node. If kubelet/runtime is unhealthy, route through the managed-node recovery primitive first.
+- If the old owner node is healthy but the pod remains deleting past the threshold, issue a bounded force deletion for the stale pod.
+- Wait for VolumeAttachment/Longhorn ownership to detach from the old node or attach to the replacement node.
+- Recheck the replacement pod init sequence and controller readiness before declaring the app recovered.
+- Emit one compact operator event instead of repeatedly logging the same pod recycle.
+
+Implementation notes:
+
+- This should be generic across any `ReadWriteOnce` PVC workload, not hard-coded to Firefly.
+- The safety check should reason at container/mount level: a sidecar that only mounts service-account, Vault secret, or emptyDir volumes is materially different from an app container actively mounting the PVC.
+- Tests should cover a terminating pod without finalizers holding an RWO PVC, an old node with healthy kubelet, an old node with wedged kubelet, a sidecar-only survivor, and a replacement pod blocked by `Multi-Attach`.
+- Tests should also cover the unsafe variant where the terminating pod's application container is still running and mounts the PVC. In that case Ananke should wait, stop the container through a controlled runtime path if allowed, or report that it lacks the privilege to prove storage safety; it should not blindly force-delete the API object.
+
+### Generalized Longhorn / Kubernetes Readiness Drift Case
+
+This incident exposed a reusable storage readiness gap: Kubernetes `Ready=True` is not enough for PVC workloads when Longhorn's node model disagrees.
+
+Detection signals:
+
+- Kubernetes node is `Ready=True`, untainted, and schedulable, but Longhorn Node `Ready=False`.
+- Longhorn Node reason is `ManagerPodMissing`, or the Longhorn manager DaemonSet has no pod on the node.
+- Longhorn manager DaemonSet selects a node label such as `longhorn-host=true`, and the Kubernetes node is missing it.
+- Longhorn CSI/engine pods or a Longhorn Node CR exist for the node, proving the node participates in Longhorn even though manager scheduling is broken.
+- VolumeAttachment errors say Longhorn cannot attach because the node is not ready, despite Kubernetes Ready=True.
+
+Suggested automated escalation ladder:
+
+- Compare Kubernetes node readiness, labels, taints, Longhorn Node readiness/schedulability, Longhorn manager DaemonSet selector, manager pod presence, CSI pod presence, and VolumeAttachment errors.
+- If desired-state evidence proves the node should be a Longhorn node, restore the missing manager selector label and wait for the manager pod and Longhorn Node `Ready=True`.
+- If desired-state evidence is missing, report a clear unsafe label-drift blocker rather than guessing.
+- After Longhorn Node `Ready=True`, retry stale VolumeAttachments by recreating the affected controller-owned pending pod or using a safe VolumeAttachment retry policy.
+- For Kubernetes-cordoned Longhorn nodes with `allowScheduling=true` and ready disks, either complete the host repair and uncordon or report the explicit blocker; do not leave Longhorn capacity disabled silently.
+
+Implementation notes:
+
+- This must be generic across Longhorn nodes and manager DaemonSet selectors, not hard-coded to `titan-22` or `longhorn-host`.
+- Tests should cover missing Longhorn selector labels, manager pod missing, Kubernetes Ready but Longhorn NotReady, Kubernetes cordon causing Longhorn Schedulable=False, and stale attach errors that clear after Longhorn readiness is restored.

internal/cluster/orchestrator_access_fluxsource.go

@@ -38,8 +38,8 @@ func (o *Orchestrator) reconcileNodeAccess(ctx context.Context, nodes []string)
 			defer wg.Done()
 			sem <- struct{}{}
 			defer func() { <-sem }()
-			if _, err := o.ssh(ctx, node, "sudo -n /usr/bin/systemctl --version"); err != nil {
+			if err := o.preflightHostPrivilege(ctx, node); err != nil {
-				errCh <- fmt.Errorf("%s: missing sudo access to /usr/bin/systemctl (--version): %w", node, err)
+				errCh <- fmt.Errorf("%s: host privilege preflight failed: %w", node, err)
 			}
 		}()
 	}
@@ -49,13 +49,16 @@ func (o *Orchestrator) reconcileNodeAccess(ctx context.Context, nodes []string)
 		return nil
 	}
 	samples := []string{}
+	errCount := 0
 	for err := range errCh {
+		errCount++
 		samples = append(samples, err.Error())
 		if len(samples) >= 4 {
 			break
 		}
 	}
-	return fmt.Errorf("access validation had %d errors (first: %s)", len(errCh), strings.Join(samples, " | "))
+	errCount += len(errCh)
+	return fmt.Errorf("access validation had %d errors (first: %s)", errCount, strings.Join(samples, " | "))
 }
 
 // waitForNodeSSHAuth runs one orchestration or CLI step.
@@ -77,7 +80,7 @@ func (o *Orchestrator) waitForNodeSSHAuth(ctx context.Context, nodes []string) e
 	ignored := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
 	seen := map[string]struct{}{}
 	targets := make([]string, 0, len(nodes))
-	for _, node := range nodes {
+	for _, node := range startupRequiredNodes(nodes, o.cfg.Startup.NodeSSHAuthRequiredNodes) {
 		node = strings.TrimSpace(node)
 		if node == "" {
 			continue

internal/cluster/orchestrator_autorepair.go

@@ -0,0 +1,416 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+type nodeReadyList struct {
+	Items []nodeReadyItem `json:"items"`
+}
+
+type nodeReadyItem struct {
+	Metadata struct {
+		Name        string            `json:"name"`
+		Annotations map[string]string `json:"annotations"`
+		Labels      map[string]string `json:"labels"`
+	} `json:"metadata"`
+	Spec struct {
+		Unschedulable bool `json:"unschedulable"`
+		Taints        []struct {
+			Key       string    `json:"key"`
+			TimeAdded time.Time `json:"timeAdded"`
+		} `json:"taints"`
+	} `json:"spec"`
+	Status struct {
+		Conditions []struct {
+			Type   string `json:"type"`
+			Status string `json:"status"`
+		} `json:"conditions"`
+	} `json:"status"`
+}
+
+type readyNodeCandidate struct {
+	Name          string
+	Unschedulable bool
+}
+
+type podDeleteList struct {
+	Items []struct {
+		Metadata struct {
+			Namespace         string     `json:"namespace"`
+			Name              string     `json:"name"`
+			DeletionTimestamp *time.Time `json:"deletionTimestamp"`
+		} `json:"metadata"`
+		Spec struct {
+			NodeName string `json:"nodeName"`
+		} `json:"spec"`
+	} `json:"items"`
+}
+
+// RunPostStartAutoHeal runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) RunPostStartAutoHeal(ctx context.Context) error.
+// Why: gives the long-running daemon a narrow, testable repair entrypoint for
+// post-start drift without rerunning the full startup flow.
+func (o *Orchestrator) RunPostStartAutoHeal(ctx context.Context) error {
+	return o.postStartAutoHeal(ctx)
+}
+
+// postStartAutoHeal runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) postStartAutoHeal(ctx context.Context) error.
+// Why: centralizes bounded post-start repair actions so recurring outage
+// patterns only trigger the specific remediation they need.
+func (o *Orchestrator) postStartAutoHeal(ctx context.Context) error {
+	if o.runner.DryRun {
+		return nil
+	}
+
+	errs := []string{}
+	requestReconcile := false
+
+	if err := o.ensureRequiredNodeLabels(ctx); err != nil {
+		errs = append(errs, fmt.Sprintf("required node labels: %v", err))
+	}
+
+	longhornRepairs, err := o.reconcileLonghornKubernetesReadiness(ctx)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("longhorn/kubernetes readiness drift: %v", err))
+	} else if longhornRepairs > 0 {
+		requestReconcile = true
+		o.log.Printf("post-start auto-heal repaired %d Longhorn/Kubernetes readiness drift item(s)", longhornRepairs)
+	}
+
+	releasedCordons, err := o.enforceRecoveryCordonLeases(ctx)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("recovery cordon lease check: %v", err))
+	} else if releasedCordons > 0 {
+		requestReconcile = true
+	}
+
+	vaultRecovered, err := o.autoRecoverSealedVault(ctx)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("vault auto-recovery: %v", err))
+	} else if vaultRecovered {
+		requestReconcile = true
+		if err := o.rerunVaultK8sAuthConfigJob(ctx); err != nil {
+			errs = append(errs, fmt.Sprintf("vault k8s auth config rerun: %v", err))
+		}
+	}
+
+	cleaned, err := o.cleanupTerminatingPodsOnUnavailableNodes(ctx)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("dead-node terminating pod cleanup: %v", err))
+	} else if cleaned > 0 {
+		requestReconcile = true
+	}
+
+	serviceRepairs, err := o.healUnreadyConfiguredServiceBackends(ctx)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("configured service backend repair: %v", err))
+	} else if len(serviceRepairs) > 0 {
+		requestReconcile = true
+		sort.Strings(serviceRepairs)
+		o.log.Printf("post-start auto-heal repaired configured service backend(s): %s", joinLimited(serviceRepairs, 8))
+	}
+
+	imagePullCredentialRepairs, err := o.healImagePullCredentialSync(ctx)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("image-pull credential sync repair: %v", err))
+	} else if len(imagePullCredentialRepairs) > 0 {
+		requestReconcile = true
+		sort.Strings(imagePullCredentialRepairs)
+		o.log.Printf("post-start auto-heal refreshed image-pull credential sync deployment(s): %s", joinLimited(imagePullCredentialRepairs, 8))
+		o.noteStartupAutoHeal(fmt.Sprintf("refreshed image-pull credential sync deployment(s): %s", joinLimited(imagePullCredentialRepairs, 8)))
+	}
+
+	repairedProxies, err := o.repairBrokenKubeletProxies(ctx)
+	if err != nil {
+		errs = append(errs, fmt.Sprintf("kubelet proxy auto-repair: %v", err))
+	} else if repairedProxies > 0 {
+		o.log.Printf("post-start auto-heal repaired %d broken kubelet proxy node(s)", repairedProxies)
+	}
+
+	if requestReconcile {
+		o.bestEffort("request flux reconcile after post-start auto-heal", func() error {
+			return o.requestFluxReconcile(ctx)
+		})
+	}
+
+	if len(errs) > 0 {
+		return errors.New(strings.Join(errs, "; "))
+	}
+	return nil
+}
+
+// autoRecoverSealedVault runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) autoRecoverSealedVault(ctx context.Context) (bool, error).
+// Why: lets the daemon repair a later Vault reseal without waiting for a new
+// bootstrap run.
+func (o *Orchestrator) autoRecoverSealedVault(ctx context.Context) (bool, error) {
+	if o.runner.DryRun {
+		return false, nil
+	}
+
+	phase, err := o.kubectl(ctx, 15*time.Second, "-n", "vault", "get", "pod", "vault-0", "-o", "jsonpath={.status.phase}")
+	if err != nil {
+		if isNotFoundErr(err) {
+			return false, nil
+		}
+		return false, fmt.Errorf("vault pod phase check failed: %w", err)
+	}
+	if strings.TrimSpace(phase) != "Running" {
+		return false, nil
+	}
+
+	sealed, err := o.vaultSealed(ctx)
+	if err != nil {
+		return false, err
+	}
+	if !sealed {
+		return false, nil
+	}
+
+	o.log.Printf("warning: detected sealed Vault after startup; attempting post-start auto-recovery")
+	if err := o.ensureVaultUnsealed(ctx); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
+// rerunVaultK8sAuthConfigJob runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) rerunVaultK8sAuthConfigJob(ctx context.Context) error.
+// Why: post-unseal Vault recovery needs the auth-config job retriggered so
+// downstream secret consumers stop carrying stale failures from the sealed window.
+func (o *Orchestrator) rerunVaultK8sAuthConfigJob(ctx context.Context) error {
+	if o.runner.DryRun {
+		return nil
+	}
+	jobName := fmt.Sprintf("vault-k8s-auth-config-autoheal-%d", time.Now().Unix())
+	if _, err := o.kubectl(
+		ctx,
+		25*time.Second,
+		"-n", "vault",
+		"create", "job",
+		"--from=cronjob/vault-k8s-auth-config",
+		jobName,
+	); err != nil {
+		return fmt.Errorf("create job %s: %w", jobName, err)
+	}
+	o.log.Printf("triggered vault k8s auth config job %s after vault recovery", jobName)
+	return nil
+}
+
+// repairBrokenKubeletProxies runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) repairBrokenKubeletProxies(ctx context.Context) (int, error).
+// Why: a Ready node can still have a dead kubelet tunnel, which breaks Jenkins
+// exec/websocket agents until the k3s agent is restarted on that exact node.
+func (o *Orchestrator) repairBrokenKubeletProxies(ctx context.Context) (int, error) {
+	if o.runner.DryRun {
+		return 0, nil
+	}
+
+	nodes, err := o.readyNodeCandidates(ctx)
+	if err != nil {
+		return 0, err
+	}
+
+	repaired := 0
+	errs := []string{}
+	for _, node := range nodes {
+		healthy, checkErr := o.kubeletProxyHealthy(ctx, node.Name)
+		if healthy {
+			continue
+		}
+		if checkErr != nil && !isRepairableKubeletProxyErr(checkErr) {
+			errs = append(errs, fmt.Sprintf("%s proxy health check: %v", node.Name, checkErr))
+			continue
+		}
+		if !o.sshManaged(node.Name) {
+			errs = append(errs, fmt.Sprintf("%s proxy broken but node is not SSH-managed", node.Name))
+			continue
+		}
+
+		if err := o.recoverManagedNodeRuntime(ctx, node.Name, node.Unschedulable, "broken kubelet proxy before k3s-agent restart"); err != nil {
+			errs = append(errs, fmt.Sprintf("%s %v", node.Name, err))
+			continue
+		}
+		repaired++
+	}
+
+	if len(errs) > 0 {
+		return repaired, errors.New(strings.Join(errs, "; "))
+	}
+	return repaired, nil
+}
+
+// readyNodeCandidates runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) readyNodeCandidates(ctx context.Context) ([]readyNodeCandidate, error).
+// Why: kubelet proxy repair should only touch nodes Kubernetes says are Ready,
+// preserving existing cordons when a node was intentionally kept out of service.
+func (o *Orchestrator) readyNodeCandidates(ctx context.Context) ([]readyNodeCandidate, error) {
+	out, err := o.kubectl(ctx, 20*time.Second, "get", "nodes", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query nodes: %w", err)
+	}
+	var nodes nodeReadyList
+	if err := json.Unmarshal([]byte(out), &nodes); err != nil {
+		return nil, fmt.Errorf("decode nodes: %w", err)
+	}
+
+	readyNodes := []readyNodeCandidate{}
+	for _, item := range nodes.Items {
+		ready := ""
+		for _, cond := range item.Status.Conditions {
+			if strings.EqualFold(strings.TrimSpace(cond.Type), "Ready") {
+				ready = strings.TrimSpace(cond.Status)
+				break
+			}
+		}
+		if ready == "True" && item.Metadata.Name != "" {
+			readyNodes = append(readyNodes, readyNodeCandidate{
+				Name:          item.Metadata.Name,
+				Unschedulable: item.Spec.Unschedulable,
+			})
+		}
+	}
+	return readyNodes, nil
+}
+
+// kubeletProxyHealthy runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) kubeletProxyHealthy(ctx context.Context, node string) (bool, error).
+// Why: the apiserver node proxy is the path Jenkins uses for pod exec; checking
+// it catches Ready-but-unusable nodes before agents start failing websockets.
+func (o *Orchestrator) kubeletProxyHealthy(ctx context.Context, node string) (bool, error) {
+	var lastErr error
+	for attempt := 0; attempt < 2; attempt++ {
+		healthy, err := o.kubeletProxyHealthyOnce(ctx, node)
+		if err == nil || !isTransientKubeletProxyCheckErr(err) {
+			return healthy, err
+		}
+		lastErr = err
+		o.log.Printf("warning: transient kubelet proxy health check failure for %s: %v; retrying", node, err)
+	}
+	return false, lastErr
+}
+
+// kubeletProxyHealthyOnce runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) kubeletProxyHealthyOnce(ctx context.Context, node string) (bool, error).
+// Why: transient apiserver/kubectl cancellations should be retried by the caller
+// without losing the exact kubelet proxy error when the node is really broken.
+func (o *Orchestrator) kubeletProxyHealthyOnce(ctx context.Context, node string) (bool, error) {
+	out, err := o.kubectl(ctx, 25*time.Second, "get", "--raw", fmt.Sprintf("/api/v1/nodes/%s/proxy/healthz", node), "--request-timeout=20s")
+	if err != nil {
+		if strings.TrimSpace(out) != "" {
+			return false, fmt.Errorf("%w: %s", err, strings.TrimSpace(out))
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+// isTransientKubeletProxyCheckErr runs one orchestration or CLI step.
+// Signature: isTransientKubeletProxyCheckErr(err error) bool.
+// Why: a killed or timed-out kubectl health probe is not proof the node proxy is
+// broken; retry before reporting or repairing a Ready node.
+func isTransientKubeletProxyCheckErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := strings.ToLower(err.Error())
+	transient := []string{
+		"signal: killed",
+		"context deadline exceeded",
+		"deadline exceeded",
+		"i/o timeout",
+		"request canceled",
+		"client.timeout",
+		"timeout awaiting",
+	}
+	for _, needle := range transient {
+		if strings.Contains(msg, needle) {
+			return true
+		}
+	}
+	return false
+}
+
+// isRepairableKubeletProxyErr runs one orchestration or CLI step.
+// Signature: isRepairableKubeletProxyErr(err error) bool.
+// Why: keep this repair narrow so Ananke restarts k3s-agent for the known
+// kubelet-tunnel failure, not for every transient kubectl problem.
+func isRepairableKubeletProxyErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := strings.ToLower(err.Error())
+	repairable := []string{
+		"502",
+		"bad gateway",
+		"failed to find session",
+		"error trying to reach service",
+		"10250",
+	}
+	for _, needle := range repairable {
+		if strings.Contains(msg, needle) {
+			return true
+		}
+	}
+	return false
+}
+
+// requestFluxReconcile runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) requestFluxReconcile(ctx context.Context) error.
+// Why: post-start repairs need a lightweight way to refresh GitOps health
+// without reusing the broader startup flux-resume flow.
+func (o *Orchestrator) requestFluxReconcile(ctx context.Context) error {
+	if o.runner.DryRun {
+		return nil
+	}
+
+	now := time.Now().UTC().Format(time.RFC3339)
+	if _, err := o.kubectl(
+		ctx,
+		25*time.Second,
+		"-n", "flux-system",
+		"annotate", "gitrepository", "flux-system",
+		"reconcile.fluxcd.io/requestedAt="+now,
+		"--overwrite",
+	); err != nil {
+		return fmt.Errorf("annotate flux source reconcile: %w", err)
+	}
+	if _, err := o.kubectl(
+		ctx,
+		25*time.Second,
+		"-n", "flux-system",
+		"annotate",
+		"kustomizations.kustomize.toolkit.fluxcd.io",
+		"--all",
+		"reconcile.fluxcd.io/requestedAt="+now,
+		"--overwrite",
+	); err != nil {
+		return fmt.Errorf("annotate flux kustomizations reconcile: %w", err)
+	}
+	if _, err := o.kubectl(
+		ctx,
+		25*time.Second,
+		"annotate",
+		"--all-namespaces",
+		"helmreleases.helm.toolkit.fluxcd.io",
+		"--all",
+		"reconcile.fluxcd.io/requestedAt="+now,
+		"--overwrite",
+	); err != nil {
+		o.log.Printf("warning: annotate helmreleases for post-start reconcile failed: %v", err)
+	}
+	if o.runOverride == nil && o.runner.CommandExists("flux") {
+		if _, err := o.run(ctx, 75*time.Second, "flux", "reconcile", "source", "git", "flux-system", "-n", "flux-system", "--timeout=60s"); err != nil {
+			o.log.Printf("warning: flux source reconcile command failed during post-start auto-heal: %v", err)
+		}
+	}
+	return nil
+}

internal/cluster/orchestrator_autorepair_cleanup.go

@@ -0,0 +1,97 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// cleanupTerminatingPodsOnUnavailableNodes runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error).
+// Why: dead nodes can strand terminating pods indefinitely, so the daemon should
+// clear only that narrow failure class instead of leaving garbage behind forever.
+func (o *Orchestrator) cleanupTerminatingPodsOnUnavailableNodes(ctx context.Context) (int, error) {
+	if o.runner.DryRun {
+		return 0, nil
+	}
+
+	unavailable, err := o.unavailableNodeSet(ctx)
+	if err != nil {
+		return 0, err
+	}
+	if len(unavailable) == 0 {
+		return 0, nil
+	}
+
+	out, err := o.kubectl(ctx, 30*time.Second, "get", "pods", "-A", "-o", "json")
+	if err != nil {
+		return 0, fmt.Errorf("query pods: %w", err)
+	}
+	var pods podDeleteList
+	if err := json.Unmarshal([]byte(out), &pods); err != nil {
+		return 0, fmt.Errorf("decode pods: %w", err)
+	}
+
+	grace := time.Duration(o.cfg.Startup.DeadNodeCleanupGraceSeconds) * time.Second
+	now := time.Now()
+	count := 0
+	for _, item := range pods.Items {
+		if item.Metadata.DeletionTimestamp == nil || item.Spec.NodeName == "" {
+			continue
+		}
+		if _, badNode := unavailable[item.Spec.NodeName]; !badNode {
+			continue
+		}
+		if now.Sub(*item.Metadata.DeletionTimestamp) < grace {
+			continue
+		}
+		o.log.Printf("warning: force deleting terminating pod %s/%s on unavailable node %s", item.Metadata.Namespace, item.Metadata.Name, item.Spec.NodeName)
+		if _, err := o.kubectl(
+			ctx,
+			20*time.Second,
+			"-n", item.Metadata.Namespace,
+			"delete", "pod", item.Metadata.Name,
+			"--grace-period=0",
+			"--force",
+			"--wait=false",
+		); err != nil && !isNotFoundErr(err) {
+			return count, fmt.Errorf("delete pod %s/%s: %w", item.Metadata.Namespace, item.Metadata.Name, err)
+		}
+		count++
+	}
+	if count > 0 {
+		o.log.Printf("post-start auto-heal cleaned %d terminating pod(s) from unavailable nodes", count)
+	}
+	return count, nil
+}
+
+// unavailableNodeSet runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error).
+// Why: isolates Ready-condition parsing so dead-node cleanup stays targeted.
+func (o *Orchestrator) unavailableNodeSet(ctx context.Context) (map[string]struct{}, error) {
+	out, err := o.kubectl(ctx, 20*time.Second, "get", "nodes", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query nodes: %w", err)
+	}
+	var nodes nodeReadyList
+	if err := json.Unmarshal([]byte(out), &nodes); err != nil {
+		return nil, fmt.Errorf("decode nodes: %w", err)
+	}
+
+	unavailable := map[string]struct{}{}
+	for _, item := range nodes.Items {
+		ready := ""
+		for _, cond := range item.Status.Conditions {
+			if strings.EqualFold(strings.TrimSpace(cond.Type), "Ready") {
+				ready = strings.TrimSpace(cond.Status)
+				break
+			}
+		}
+		if ready != "True" {
+			unavailable[item.Metadata.Name] = struct{}{}
+		}
+	}
+	return unavailable, nil
+}

internal/cluster/orchestrator_autorepair_cleanup_test.go

@@ -0,0 +1,296 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+	"scm.bstein.dev/bstein/ananke/internal/execx"
+	"scm.bstein.dev/bstein/ananke/internal/state"
+)
+
+// TestCleanupTerminatingPodsOnUnavailableNodesBranches runs one orchestration or CLI step.
+// Signature: TestCleanupTerminatingPodsOnUnavailableNodesBranches(t *testing.T).
+// Why: cleanup on dead nodes must be selective so Ananke only force-deletes the
+// truly stranded pods and tolerates already-gone objects.
+func TestCleanupTerminatingPodsOnUnavailableNodesBranches(t *testing.T) {
+	t.Run("dry run skips", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, nil)
+		orch.runner.DryRun = true
+		count, err := orch.cleanupTerminatingPodsOnUnavailableNodes(context.Background())
+		if err != nil || count != 0 {
+			t.Fatalf("expected dry-run skip, got count=%d err=%v", count, err)
+		}
+	})
+
+	t.Run("selective cleanup tolerates not found", func(t *testing.T) {
+		oldDelete := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
+		recentDelete := time.Now().Add(-2 * time.Minute).UTC().Format(time.RFC3339)
+		orch := buildOrchestratorWithStubs(t, config.Config{
+			Startup: config.Startup{DeadNodeCleanupGraceSeconds: 300},
+		}, []commandStub{
+			{
+				match: matchContains("kubectl", "get nodes -o json"),
+				out:   `{"items":[{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"Ready","status":"Unknown"}]}},{"metadata":{"name":"titan-07"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`,
+			},
+			{
+				match: matchContains("kubectl", "get pods -A -o json"),
+				out: `{"items":[` +
+					`{"metadata":{"namespace":"maintenance","name":"old-stale","deletionTimestamp":"` + oldDelete + `"},"spec":{"nodeName":"titan-22"}},` +
+					`{"metadata":{"namespace":"maintenance","name":"fresh-stale","deletionTimestamp":"` + recentDelete + `"},"spec":{"nodeName":"titan-22"}},` +
+					`{"metadata":{"namespace":"logging","name":"healthy-node","deletionTimestamp":"` + oldDelete + `"},"spec":{"nodeName":"titan-18"}},` +
+					`{"metadata":{"namespace":"logging","name":"no-delete"},"spec":{"nodeName":"titan-22"}}]}`,
+			},
+			{
+				match: matchContains("kubectl", "-n maintenance delete pod old-stale --grace-period=0 --force --wait=false"),
+				err:   errors.New("pod old-stale not found"),
+			},
+		})
+
+		count, err := orch.cleanupTerminatingPodsOnUnavailableNodes(context.Background())
+		if err != nil {
+			t.Fatalf("cleanupTerminatingPodsOnUnavailableNodes failed: %v", err)
+		}
+		if count != 1 {
+			t.Fatalf("expected one cleaned pod, got %d", count)
+		}
+	})
+
+	t.Run("query and decode errors surface", func(t *testing.T) {
+		queryErrOrch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "get nodes -o json"),
+				err:   errors.New("nodes failed"),
+			},
+		})
+		if _, err := queryErrOrch.cleanupTerminatingPodsOnUnavailableNodes(context.Background()); err == nil || !strings.Contains(err.Error(), "query nodes") {
+			t.Fatalf("expected node query error, got %v", err)
+		}
+
+		decodeErrOrch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "get nodes -o json"),
+				out:   `{"items":[{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"Ready","status":"Unknown"}]}}]}`,
+			},
+			{
+				match: matchContains("kubectl", "get pods -A -o json"),
+				out:   `{bad json`,
+			},
+		})
+		if _, err := decodeErrOrch.cleanupTerminatingPodsOnUnavailableNodes(context.Background()); err == nil || !strings.Contains(err.Error(), "decode pods") {
+			t.Fatalf("expected pod decode error, got %v", err)
+		}
+	})
+
+	t.Run("delete hard error surfaces", func(t *testing.T) {
+		oldDelete := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
+		orch := buildOrchestratorWithStubs(t, config.Config{
+			Startup: config.Startup{DeadNodeCleanupGraceSeconds: 300},
+		}, []commandStub{
+			{
+				match: matchContains("kubectl", "get nodes -o json"),
+				out:   `{"items":[{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"Ready","status":"False"}]}}]}`,
+			},
+			{
+				match: matchContains("kubectl", "get pods -A -o json"),
+				out:   `{"items":[{"metadata":{"namespace":"maintenance","name":"old-stale","deletionTimestamp":"` + oldDelete + `"},"spec":{"nodeName":"titan-22"}}]}`,
+			},
+			{
+				match: matchContains("kubectl", "-n maintenance delete pod old-stale --grace-period=0 --force --wait=false"),
+				err:   errors.New("delete failed"),
+			},
+		})
+
+		count, err := orch.cleanupTerminatingPodsOnUnavailableNodes(context.Background())
+		if count != 0 || err == nil || !strings.Contains(err.Error(), "delete pod maintenance/old-stale") {
+			t.Fatalf("expected delete failure, got count=%d err=%v", count, err)
+		}
+	})
+}
+
+// TestUnavailableNodeSetBranches runs one orchestration or CLI step.
+// Signature: TestUnavailableNodeSetBranches(t *testing.T).
+// Why: node Ready parsing drives dead-node cleanup, so malformed and missing
+// Ready condition payloads need direct coverage too.
+func TestUnavailableNodeSetBranches(t *testing.T) {
+	t.Run("decode error surfaces", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{match: matchContains("kubectl", "get nodes -o json"), out: `{bad json`},
+		})
+		if _, err := orch.unavailableNodeSet(context.Background()); err == nil || !strings.Contains(err.Error(), "decode nodes") {
+			t.Fatalf("expected decode error, got %v", err)
+		}
+	})
+
+	t.Run("missing ready condition counts as unavailable", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "get nodes -o json"),
+				out:   `{"items":[{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"MemoryPressure","status":"False"}]}},{"metadata":{"name":"titan-07"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`,
+			},
+		})
+		nodes, err := orch.unavailableNodeSet(context.Background())
+		if err != nil {
+			t.Fatalf("unavailableNodeSet failed: %v", err)
+		}
+		if _, ok := nodes["titan-22"]; !ok {
+			t.Fatalf("expected titan-22 to be treated as unavailable")
+		}
+		if _, ok := nodes["titan-07"]; ok {
+			t.Fatalf("did not expect titan-07 to be treated as unavailable")
+		}
+	})
+}
+
+// TestRequestFluxReconcileBranches runs one orchestration or CLI step.
+// Signature: TestRequestFluxReconcileBranches(t *testing.T).
+// Why: the post-start repair loop needs predictable Flux refresh behavior even
+// when one annotation call is flaky.
+func TestRequestFluxReconcileBranches(t *testing.T) {
+	t.Run("dry run skips", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, nil)
+		orch.runner.DryRun = true
+		if err := orch.requestFluxReconcile(context.Background()); err != nil {
+			t.Fatalf("dry-run requestFluxReconcile failed: %v", err)
+		}
+	})
+
+	t.Run("git source annotate error surfaces", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "-n flux-system annotate gitrepository flux-system reconcile.fluxcd.io/requestedAt="),
+				err:   errors.New("annotate failed"),
+			},
+		})
+		if err := orch.requestFluxReconcile(context.Background()); err == nil || !strings.Contains(err.Error(), "annotate flux source reconcile") {
+			t.Fatalf("expected gitrepository annotate error, got %v", err)
+		}
+	})
+
+	t.Run("kustomization annotate error surfaces", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "-n flux-system annotate gitrepository flux-system reconcile.fluxcd.io/requestedAt="),
+				out:   "",
+			},
+			{
+				match: matchContains("kubectl", "-n flux-system annotate kustomizations.kustomize.toolkit.fluxcd.io --all reconcile.fluxcd.io/requestedAt="),
+				err:   errors.New("annotate failed"),
+			},
+		})
+		if err := orch.requestFluxReconcile(context.Background()); err == nil || !strings.Contains(err.Error(), "annotate flux kustomizations reconcile") {
+			t.Fatalf("expected kustomization annotate error, got %v", err)
+		}
+	})
+
+	t.Run("helm annotate warning and flux command path", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		callLog := filepath.Join(tmpDir, "calls.log")
+		kubectlPath := filepath.Join(tmpDir, "kubectl")
+		fluxPath := filepath.Join(tmpDir, "flux")
+
+		kubectlScript := "#!/bin/sh\n" +
+			"printf '%s\\n' \"$*\" >> \"" + callLog + "\"\n" +
+			"case \"$*\" in\n" +
+			"  *helmreleases.helm.toolkit.fluxcd.io*) echo helm annotate failed >&2; exit 1 ;;\n" +
+			"esac\n" +
+			"exit 0\n"
+		fluxScript := "#!/bin/sh\n" +
+			"printf 'flux %s\\n' \"$*\" >> \"" + callLog + "\"\n" +
+			"exit 0\n"
+
+		if err := os.WriteFile(kubectlPath, []byte(kubectlScript), 0o755); err != nil {
+			t.Fatalf("write fake kubectl: %v", err)
+		}
+		if err := os.WriteFile(fluxPath, []byte(fluxScript), 0o755); err != nil {
+			t.Fatalf("write fake flux: %v", err)
+		}
+		t.Setenv("PATH", tmpDir+":"+os.Getenv("PATH"))
+
+		cfg := config.Config{
+			State: config.State{
+				Dir:            t.TempDir(),
+				ReportsDir:     filepath.Join(t.TempDir(), "reports"),
+				RunHistoryPath: filepath.Join(t.TempDir(), "runs.json"),
+			},
+		}
+		orch := &Orchestrator{
+			cfg:    cfg,
+			runner: &execx.Runner{},
+			store:  state.New(cfg.State.RunHistoryPath),
+			log:    log.New(io.Discard, "", 0),
+		}
+
+		if err := orch.requestFluxReconcile(context.Background()); err != nil {
+			t.Fatalf("requestFluxReconcile with fake binaries failed: %v", err)
+		}
+		calls, err := os.ReadFile(callLog)
+		if err != nil {
+			t.Fatalf("read fake command log: %v", err)
+		}
+		logText := string(calls)
+		if !strings.Contains(logText, "annotate gitrepository flux-system") {
+			t.Fatalf("expected gitrepository annotate call, got %q", logText)
+		}
+		if !strings.Contains(logText, "annotate kustomizations.kustomize.toolkit.fluxcd.io --all") {
+			t.Fatalf("expected kustomization annotate call, got %q", logText)
+		}
+		if !strings.Contains(logText, "flux reconcile source git flux-system -n flux-system --timeout=60s") {
+			t.Fatalf("expected flux reconcile command, got %q", logText)
+		}
+	})
+
+	t.Run("flux command failure is tolerated", func(t *testing.T) {
+		tmpDir := t.TempDir()
+		callLog := filepath.Join(tmpDir, "calls.log")
+		kubectlPath := filepath.Join(tmpDir, "kubectl")
+		fluxPath := filepath.Join(tmpDir, "flux")
+
+		kubectlScript := "#!/bin/sh\n" +
+			"printf '%s\\n' \"$*\" >> \"" + callLog + "\"\n" +
+			"exit 0\n"
+		fluxScript := "#!/bin/sh\n" +
+			"printf 'flux-fail %s\\n' \"$*\" >> \"" + callLog + "\"\n" +
+			"exit 1\n"
+
+		if err := os.WriteFile(kubectlPath, []byte(kubectlScript), 0o755); err != nil {
+			t.Fatalf("write fake kubectl: %v", err)
+		}
+		if err := os.WriteFile(fluxPath, []byte(fluxScript), 0o755); err != nil {
+			t.Fatalf("write fake flux: %v", err)
+		}
+		t.Setenv("PATH", tmpDir+":"+os.Getenv("PATH"))
+
+		cfg := config.Config{
+			State: config.State{
+				Dir:            t.TempDir(),
+				ReportsDir:     filepath.Join(t.TempDir(), "reports"),
+				RunHistoryPath: filepath.Join(t.TempDir(), "runs.json"),
+			},
+		}
+		orch := &Orchestrator{
+			cfg:    cfg,
+			runner: &execx.Runner{},
+			store:  state.New(cfg.State.RunHistoryPath),
+			log:    log.New(io.Discard, "", 0),
+		}
+
+		if err := orch.requestFluxReconcile(context.Background()); err != nil {
+			t.Fatalf("requestFluxReconcile should tolerate flux failure, got %v", err)
+		}
+		calls, err := os.ReadFile(callLog)
+		if err != nil {
+			t.Fatalf("read fake command log: %v", err)
+		}
+		if !strings.Contains(string(calls), "flux-fail reconcile source git flux-system -n flux-system --timeout=60s") {
+			t.Fatalf("expected failing flux command to be attempted, got %q", string(calls))
+		}
+	})
+}

internal/cluster/orchestrator_autorepair_proxy_test.go

@@ -0,0 +1,381 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestRepairBrokenKubeletProxiesRestartsSchedulableNode runs one orchestration or CLI step.
+// Signature: TestRepairBrokenKubeletProxiesRestartsSchedulableNode(t *testing.T).
+// Why: Jenkins agents depend on apiserver-to-kubelet exec; a Ready node with a
+// broken proxy needs a narrow k3s-agent restart, not a broad cluster restart.
+func TestRepairBrokenKubeletProxiesRestartsSchedulableNode(t *testing.T) {
+	cfg := config.Config{SSHManagedNodes: []string{"titan-07"}}
+	orch := buildOrchestratorWithStubs(t, cfg, nil)
+
+	healthChecks := 0
+	cordoned := false
+	restarted := false
+	waited := false
+	uncordoned := false
+	dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		joined := strings.Join(args, " ")
+		switch {
+		case name == "kubectl" && strings.Contains(joined, "get nodes -o json"):
+			return `{"items":[{"metadata":{"name":"titan-07"},"spec":{"unschedulable":false},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"titan-18"},"status":{"conditions":[{"type":"Ready","status":"Unknown"}]}}]}`, nil
+		case name == "kubectl" && strings.Contains(joined, "get --raw /api/v1/nodes/titan-07/proxy/healthz"):
+			healthChecks++
+			if healthChecks == 1 {
+				return "proxy error from 127.0.0.1:6443 while dialing 192.168.22.33:10250, code 502: 502 Bad Gateway", errors.New("exit status 1")
+			}
+			return "ok", nil
+		case name == "kubectl" && strings.Contains(joined, "uncordon titan-07"):
+			uncordoned = true
+			return "", nil
+		case name == "kubectl" && strings.Contains(joined, "cordon titan-07"):
+			cordoned = true
+			return "", nil
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl --no-block restart k3s-agent"):
+			restarted = true
+			return "", nil
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl is-active k3s-agent"):
+			return "active", nil
+		case name == "kubectl" && strings.Contains(joined, "wait node/titan-07 --for=condition=Ready --timeout=120s"):
+			waited = true
+			return "", nil
+		default:
+			return "", nil
+		}
+	}
+	orch.SetCommandOverrides(dispatch, dispatch)
+
+	repaired, err := orch.repairBrokenKubeletProxies(context.Background())
+	if err != nil {
+		t.Fatalf("repairBrokenKubeletProxies failed: %v", err)
+	}
+	if repaired != 1 {
+		t.Fatalf("expected one repaired node, got %d", repaired)
+	}
+	for name, ok := range map[string]bool{
+		"cordoned":   cordoned,
+		"restarted":  restarted,
+		"waited":     waited,
+		"uncordoned": uncordoned,
+	} {
+		if !ok {
+			t.Fatalf("expected %s action", name)
+		}
+	}
+	if healthChecks != 2 {
+		t.Fatalf("expected health check before and after repair, got %d", healthChecks)
+	}
+}
+
+// TestRepairBrokenKubeletProxiesPreservesExistingCordon runs one orchestration or CLI step.
+// Signature: TestRepairBrokenKubeletProxiesPreservesExistingCordon(t *testing.T).
+// Why: nodes intentionally kept out of service must not be accidentally
+// uncordoned just because Ananke repaired their kubelet proxy.
+func TestRepairBrokenKubeletProxiesPreservesExistingCordon(t *testing.T) {
+	cfg := config.Config{SSHManagedNodes: []string{"titan-18"}}
+	orch := buildOrchestratorWithStubs(t, cfg, nil)
+
+	healthChecks := 0
+	cordonTouched := false
+	restarted := false
+	dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		joined := strings.Join(args, " ")
+		switch {
+		case name == "kubectl" && strings.Contains(joined, "get nodes -o json"):
+			return `{"items":[{"metadata":{"name":"titan-18"},"spec":{"unschedulable":true},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`, nil
+		case name == "kubectl" && strings.Contains(joined, "get --raw /api/v1/nodes/titan-18/proxy/healthz"):
+			healthChecks++
+			if healthChecks == 1 {
+				return "error trying to reach service: proxy error from 127.0.0.1:6443 while dialing 192.168.22.46:10250", errors.New("exit status 1")
+			}
+			return "ok", nil
+		case name == "kubectl" && (strings.Contains(joined, "cordon titan-18") || strings.Contains(joined, "uncordon titan-18")):
+			cordonTouched = true
+			return "", nil
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl --no-block restart k3s-agent"):
+			restarted = true
+			return "", nil
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl is-active k3s-agent"):
+			return "active", nil
+		case name == "kubectl" && strings.Contains(joined, "wait node/titan-18 --for=condition=Ready --timeout=120s"):
+			return "", nil
+		default:
+			return "", nil
+		}
+	}
+	orch.SetCommandOverrides(dispatch, dispatch)
+
+	repaired, err := orch.repairBrokenKubeletProxies(context.Background())
+	if err != nil {
+		t.Fatalf("repairBrokenKubeletProxies failed: %v", err)
+	}
+	if repaired != 1 {
+		t.Fatalf("expected one repaired node, got %d", repaired)
+	}
+	if !restarted {
+		t.Fatalf("expected k3s-agent restart")
+	}
+	if cordonTouched {
+		t.Fatalf("did not expect cordon state to change for already-unschedulable node")
+	}
+}
+
+// TestRepairBrokenKubeletProxiesFailureBranches runs one orchestration or CLI step.
+// Signature: TestRepairBrokenKubeletProxiesFailureBranches(t *testing.T).
+// Why: auto-repair must report exact blockers without broadening into unsafe
+// node restarts for unrelated kubectl or access failures.
+func TestRepairBrokenKubeletProxiesFailureBranches(t *testing.T) {
+	t.Run("dry run skips", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, nil)
+		orch.runner.DryRun = true
+		repaired, err := orch.repairBrokenKubeletProxies(context.Background())
+		if err != nil || repaired != 0 {
+			t.Fatalf("expected dry-run skip, got repaired=%d err=%v", repaired, err)
+		}
+	})
+
+	t.Run("node query error surfaces", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{match: matchContains("kubectl", "get nodes -o json"), err: errors.New("api down")},
+		})
+		_, err := orch.repairBrokenKubeletProxies(context.Background())
+		if err == nil || !strings.Contains(err.Error(), "query nodes") {
+			t.Fatalf("expected node query error, got %v", err)
+		}
+	})
+
+	t.Run("node json decode error surfaces", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{match: matchContains("kubectl", "get nodes -o json"), out: `{bad json`},
+		})
+		_, err := orch.readyNodeCandidates(context.Background())
+		if err == nil || !strings.Contains(err.Error(), "decode nodes") {
+			t.Fatalf("expected decode error, got %v", err)
+		}
+	})
+
+	t.Run("non repairable health error is reported", func(t *testing.T) {
+		orch := kubeletProxyRepairStub(t, config.Config{}, false, []kubeletProxyRepairAction{
+			{cmd: "health", out: "forbidden", err: errors.New("forbidden")},
+		})
+		repaired, err := orch.repairBrokenKubeletProxies(context.Background())
+		if repaired != 0 || err == nil || !strings.Contains(err.Error(), "proxy health check") {
+			t.Fatalf("expected health check error, repaired=%d err=%v", repaired, err)
+		}
+	})
+
+	t.Run("unmanaged broken node is reported", func(t *testing.T) {
+		cfg := config.Config{SSHManagedNodes: []string{"other"}}
+		orch := kubeletProxyRepairStub(t, cfg, false, []kubeletProxyRepairAction{
+			{cmd: "health", out: "502 bad gateway", err: errors.New("exit status 1")},
+		})
+		repaired, err := orch.repairBrokenKubeletProxies(context.Background())
+		if repaired != 0 || err == nil || !strings.Contains(err.Error(), "not SSH-managed") {
+			t.Fatalf("expected unmanaged node error, repaired=%d err=%v", repaired, err)
+		}
+	})
+
+	t.Run("cordon failure is reported", func(t *testing.T) {
+		orch := kubeletProxyRepairStub(t, config.Config{}, false, []kubeletProxyRepairAction{
+			{cmd: "health", out: "502 bad gateway", err: errors.New("exit status 1")},
+			{cmd: "cordon", err: errors.New("cordon denied")},
+		})
+		repaired, err := orch.repairBrokenKubeletProxies(context.Background())
+		if repaired != 0 || err == nil || !strings.Contains(err.Error(), "cordon before kubelet restart") {
+			t.Fatalf("expected cordon error, repaired=%d err=%v", repaired, err)
+		}
+	})
+
+	t.Run("restart failure uncordons schedulable node", func(t *testing.T) {
+		uncordoned := false
+		orch := kubeletProxyRepairStub(t, config.Config{}, false, []kubeletProxyRepairAction{
+			{cmd: "health", out: "502 bad gateway", err: errors.New("exit status 1")},
+			{cmd: "cordon"},
+			{cmd: "restart", err: errors.New("sudo rejected")},
+			{cmd: "uncordon", sideEffect: func() { uncordoned = true }},
+		})
+		repaired, err := orch.repairBrokenKubeletProxies(context.Background())
+		if repaired != 0 || err == nil || !strings.Contains(err.Error(), "restart k3s-agent") {
+			t.Fatalf("expected restart error, repaired=%d err=%v", repaired, err)
+		}
+		if !uncordoned {
+			t.Fatalf("expected best-effort uncordon after restart failure")
+		}
+	})
+
+	t.Run("wait failure is reported", func(t *testing.T) {
+		orch := kubeletProxyRepairStub(t, config.Config{}, false, []kubeletProxyRepairAction{
+			{cmd: "health", out: "502 bad gateway", err: errors.New("exit status 1")},
+			{cmd: "cordon"},
+			{cmd: "restart"},
+			{cmd: "wait", err: errors.New("not ready")},
+		})
+		repaired, err := orch.repairBrokenKubeletProxies(context.Background())
+		if repaired != 0 || err == nil || !strings.Contains(err.Error(), "wait Ready") {
+			t.Fatalf("expected wait error, repaired=%d err=%v", repaired, err)
+		}
+	})
+
+	t.Run("post restart health failure is reported", func(t *testing.T) {
+		orch := kubeletProxyRepairStub(t, config.Config{}, false, []kubeletProxyRepairAction{
+			{cmd: "health", out: "502 bad gateway", err: errors.New("exit status 1")},
+			{cmd: "cordon"},
+			{cmd: "restart"},
+			{cmd: "wait"},
+			{cmd: "health", out: "failed to find Session for client", err: errors.New("exit status 1")},
+		})
+		repaired, err := orch.repairBrokenKubeletProxies(context.Background())
+		if repaired != 0 || err == nil || !strings.Contains(err.Error(), "proxy still broken") {
+			t.Fatalf("expected post-restart health error, repaired=%d err=%v", repaired, err)
+		}
+	})
+
+	t.Run("uncordon failure is reported", func(t *testing.T) {
+		orch := kubeletProxyRepairStub(t, config.Config{}, false, []kubeletProxyRepairAction{
+			{cmd: "health", out: "502 bad gateway", err: errors.New("exit status 1")},
+			{cmd: "cordon"},
+			{cmd: "restart"},
+			{cmd: "wait"},
+			{cmd: "health", out: "ok"},
+			{cmd: "active", out: "active"},
+			{cmd: "uncordon", err: errors.New("uncordon denied")},
+		})
+		repaired, err := orch.repairBrokenKubeletProxies(context.Background())
+		if repaired != 0 || err == nil || !strings.Contains(err.Error(), "uncordon after kubelet proxy repair") {
+			t.Fatalf("expected uncordon error, repaired=%d err=%v", repaired, err)
+		}
+	})
+}
+
+type kubeletProxyRepairAction struct {
+	cmd        string
+	out        string
+	err        error
+	sideEffect func()
+}
+
+// kubeletProxyRepairStub runs one orchestration or CLI step.
+// Signature: kubeletProxyRepairStub(t *testing.T, cfg config.Config, unschedulable bool, actions []kubeletProxyRepairAction) *Orchestrator.
+// Why: keeps the kubelet proxy repair branch tests readable while preserving
+// strict command order for safety-sensitive node operations.
+func kubeletProxyRepairStub(t *testing.T, cfg config.Config, unschedulable bool, actions []kubeletProxyRepairAction) *Orchestrator {
+	t.Helper()
+	orch := buildOrchestratorWithStubs(t, cfg, nil)
+	index := 0
+	dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		joined := strings.Join(args, " ")
+		if name == "kubectl" && strings.Contains(joined, "get nodes -o json") {
+			return fmt.Sprintf(`{"items":[{"metadata":{"name":"titan-07"},"spec":{"unschedulable":%t},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`, unschedulable), nil
+		}
+		actual := ""
+		switch {
+		case name == "kubectl" && strings.Contains(joined, "get --raw /api/v1/nodes/titan-07/proxy/healthz"):
+			actual = "health"
+		case name == "kubectl" && strings.Contains(joined, "uncordon titan-07"):
+			actual = "uncordon"
+		case name == "kubectl" && strings.Contains(joined, "cordon titan-07"):
+			actual = "cordon"
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl --no-block restart k3s-agent"):
+			actual = "restart"
+		case name == "ssh" && strings.Contains(joined, "sudo -n systemctl is-active k3s-agent"):
+			actual = "active"
+		case name == "kubectl" && strings.Contains(joined, "wait node/titan-07 --for=condition=Ready --timeout=120s"):
+			actual = "wait"
+		default:
+			return "", nil
+		}
+		if index >= len(actions) {
+			t.Fatalf("unexpected %s command after actions exhausted: %s %s", actual, name, joined)
+		}
+		action := actions[index]
+		index++
+		if action.cmd != actual {
+			t.Fatalf("expected %s command, got %s (%s %s)", action.cmd, actual, name, joined)
+		}
+		if action.sideEffect != nil {
+			action.sideEffect()
+		}
+		return action.out, action.err
+	}
+	orch.SetCommandOverrides(dispatch, dispatch)
+	return orch
+}
+
+// TestKubeletProxyHealthAndRepairableErrorHelpers runs one orchestration or CLI step.
+// Signature: TestKubeletProxyHealthAndRepairableErrorHelpers(t *testing.T).
+// Why: helper branches decide whether Ananke restarts a node agent, so both
+// positive and negative cases need direct coverage.
+func TestKubeletProxyHealthAndRepairableErrorHelpers(t *testing.T) {
+	t.Run("health error without output is preserved", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "get --raw /api/v1/nodes/titan-07/proxy/healthz"),
+				err:   errors.New("network timeout"),
+			},
+		})
+		healthy, err := orch.kubeletProxyHealthy(context.Background(), "titan-07")
+		if healthy || err == nil || !strings.Contains(err.Error(), "network timeout") {
+			t.Fatalf("expected raw health error, healthy=%v err=%v", healthy, err)
+		}
+	})
+
+	t.Run("transient killed health check is retried", func(t *testing.T) {
+		checks := 0
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: func(name string, args []string) bool {
+					if checks > 0 {
+						return false
+					}
+					if !matchContains("kubectl", "get --raw /api/v1/nodes/titan-07/proxy/healthz")(name, args) {
+						return false
+					}
+					checks++
+					return true
+				},
+				err: errors.New("signal: killed"),
+			},
+			{
+				match: matchContains("kubectl", "get --raw /api/v1/nodes/titan-07/proxy/healthz"),
+				out:   "ok",
+			},
+		})
+		healthy, err := orch.kubeletProxyHealthy(context.Background(), "titan-07")
+		if !healthy || err != nil {
+			t.Fatalf("expected retry to pass, healthy=%v err=%v", healthy, err)
+		}
+		if checks != 1 {
+			t.Fatalf("expected one transient failure before retry, got %d", checks)
+		}
+	})
+
+	for _, tc := range []struct {
+		name string
+		err  error
+		want bool
+	}{
+		{name: "nil", err: nil, want: false},
+		{name: "repairable", err: errors.New("502 Bad Gateway while dialing 10250"), want: true},
+		{name: "not repairable", err: errors.New("forbidden"), want: false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			if got := isRepairableKubeletProxyErr(tc.err); got != tc.want {
+				t.Fatalf("isRepairableKubeletProxyErr()=%v want %v", got, tc.want)
+			}
+		})
+	}
+
+	if !isTransientKubeletProxyCheckErr(errors.New("signal: killed")) {
+		t.Fatalf("expected signal killed to be transient kubelet proxy check error")
+	}
+}

internal/cluster/orchestrator_autorepair_service_test.go

@@ -0,0 +1,64 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestConfiguredServiceBackendRepairContinuesPastNoopEndpoint runs one orchestration or CLI step.
+// Signature: TestConfiguredServiceBackendRepairContinuesPastNoopEndpoint(t *testing.T).
+// Why: one unhealthy endpoint without a matching workload must not prevent later
+// critical services, especially Mailu, from getting their backend repair pass.
+func TestConfiguredServiceBackendRepairContinuesPastNoopEndpoint(t *testing.T) {
+	cfg := config.Config{
+		Startup: config.Startup{
+			CriticalServiceEndpoints: []string{
+				"orphan/missing-service",
+				"mailu-mailserver/mailu-admin",
+			},
+		},
+	}
+	orch := buildOrchestratorWithStubs(t, cfg, nil)
+
+	scaledMail := false
+	rolledOutMail := false
+	dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		if name != "kubectl" {
+			return "", nil
+		}
+		joined := strings.Join(args, " ")
+		switch {
+		case strings.Contains(joined, "-n orphan get endpoints missing-service"):
+			return "", nil
+		case strings.Contains(joined, "-n orphan scale deployment missing-service --replicas=1"):
+			return "", errors.New("not found")
+		case strings.Contains(joined, "-n orphan scale statefulset missing-service --replicas=1"):
+			return "", errors.New("not found")
+		case strings.Contains(joined, "-n mailu-mailserver get endpoints mailu-admin"):
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver scale deployment mailu-admin --replicas=1"):
+			scaledMail = true
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver rollout status deployment/mailu-admin"):
+			rolledOutMail = true
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver scale statefulset mailu-admin --replicas=1"):
+			return "", errors.New("not found")
+		default:
+			return "", nil
+		}
+	}
+	orch.SetCommandOverrides(dispatch, dispatch)
+
+	if _, err := orch.healUnreadyConfiguredServiceBackends(context.Background()); err != nil {
+		t.Fatalf("healUnreadyConfiguredServiceBackends failed: %v", err)
+	}
+	if !scaledMail || !rolledOutMail {
+		t.Fatalf("expected later mail endpoint to be repaired, scaled=%v rolledOut=%v", scaledMail, rolledOutMail)
+	}
+}

internal/cluster/orchestrator_autorepair_test.go

@@ -0,0 +1,453 @@
+package cluster
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"io"
+	"log"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+	"scm.bstein.dev/bstein/ananke/internal/execx"
+	"scm.bstein.dev/bstein/ananke/internal/state"
+)
+
+// TestPostStartAutoHealRepairsVaultAndUnavailableNodes runs one orchestration or CLI step.
+// Signature: TestPostStartAutoHealRepairsVaultAndUnavailableNodes(t *testing.T).
+// Why: covers the new daemon-triggered repair path for late Vault reseals and
+// stale terminating pods anchored to unavailable nodes.
+func TestPostStartAutoHealRepairsVaultAndUnavailableNodes(t *testing.T) {
+	cfg := config.Config{
+		Startup: config.Startup{
+			DeadNodeCleanupGraceSeconds: 300,
+			RequiredNodeLabels: map[string]map[string]string{
+				"titan-07": {"node-role.kubernetes.io/worker": "true"},
+			},
+		},
+		State: config.State{
+			Dir:            t.TempDir(),
+			ReportsDir:     filepath.Join(t.TempDir(), "reports"),
+			RunHistoryPath: filepath.Join(t.TempDir(), "runs.json"),
+		},
+	}
+	orch := &Orchestrator{
+		cfg:    cfg,
+		runner: &execx.Runner{},
+		store:  state.New(filepath.Join(t.TempDir(), "runs.json")),
+		log:    log.New(io.Discard, "", 0),
+	}
+
+	oldDelete := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
+	unsealCalls := 0
+	jobCreated := false
+	reconciled := false
+	deleted := map[string]bool{}
+	dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		if name != "kubectl" {
+			return "", nil
+		}
+		joined := strings.Join(args, " ")
+		switch {
+		case strings.Contains(joined, "label node titan-07 --overwrite node-role.kubernetes.io/worker=true"):
+			return "", nil
+		case strings.Contains(joined, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
+			return "Running", nil
+		case strings.Contains(joined, "VAULT_ADDR=http://127.0.0.1:8200 vault status -format=json"):
+			if unsealCalls == 0 {
+				return `{"initialized":true,"sealed":true}`, nil
+			}
+			return `{"initialized":true,"sealed":false}`, nil
+		case strings.Contains(joined, "-n vault get secret vault-init -o jsonpath={.data.unseal_key_b64}"):
+			return base64.StdEncoding.EncodeToString([]byte("vault-unseal-key")), nil
+		case strings.Contains(joined, "vault operator unseal"):
+			unsealCalls++
+			return "", nil
+		case strings.Contains(joined, "-n vault create job --from=cronjob/vault-k8s-auth-config"):
+			jobCreated = true
+			return "", nil
+		case strings.Contains(joined, "get nodes -o json"):
+			return `{"items":[{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"Ready","status":"Unknown"}]}},{"metadata":{"name":"titan-07"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`, nil
+		case strings.Contains(joined, "get pods -A -o json"):
+			return `{"items":[{"metadata":{"namespace":"maintenance","name":"stale-pod","deletionTimestamp":"` + oldDelete + `"},"spec":{"nodeName":"titan-22"}},{"metadata":{"namespace":"logging","name":"healthy-node-pod","deletionTimestamp":"` + oldDelete + `"},"spec":{"nodeName":"titan-18"}}]}`, nil
+		case strings.Contains(joined, "-n maintenance delete pod stale-pod --grace-period=0 --force --wait=false"):
+			deleted["maintenance/stale-pod"] = true
+			return "", nil
+		case strings.Contains(joined, "-n flux-system annotate gitrepository flux-system reconcile.fluxcd.io/requestedAt="):
+			reconciled = true
+			return "", nil
+		case strings.Contains(joined, "-n flux-system annotate kustomizations.kustomize.toolkit.fluxcd.io --all reconcile.fluxcd.io/requestedAt="):
+			return "", nil
+		case strings.Contains(joined, "annotate --all-namespaces helmreleases.helm.toolkit.fluxcd.io --all reconcile.fluxcd.io/requestedAt="):
+			return "", nil
+		default:
+			return "", nil
+		}
+	}
+	orch.SetCommandOverrides(dispatch, dispatch)
+
+	if err := orch.postStartAutoHeal(context.Background()); err != nil {
+		t.Fatalf("postStartAutoHeal failed: %v", err)
+	}
+	if unsealCalls != 1 {
+		t.Fatalf("expected one Vault unseal attempt, got %d", unsealCalls)
+	}
+	if !jobCreated {
+		t.Fatalf("expected vault k8s auth config job to be created")
+	}
+	if !deleted["maintenance/stale-pod"] {
+		t.Fatalf("expected stale unavailable-node pod to be deleted")
+	}
+	if !reconciled {
+		t.Fatalf("expected flux reconcile request after repairs")
+	}
+	if deleted["logging/healthy-node-pod"] {
+		t.Fatalf("did not expect terminating pod on healthy node to be deleted")
+	}
+}
+
+// TestPostStartAutoHealSkipsWhenClusterIsHealthy runs one orchestration or CLI step.
+// Signature: TestPostStartAutoHealSkipsWhenClusterIsHealthy(t *testing.T).
+// Why: proves the new post-start repair loop stays quiet when the specific
+// failure patterns are absent.
+func TestPostStartAutoHealSkipsWhenClusterIsHealthy(t *testing.T) {
+	cfg := config.Config{
+		Startup: config.Startup{
+			DeadNodeCleanupGraceSeconds: 300,
+		},
+		State: config.State{
+			Dir:            t.TempDir(),
+			ReportsDir:     filepath.Join(t.TempDir(), "reports"),
+			RunHistoryPath: filepath.Join(t.TempDir(), "runs.json"),
+		},
+	}
+	orch := &Orchestrator{
+		cfg:    cfg,
+		runner: &execx.Runner{},
+		store:  state.New(filepath.Join(t.TempDir(), "runs.json")),
+		log:    log.New(io.Discard, "", 0),
+	}
+
+	unsealCalls := 0
+	jobCreated := false
+	reconciled := false
+	dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		if name != "kubectl" {
+			return "", nil
+		}
+		joined := strings.Join(args, " ")
+		switch {
+		case strings.Contains(joined, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
+			return "Running", nil
+		case strings.Contains(joined, "VAULT_ADDR=http://127.0.0.1:8200 vault status -format=json"):
+			return `{"initialized":true,"sealed":false}`, nil
+		case strings.Contains(joined, "-n vault create job --from=cronjob/vault-k8s-auth-config"):
+			jobCreated = true
+			return "", nil
+		case strings.Contains(joined, "vault operator unseal"):
+			unsealCalls++
+			return "", nil
+		case strings.Contains(joined, "get nodes -o json"):
+			return `{"items":[{"metadata":{"name":"titan-07"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`, nil
+		case strings.Contains(joined, "get pods -A -o json"):
+			return `{"items":[]}`, nil
+		case strings.Contains(joined, "reconcile.fluxcd.io/requestedAt="):
+			reconciled = true
+			return "", nil
+		default:
+			return "", nil
+		}
+	}
+	orch.SetCommandOverrides(dispatch, dispatch)
+
+	if err := orch.postStartAutoHeal(context.Background()); err != nil {
+		t.Fatalf("postStartAutoHeal failed: %v", err)
+	}
+	if unsealCalls != 0 {
+		t.Fatalf("did not expect Vault unseal calls, got %d", unsealCalls)
+	}
+	if jobCreated {
+		t.Fatalf("did not expect vault auth config job creation")
+	}
+	if reconciled {
+		t.Fatalf("did not expect flux reconcile request for healthy cluster")
+	}
+}
+
+// TestPostStartAutoHealRepairsConfiguredMailBackends runs one orchestration or CLI step.
+// Signature: TestPostStartAutoHealRepairsConfiguredMailBackends(t *testing.T).
+// Why: Mailu readiness problems should be handled by Ananke's daemon repair
+// loop after startup, not only by a fresh bootstrap run.
+func TestPostStartAutoHealRepairsConfiguredMailBackends(t *testing.T) {
+	cfg := config.Config{
+		Startup: config.Startup{
+			DeadNodeCleanupGraceSeconds:     300,
+			CriticalServiceEndpoints:        []string{"mailu-mailserver/mailu-admin"},
+			NodeRuntimeRestartWaitSeconds:   1,
+			NodeRuntimeRebootWaitSeconds:    1,
+			HostPrivilegedCommandTimeoutSec: 1,
+		},
+	}
+	orch := buildOrchestratorWithStubs(t, cfg, nil)
+
+	scaled := false
+	rolledOut := false
+	reconciled := false
+	dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		if name != "kubectl" {
+			return "", nil
+		}
+		joined := strings.Join(args, " ")
+		switch {
+		case strings.Contains(joined, "-n longhorn-system get nodes.longhorn.io -o json"):
+			return `{"items":[]}`, nil
+		case strings.Contains(joined, "-n vault get pod vault-0 -o jsonpath={.status.phase}"):
+			return "", nil
+		case strings.Contains(joined, "get nodes -o json"):
+			return `{"items":[]}`, nil
+		case strings.Contains(joined, "get pods -A -o json"):
+			return `{"items":[]}`, nil
+		case strings.Contains(joined, "get events -A -o json"):
+			return `{"items":[]}`, nil
+		case strings.Contains(joined, "get pvc -A -o json"):
+			return `{"items":[]}`, nil
+		case strings.Contains(joined, "-n mailu-mailserver get endpoints mailu-admin"):
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver scale deployment mailu-admin --replicas=1"):
+			scaled = true
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver rollout status deployment/mailu-admin"):
+			rolledOut = true
+			return "", nil
+		case strings.Contains(joined, "-n mailu-mailserver scale statefulset mailu-admin --replicas=1"):
+			return "", errors.New("not found")
+		case strings.Contains(joined, "-n flux-system annotate gitrepository flux-system reconcile.fluxcd.io/requestedAt="):
+			reconciled = true
+			return "", nil
+		case strings.Contains(joined, "-n flux-system annotate kustomizations.kustomize.toolkit.fluxcd.io --all reconcile.fluxcd.io/requestedAt="):
+			return "", nil
+		case strings.Contains(joined, "annotate --all-namespaces helmreleases.helm.toolkit.fluxcd.io --all reconcile.fluxcd.io/requestedAt="):
+			return "", nil
+		default:
+			return "", nil
+		}
+	}
+	orch.SetCommandOverrides(dispatch, dispatch)
+
+	if err := orch.postStartAutoHeal(context.Background()); err != nil {
+		t.Fatalf("postStartAutoHeal failed: %v", err)
+	}
+	if !scaled || !rolledOut {
+		t.Fatalf("expected mailu-admin deployment to be scaled and rolled out, scaled=%v rolledOut=%v", scaled, rolledOut)
+	}
+	if !reconciled {
+		t.Fatalf("expected flux reconcile after mail backend repair")
+	}
+}
+
+// TestRunPostStartAutoHealDryRun runs one orchestration or CLI step.
+// Signature: TestRunPostStartAutoHealDryRun(t *testing.T).
+// Why: covers the exported wrapper and the top-level dry-run guard so daemon
+// auto-heal never mutates cluster state during rehearsal runs.
+func TestRunPostStartAutoHealDryRun(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{}, nil)
+	orch.runner.DryRun = true
+
+	if err := orch.RunPostStartAutoHeal(context.Background()); err != nil {
+		t.Fatalf("RunPostStartAutoHeal dry-run failed: %v", err)
+	}
+}
+
+// TestPostStartAutoHealAggregatesErrors runs one orchestration or CLI step.
+// Signature: TestPostStartAutoHealAggregatesErrors(t *testing.T).
+// Why: proves the daemon reports each failed sub-repair together instead of
+// hiding later failures behind the first problem.
+func TestPostStartAutoHealAggregatesErrors(t *testing.T) {
+	cfg := config.Config{
+		Startup: config.Startup{
+			DeadNodeCleanupGraceSeconds: 300,
+			RequiredNodeLabels: map[string]map[string]string{
+				"titan-07": {"node-role.kubernetes.io/worker": "true"},
+			},
+		},
+	}
+	orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
+		{
+			match: matchContains("kubectl", "label node titan-07 --overwrite node-role.kubernetes.io/worker=true"),
+			err:   errors.New("label failed"),
+		},
+		{
+			match: matchContains("kubectl", "-n vault get pod vault-0 -o jsonpath={.status.phase}"),
+			err:   errors.New("vault phase failed"),
+		},
+		{
+			match: matchContains("kubectl", "get nodes -o json"),
+			err:   errors.New("node query failed"),
+		},
+	})
+
+	err := orch.postStartAutoHeal(context.Background())
+	if err == nil {
+		t.Fatalf("expected aggregated error")
+	}
+	msg := err.Error()
+	for _, want := range []string{
+		"required node labels:",
+		"vault auto-recovery:",
+		"dead-node terminating pod cleanup:",
+	} {
+		if !strings.Contains(msg, want) {
+			t.Fatalf("expected %q in %q", want, msg)
+		}
+	}
+}
+
+// TestAutoRecoverSealedVaultBranches runs one orchestration or CLI step.
+// Signature: TestAutoRecoverSealedVaultBranches(t *testing.T).
+// Why: late Vault reseals are a high-risk failure path, so the daemon needs
+// coverage across the quiet-skip, parse-failure, and unseal-failure branches.
+func TestAutoRecoverSealedVaultBranches(t *testing.T) {
+	t.Run("dry run skips", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, nil)
+		orch.runner.DryRun = true
+
+		recovered, err := orch.autoRecoverSealedVault(context.Background())
+		if err != nil || recovered {
+			t.Fatalf("expected dry-run skip, got recovered=%v err=%v", recovered, err)
+		}
+	})
+
+	t.Run("pod missing is quiet", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "-n vault get pod vault-0 -o jsonpath={.status.phase}"),
+				err:   errors.New("vault-0 not found"),
+			},
+		})
+
+		recovered, err := orch.autoRecoverSealedVault(context.Background())
+		if err != nil || recovered {
+			t.Fatalf("expected quiet skip, got recovered=%v err=%v", recovered, err)
+		}
+	})
+
+	t.Run("phase check error surfaces", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "-n vault get pod vault-0 -o jsonpath={.status.phase}"),
+				err:   errors.New("phase check failed"),
+			},
+		})
+
+		recovered, err := orch.autoRecoverSealedVault(context.Background())
+		if recovered || err == nil || !strings.Contains(err.Error(), "vault pod phase check failed") {
+			t.Fatalf("expected phase check error, got recovered=%v err=%v", recovered, err)
+		}
+	})
+
+	t.Run("non-running pod defers", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "-n vault get pod vault-0 -o jsonpath={.status.phase}"),
+				out:   "Pending",
+			},
+		})
+
+		recovered, err := orch.autoRecoverSealedVault(context.Background())
+		if err != nil || recovered {
+			t.Fatalf("expected pending pod skip, got recovered=%v err=%v", recovered, err)
+		}
+	})
+
+	t.Run("status parse failure surfaces", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "-n vault get pod vault-0 -o jsonpath={.status.phase}"),
+				out:   "Running",
+			},
+			{
+				match: matchContains("kubectl", "VAULT_ADDR=http://127.0.0.1:8200 vault status -format=json"),
+				out:   "garbage",
+			},
+		})
+
+		recovered, err := orch.autoRecoverSealedVault(context.Background())
+		if recovered || err == nil || !strings.Contains(err.Error(), "parse vault status") {
+			t.Fatalf("expected parse error, got recovered=%v err=%v", recovered, err)
+		}
+	})
+
+	t.Run("already unsealed stays quiet", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "-n vault get pod vault-0 -o jsonpath={.status.phase}"),
+				out:   "Running",
+			},
+			{
+				match: matchContains("kubectl", "VAULT_ADDR=http://127.0.0.1:8200 vault status -format=json"),
+				out:   `{"sealed":false}`,
+			},
+		})
+
+		recovered, err := orch.autoRecoverSealedVault(context.Background())
+		if err != nil || recovered {
+			t.Fatalf("expected already-unsealed skip, got recovered=%v err=%v", recovered, err)
+		}
+	})
+
+	t.Run("unseal failure surfaces", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "-n vault get pod vault-0 -o jsonpath={.status.phase}"),
+				out:   "Running",
+			},
+			{
+				match: matchContains("kubectl", "VAULT_ADDR=http://127.0.0.1:8200 vault status -format=json"),
+				out:   `{"sealed":true}`,
+			},
+			{
+				match: matchContains("kubectl", "-n vault get secret vault-init -o jsonpath={.data.unseal_key_b64}"),
+				out:   base64.StdEncoding.EncodeToString([]byte("vault-unseal-key")),
+			},
+			{
+				match: matchContains("kubectl", "vault operator unseal"),
+				err:   errors.New("exec boom"),
+			},
+		})
+
+		recovered, err := orch.autoRecoverSealedVault(context.Background())
+		if recovered || err == nil || !strings.Contains(err.Error(), "vault unseal attempt 1 failed") {
+			t.Fatalf("expected unseal failure, got recovered=%v err=%v", recovered, err)
+		}
+	})
+}
+
+// TestRerunVaultK8sAuthConfigJobBranches runs one orchestration or CLI step.
+// Signature: TestRerunVaultK8sAuthConfigJobBranches(t *testing.T).
+// Why: the post-unseal auth job is part of the production recovery chain, so
+// dry-run and create-error behavior both need explicit coverage.
+func TestRerunVaultK8sAuthConfigJobBranches(t *testing.T) {
+	t.Run("dry run skips", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, nil)
+		orch.runner.DryRun = true
+		if err := orch.rerunVaultK8sAuthConfigJob(context.Background()); err != nil {
+			t.Fatalf("dry-run rerunVaultK8sAuthConfigJob failed: %v", err)
+		}
+	})
+
+	t.Run("create error surfaces", func(t *testing.T) {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{
+				match: matchContains("kubectl", "-n vault create job --from=cronjob/vault-k8s-auth-config"),
+				err:   errors.New("create failed"),
+			},
+		})
+		err := orch.rerunVaultK8sAuthConfigJob(context.Background())
+		if err == nil || !strings.Contains(err.Error(), "create job vault-k8s-auth-config-autoheal-") {
+			t.Fatalf("expected create-job error, got %v", err)
+		}
+	})
+}

internal/cluster/orchestrator_cordon_lease.go

@@ -0,0 +1,354 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+)
+
+const (
+	anankeCordonOwnerAnnotation        = "ananke.bstein.dev/cordon-owner"
+	anankeCordonReasonAnnotation       = "ananke.bstein.dev/cordon-reason"
+	anankeCordonDetailAnnotation       = "ananke.bstein.dev/cordon-detail"
+	anankeCordonCreatedAnnotation      = "ananke.bstein.dev/cordon-created-at"
+	anankeCordonDeadlineAnnotation     = "ananke.bstein.dev/cordon-deadline"
+	anankeCordonManualActionAnnotation = "ananke.bstein.dev/manual-action"
+	anankeCordonManualAtAnnotation     = "ananke.bstein.dev/manual-action-at"
+)
+
+const (
+	cordonReasonMissingCryptsetup = "missing-cryptsetup"
+	cordonReasonRuntimeWedge      = "container-runtime-wedge"
+	cordonReasonKubeletProxy      = "kubelet-proxy-repair"
+)
+
+// cordonNodeWithLease cordons a node and records an expiry-bound owner/reason.
+// Signature: (o *Orchestrator) cordonNodeWithLease(ctx context.Context, node, reason, detail string) error.
+// Why: recovery cordons must be accountable leases so a node cannot be stranded
+// indefinitely after an automatic repair attempt.
+func (o *Orchestrator) cordonNodeWithLease(ctx context.Context, node string, reason string, detail string) error {
+	node = strings.TrimSpace(node)
+	reason = strings.TrimSpace(reason)
+	detail = sanitizeCordonAnnotationValue(detail)
+	if node == "" {
+		return fmt.Errorf("node must not be empty")
+	}
+	if reason == "" {
+		reason = "recovery"
+	}
+	now := time.Now().UTC()
+	deadline := now.Add(o.recoveryCordonMaxDuration())
+	if err := o.annotateCordonLease(ctx, node, reason, detail, now, deadline); err != nil {
+		return err
+	}
+	if _, err := o.kubectl(ctx, 30*time.Second, "cordon", node); err != nil {
+		o.bestEffort("clear cordon lease after cordon failure", func() error {
+			return o.clearCordonLease(ctx, node)
+		})
+		return err
+	}
+	o.log.Printf("cordoned node %s with ananke lease reason=%s deadline=%s", node, reason, deadline.Format(time.RFC3339))
+	return nil
+}
+
+// annotateCordonLease records why Ananke is cordoning a node and when it expires.
+// Signature: (o *Orchestrator) annotateCordonLease(ctx context.Context, node, reason, detail string, created, deadline time.Time) error.
+// Why: the annotations are the durable handoff between startup recovery and the
+// daemon's post-start reconciliation loop.
+func (o *Orchestrator) annotateCordonLease(ctx context.Context, node string, reason string, detail string, created time.Time, deadline time.Time) error {
+	args := []string{
+		"annotate", "node", node,
+		anankeCordonOwnerAnnotation + "=ananke",
+		anankeCordonReasonAnnotation + "=" + reason,
+		anankeCordonDetailAnnotation + "=" + detail,
+		anankeCordonCreatedAnnotation + "=" + created.Format(time.RFC3339),
+		anankeCordonDeadlineAnnotation + "=" + deadline.Format(time.RFC3339),
+		"--overwrite",
+	}
+	if _, err := o.kubectl(ctx, 20*time.Second, args...); err != nil {
+		return fmt.Errorf("annotate cordon lease on %s: %w", node, err)
+	}
+	return nil
+}
+
+// clearCordonLease removes Ananke's cordon ownership markers from a node.
+// Signature: (o *Orchestrator) clearCordonLease(ctx context.Context, node string) error.
+// Why: once a node is safe to schedule again, stale lease annotations should not
+// keep paging operators or confusing later recovery loops.
+func (o *Orchestrator) clearCordonLease(ctx context.Context, node string) error {
+	args := []string{
+		"annotate", "node", node,
+		anankeCordonOwnerAnnotation + "-",
+		anankeCordonReasonAnnotation + "-",
+		anankeCordonDetailAnnotation + "-",
+		anankeCordonCreatedAnnotation + "-",
+		anankeCordonDeadlineAnnotation + "-",
+		anankeCordonManualActionAnnotation + "-",
+		anankeCordonManualAtAnnotation + "-",
+		"--overwrite",
+	}
+	if _, err := o.kubectl(ctx, 20*time.Second, args...); err != nil && !isNotFoundErr(err) {
+		return fmt.Errorf("clear cordon lease on %s: %w", node, err)
+	}
+	return nil
+}
+
+// uncordonAndClearCordonLease returns a recovered node to service.
+// Signature: (o *Orchestrator) uncordonAndClearCordonLease(ctx context.Context, node, reason string) error.
+// Why: the uncordon and annotation cleanup should happen as one logical recovery
+// action so leased cordons do not linger after the hazard is gone.
+func (o *Orchestrator) uncordonAndClearCordonLease(ctx context.Context, node string, reason string) error {
+	if _, err := o.kubectl(ctx, 30*time.Second, "uncordon", node); err != nil {
+		return fmt.Errorf("uncordon %s after %s recovery: %w", node, reason, err)
+	}
+	if err := o.clearCordonLease(ctx, node); err != nil {
+		o.log.Printf("warning: node %s uncordoned but lease cleanup failed: %v", node, err)
+	}
+	o.log.Printf("uncordoned node %s after recovering leased cordon reason=%s", node, reason)
+	o.noteStartupAutoHeal(fmt.Sprintf("uncordoned %s after recovering %s", node, reason))
+	return nil
+}
+
+// enforceRecoveryCordonLeases reconciles Ananke-created cordons and reports stale
+// manual cordons.
+// Signature: (o *Orchestrator) enforceRecoveryCordonLeases(ctx context.Context) (int, error).
+// Why: automatic recovery can temporarily cordon a node, but it must either clear
+// that cordon or tell the operator before the resource is stranded.
+func (o *Orchestrator) enforceRecoveryCordonLeases(ctx context.Context) (int, error) {
+	nodes, err := o.queryReadyNodes(ctx)
+	if err != nil {
+		return 0, err
+	}
+	maxAge := o.recoveryCordonMaxDuration()
+	now := time.Now().UTC()
+	released := 0
+	errs := []string{}
+	for _, node := range nodes.Items {
+		name := strings.TrimSpace(node.Metadata.Name)
+		if name == "" {
+			continue
+		}
+		ann := node.Metadata.Annotations
+		if !node.Spec.Unschedulable {
+			if ann[anankeCordonOwnerAnnotation] == "ananke" {
+				if err := o.clearCordonLease(ctx, name); err != nil {
+					errs = append(errs, err.Error())
+				}
+			}
+			continue
+		}
+		if ann[anankeCordonOwnerAnnotation] != "ananke" {
+			cordonTime := nodeUnschedulableSince(node)
+			if !cordonTime.IsZero() && now.Sub(cordonTime) > maxAge {
+				message := fmt.Sprintf("manual action required: node %s has an unowned cordon older than %s; Ananke will not silently leave resources stranded or override a non-Ananke cordon", name, maxAge)
+				if !manualActionAlreadyMarked(ann, message) {
+					o.markManualActionRequired(ctx, name, message)
+					errs = append(errs, message)
+				}
+			}
+			continue
+		}
+
+		recovered, recoverErr := o.recoverLeasedCordon(ctx, name, ann)
+		if recovered {
+			released++
+			continue
+		}
+		if recoverErr == nil {
+			continue
+		}
+		expired, deadlineText := cordonLeaseExpired(ann, now)
+		if expired {
+			message := fmt.Sprintf("manual action required: node %s remains cordoned after Ananke repair lease expired at %s: %v", name, deadlineText, recoverErr)
+			if !manualActionAlreadyMarked(ann, message) {
+				o.markManualActionRequired(ctx, name, message)
+				errs = append(errs, message)
+			}
+		} else {
+			o.log.Printf("leased cordon on %s still pending repair before %s: %v", name, deadlineText, recoverErr)
+		}
+	}
+	if len(errs) > 0 {
+		return released, errors.New(strings.Join(errs, "; "))
+	}
+	return released, nil
+}
+
+// recoverLeasedCordon tries the reason-specific repair for an Ananke cordon.
+// Signature: (o *Orchestrator) recoverLeasedCordon(ctx context.Context, node string, annotations map[string]string) (bool, error).
+// Why: each cordon reason has a different safety gate; a node only comes back when
+// the original hazard is known to be gone.
+func (o *Orchestrator) recoverLeasedCordon(ctx context.Context, node string, annotations map[string]string) (bool, error) {
+	reason := strings.TrimSpace(annotations[anankeCordonReasonAnnotation])
+	switch reason {
+	case cordonReasonMissingCryptsetup:
+		if err := o.ensureHostCryptsetup(ctx, node); err != nil {
+			return false, fmt.Errorf("cryptsetup repair still failing: %w", err)
+		}
+		if err := o.uncordonAndClearCordonLease(ctx, node, reason); err != nil {
+			return false, err
+		}
+		return true, nil
+	case cordonReasonRuntimeWedge:
+		stillWedged, err := o.nodeStillHasRuntimeWedge(ctx, node)
+		if err != nil {
+			return false, err
+		}
+		if stillWedged {
+			return false, fmt.Errorf("container runtime wedge evidence is still present")
+		}
+		if err := o.uncordonAndClearCordonLease(ctx, node, reason); err != nil {
+			return false, err
+		}
+		return true, nil
+	case cordonReasonKubeletProxy:
+		healthy, err := o.kubeletProxyHealthy(ctx, node)
+		if !healthy {
+			return false, fmt.Errorf("kubelet proxy still unhealthy: %v", err)
+		}
+		if err := o.uncordonAndClearCordonLease(ctx, node, reason); err != nil {
+			return false, err
+		}
+		return true, nil
+	default:
+		return false, fmt.Errorf("unknown Ananke cordon reason %q", reason)
+	}
+}
+
+// nodeStillHasRuntimeWedge checks whether the original runtime-wedge symptom remains.
+// Signature: (o *Orchestrator) nodeStillHasRuntimeWedge(ctx context.Context, node string) (bool, error).
+// Why: a runtime-wedge cordon can be released once the cluster no longer shows
+// repeated controller-owned, non-PVC pod start failures on that node.
+func (o *Orchestrator) nodeStillHasRuntimeWedge(ctx context.Context, node string) (bool, error) {
+	out, err := o.kubectl(ctx, 30*time.Second, "get", "pods", "-A", "-o", "json")
+	if err != nil {
+		return false, fmt.Errorf("query pods for runtime-wedge recovery: %w", err)
+	}
+	var list podList
+	if err := json.Unmarshal([]byte(out), &list); err != nil {
+		return false, fmt.Errorf("decode pods for runtime-wedge recovery: %w", err)
+	}
+	grace := time.Duration(o.cfg.Startup.StuckPodGraceSeconds) * time.Second
+	if grace <= 0 {
+		grace = 180 * time.Second
+	}
+	reasons, err := o.containerRuntimeWedgePodReasons(ctx, list, grace)
+	if err != nil {
+		return false, err
+	}
+	for _, pod := range list.Items {
+		ns := strings.TrimSpace(pod.Metadata.Namespace)
+		name := strings.TrimSpace(pod.Metadata.Name)
+		if ns == "" || name == "" || strings.TrimSpace(pod.Spec.NodeName) != node {
+			continue
+		}
+		if reasons[ns+"/"+name] == "" {
+			continue
+		}
+		if podUsesPersistentVolumeClaim(pod) || !podControllerOwned(pod) {
+			continue
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
+// queryReadyNodes fetches nodes using the shared light-weight node shape.
+// Signature: (o *Orchestrator) queryReadyNodes(ctx context.Context) (nodeReadyList, error).
+// Why: lease checks need annotations and taint timestamps in addition to Ready state.
+func (o *Orchestrator) queryReadyNodes(ctx context.Context) (nodeReadyList, error) {
+	out, err := o.kubectl(ctx, 20*time.Second, "get", "nodes", "-o", "json")
+	if err != nil {
+		return nodeReadyList{}, fmt.Errorf("query nodes: %w", err)
+	}
+	var nodes nodeReadyList
+	if err := json.Unmarshal([]byte(out), &nodes); err != nil {
+		return nodeReadyList{}, fmt.Errorf("decode nodes: %w", err)
+	}
+	return nodes, nil
+}
+
+// markManualActionRequired makes a stale cordon visible on the node object itself.
+// Signature: (o *Orchestrator) markManualActionRequired(ctx context.Context, node, message string).
+// Why: daemon logs are easy to miss during recovery; the node should carry the
+// operator-facing action that prevented automatic release.
+func (o *Orchestrator) markManualActionRequired(ctx context.Context, node string, message string) {
+	now := time.Now().UTC().Format(time.RFC3339)
+	message = sanitizeCordonAnnotationValue(message)
+	if _, err := o.kubectl(
+		ctx,
+		20*time.Second,
+		"annotate", "node", node,
+		anankeCordonManualActionAnnotation+"="+message,
+		anankeCordonManualAtAnnotation+"="+now,
+		"--overwrite",
+	); err != nil {
+		o.log.Printf("warning: mark manual action required on %s failed: %v", node, err)
+	}
+}
+
+// manualActionAlreadyMarked reports whether a node already carries this action.
+// Signature: manualActionAlreadyMarked(annotations map[string]string, message string) bool.
+// Why: repeated recovery scans should not keep rewriting the same node annotation
+// or turning an already-visible manual task into fresh log noise.
+func manualActionAlreadyMarked(annotations map[string]string, message string) bool {
+	if annotations == nil {
+		return false
+	}
+	return annotations[anankeCordonManualActionAnnotation] == sanitizeCordonAnnotationValue(message)
+}
+
+// recoveryCordonMaxDuration returns the maximum allowed automatic cordon lease.
+// Signature: (o *Orchestrator) recoveryCordonMaxDuration() time.Duration.
+// Why: all recovery cordon decisions should share the same operator promise.
+func (o *Orchestrator) recoveryCordonMaxDuration() time.Duration {
+	seconds := o.cfg.Startup.RecoveryCordonMaxSeconds
+	if seconds <= 0 {
+		seconds = 3600
+	}
+	return time.Duration(seconds) * time.Second
+}
+
+// cordonLeaseExpired reports whether a stored cordon lease is past deadline.
+// Signature: cordonLeaseExpired(annotations map[string]string, now time.Time) (bool, string).
+// Why: malformed or missing deadlines should fail closed so Ananke does not
+// treat an indefinite cordon as healthy automation.
+func cordonLeaseExpired(annotations map[string]string, now time.Time) (bool, string) {
+	rawDeadline := strings.TrimSpace(annotations[anankeCordonDeadlineAnnotation])
+	if rawDeadline == "" {
+		return true, "missing deadline"
+	}
+	deadline, err := time.Parse(time.RFC3339, rawDeadline)
+	if err != nil {
+		return true, rawDeadline
+	}
+	return !now.Before(deadline), deadline.Format(time.RFC3339)
+}
+
+// nodeUnschedulableSince reads the timestamp Kubernetes recorded for a cordon.
+// Signature: nodeUnschedulableSince(node nodeReadyItem) time.Time.
+// Why: unowned cordons need an age check before Ananke escalates them to manual
+// action.
+func nodeUnschedulableSince(node nodeReadyItem) time.Time {
+	for _, taint := range node.Spec.Taints {
+		if taint.Key == "node.kubernetes.io/unschedulable" {
+			return taint.TimeAdded
+		}
+	}
+	return time.Time{}
+}
+
+// sanitizeCordonAnnotationValue keeps node annotation values compact.
+// Signature: sanitizeCordonAnnotationValue(value string) string.
+// Why: recovery details are operator-facing, but Kubernetes annotation values
+// should stay short enough to remain readable in node listings.
+func sanitizeCordonAnnotationValue(value string) string {
+	value = strings.Join(strings.Fields(strings.TrimSpace(value)), " ")
+	if len(value) <= 240 {
+		return value
+	}
+	return value[:240]
+}

internal/cluster/orchestrator_cordon_lease_test.go

@@ -0,0 +1,175 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestEnforceRecoveryCordonLeasesUncordonsRecoveredCryptsetupNode runs one orchestration or CLI step.
+// Signature: TestEnforceRecoveryCordonLeasesUncordonsRecoveredCryptsetupNode(t *testing.T).
+// Why: a cryptsetup safety cordon should clear automatically once the host
+// prerequisite is actually present.
+func TestEnforceRecoveryCordonLeasesUncordonsRecoveredCryptsetupNode(t *testing.T) {
+	now := time.Now().UTC()
+	nodeJSON := leasedNodeJSON(
+		"titan-15",
+		cordonReasonMissingCryptsetup,
+		now.Add(-10*time.Minute),
+		now.Add(50*time.Minute),
+	)
+	uncordoned := false
+	cleared := false
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{RecoveryCordonMaxSeconds: 3600},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: nodeJSON},
+		{match: matchContains("ssh", "cryptsetup"), out: "__ANANKE_CRYPTSETUP_PRESENT__"},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "uncordon", "titan-15")(name, args) {
+					return false
+				}
+				uncordoned = true
+				return true
+			},
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "annotate", "node", "titan-15", anankeCordonOwnerAnnotation+"-")(name, args) {
+					return false
+				}
+				cleared = true
+				return true
+			},
+		},
+	})
+
+	released, err := orch.enforceRecoveryCordonLeases(context.Background())
+	if err != nil {
+		t.Fatalf("enforceRecoveryCordonLeases failed: %v", err)
+	}
+	if released != 1 || !uncordoned || !cleared {
+		t.Fatalf("expected recovered cordon release, released=%d uncordoned=%v cleared=%v", released, uncordoned, cleared)
+	}
+}
+
+// TestEnforceRecoveryCordonLeasesReportsStaleUnownedCordon runs one orchestration or CLI step.
+// Signature: TestEnforceRecoveryCordonLeasesReportsStaleUnownedCordon(t *testing.T).
+// Why: Ananke must not overwrite a manual cordon, but it also must not let it sit
+// silently beyond the recovery lease window.
+func TestEnforceRecoveryCordonLeasesReportsStaleUnownedCordon(t *testing.T) {
+	old := time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339)
+	nodeJSON := `{"items":[{"metadata":{"name":"titan-18","annotations":{}},"spec":{"unschedulable":true,"taints":[{"key":"node.kubernetes.io/unschedulable","timeAdded":"` + old + `"}]},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`
+	manualMarked := false
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{RecoveryCordonMaxSeconds: 3600},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: nodeJSON},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "annotate", "node", "titan-18", anankeCordonManualActionAnnotation+"=")(name, args) {
+					return false
+				}
+				manualMarked = true
+				return true
+			},
+		},
+	})
+
+	released, err := orch.enforceRecoveryCordonLeases(context.Background())
+	if released != 0 || err == nil || !strings.Contains(err.Error(), "unowned cordon") {
+		t.Fatalf("expected stale unowned cordon error, released=%d err=%v", released, err)
+	}
+	if !manualMarked {
+		t.Fatalf("expected manual-action annotation")
+	}
+}
+
+// TestEnforceRecoveryCordonLeasesDoesNotRepeatExistingManualAction runs one orchestration or CLI step.
+// Signature: TestEnforceRecoveryCordonLeasesDoesNotRepeatExistingManualAction(t *testing.T).
+// Why: once a manual action is visible on the node, later scans should not keep
+// rewriting the same annotation or producing fresh alert noise.
+func TestEnforceRecoveryCordonLeasesDoesNotRepeatExistingManualAction(t *testing.T) {
+	old := time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339)
+	message := "manual action required: node titan-18 has an unowned cordon older than 1h0m0s; Ananke will not silently leave resources stranded or override a non-Ananke cordon"
+	nodeJSON := `{"items":[{"metadata":{"name":"titan-18","annotations":{"` +
+		anankeCordonManualActionAnnotation + `":"` + message + `","` +
+		anankeCordonManualAtAnnotation + `":"` + old + `"}},"spec":{"unschedulable":true,"taints":[{"key":"node.kubernetes.io/unschedulable","timeAdded":"` +
+		old + `"}]},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{RecoveryCordonMaxSeconds: 3600},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: nodeJSON},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "annotate", "node", "titan-18")(name, args) {
+					return false
+				}
+				t.Fatalf("manual-action annotation should not be rewritten")
+				return true
+			},
+		},
+	})
+
+	released, err := orch.enforceRecoveryCordonLeases(context.Background())
+	if released != 0 || err != nil {
+		t.Fatalf("expected quiet existing manual action, released=%d err=%v", released, err)
+	}
+}
+
+// TestEnforceRecoveryCordonLeasesEscalatesExpiredCryptsetupRepair runs one orchestration or CLI step.
+// Signature: TestEnforceRecoveryCordonLeasesEscalatesExpiredCryptsetupRepair(t *testing.T).
+// Why: after the lease expires, a failed automatic repair must become a clear
+// manual action rather than another quiet retry.
+func TestEnforceRecoveryCordonLeasesEscalatesExpiredCryptsetupRepair(t *testing.T) {
+	now := time.Now().UTC()
+	nodeJSON := leasedNodeJSON(
+		"titan-17",
+		cordonReasonMissingCryptsetup,
+		now.Add(-2*time.Hour),
+		now.Add(-time.Hour),
+	)
+	manualMarked := false
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{RecoveryCordonMaxSeconds: 3600},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: nodeJSON},
+		{match: matchContains("ssh", "cryptsetup"), err: errors.New("sudo rejected")},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "annotate", "node", "titan-17", anankeCordonManualActionAnnotation+"=")(name, args) {
+					return false
+				}
+				manualMarked = true
+				return true
+			},
+		},
+	})
+
+	released, err := orch.enforceRecoveryCordonLeases(context.Background())
+	if released != 0 || err == nil || !strings.Contains(err.Error(), "manual action required") {
+		t.Fatalf("expected expired lease manual action, released=%d err=%v", released, err)
+	}
+	if !manualMarked {
+		t.Fatalf("expected manual-action annotation")
+	}
+}
+
+// leasedNodeJSON builds a minimal node-list payload with an Ananke cordon lease.
+// Signature: leasedNodeJSON(node, reason string, created, deadline time.Time) string.
+// Why: the lease tests only need node annotations, cordon state, and Ready
+// condition, so a small fixture keeps intent clear.
+func leasedNodeJSON(node string, reason string, created time.Time, deadline time.Time) string {
+	return `{"items":[{"metadata":{"name":"` + node + `","annotations":{"` +
+		anankeCordonOwnerAnnotation + `":"ananke","` +
+		anankeCordonReasonAnnotation + `":"` + reason + `","` +
+		anankeCordonCreatedAnnotation + `":"` + created.Format(time.RFC3339) + `","` +
+		anankeCordonDeadlineAnnotation + `":"` + deadline.Format(time.RFC3339) +
+		`"}},"spec":{"unschedulable":true,"taints":[{"key":"node.kubernetes.io/unschedulable","timeAdded":"` + created.Format(time.RFC3339) +
+		`"}]},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`
+}

internal/cluster/orchestrator_core.go

@@ -20,6 +20,8 @@ type Orchestrator struct {
 	log                  *log.Logger
 	runOverride          func(timeoutCtx context.Context, timeout time.Duration, name string, args ...string) (string, error)
 	runSensitiveOverride func(timeoutCtx context.Context, timeout time.Duration, name string, args ...string) (string, error)
+	sshInputOverride     func(timeoutCtx context.Context, timeout time.Duration, node string, command string, input string) (string, error)
+	credentialRepairMu   sync.Mutex
 	startupReportMu      sync.Mutex
 	activeStartupReport  *startupReport
 }
@@ -122,3 +124,11 @@ func (o *Orchestrator) SetCommandOverrides(run commandOverrideFunc, runSensitive
 	o.runOverride = run
 	o.runSensitiveOverride = runSensitive
 }
+
+// SetSSHInputOverride runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) SetSSHInputOverride(run func(context.Context, time.Duration, string, string, string) (string, error)).
+// Why: password-backed host-repair tests need to assert stdin behavior without
+// exposing secrets through ordinary command-argument stubs.
+func (o *Orchestrator) SetSSHInputOverride(run func(context.Context, time.Duration, string, string, string) (string, error)) {
+	o.sshInputOverride = run
+}

internal/cluster/orchestrator_critical_endpoint_additional_test.go

@@ -0,0 +1,99 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+	"scm.bstein.dev/bstein/ananke/internal/execx"
+	"scm.bstein.dev/bstein/ananke/internal/state"
+)
+
+// TestCriticalEndpointHelpers runs one orchestration or CLI step.
+// Signature: TestCriticalEndpointHelpers(t *testing.T).
+// Why: covers critical endpoint parsing and readiness checks that gate startup completion.
+func TestCriticalEndpointHelpers(t *testing.T) {
+	cfg := config.Config{
+		Startup: config.Startup{
+			CriticalServiceEndpoints: []string{"monitoring/victoria-metrics-single-server"},
+		},
+	}
+	orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
+		{match: matchContains("kubectl", "get endpoints victoria-metrics-single-server"), out: "10.42.0.10\n10.42.0.11\n"},
+	})
+	ok, detail, ns, svc, err := orch.criticalServiceEndpointsReady(context.Background())
+	if err != nil || !ok {
+		t.Fatalf("expected criticalServiceEndpointsReady success, got ok=%v detail=%q ns=%q svc=%q err=%v", ok, detail, ns, svc, err)
+	}
+	if detail != "services=1" {
+		t.Fatalf("unexpected readiness detail: %q", detail)
+	}
+	gotNS, gotSvc, err := parseCriticalServiceEndpoint("monitoring/victoria-metrics-single-server")
+	if err != nil || gotNS != "monitoring" || gotSvc != "victoria-metrics-single-server" {
+		t.Fatalf("unexpected parse result ns=%q svc=%q err=%v", gotNS, gotSvc, err)
+	}
+	if _, _, err := parseCriticalServiceEndpoint("invalid"); err == nil {
+		t.Fatalf("expected parseCriticalServiceEndpoint error")
+	}
+}
+
+// TestCriticalEndpointAutoHealWorkflow runs one orchestration or CLI step.
+// Signature: TestCriticalEndpointAutoHealWorkflow(t *testing.T).
+// Why: covers endpoint-zero recovery where startup heals workload replicas before succeeding.
+func TestCriticalEndpointAutoHealWorkflow(t *testing.T) {
+	cfg := config.Config{
+		Startup: config.Startup{
+			CriticalServiceEndpointWaitSec: 2,
+			CriticalServiceEndpointPollSec: 1,
+			CriticalServiceEndpoints:       []string{"monitoring/victoria-metrics-single-server"},
+		},
+		State: config.State{
+			Dir:            t.TempDir(),
+			ReportsDir:     filepath.Join(t.TempDir(), "reports"),
+			RunHistoryPath: filepath.Join(t.TempDir(), "runs.json"),
+		},
+	}
+	orch := &Orchestrator{
+		cfg:    cfg,
+		runner: &execx.Runner{},
+		store:  state.New(cfg.State.RunHistoryPath),
+		log:    log.New(io.Discard, "", 0),
+	}
+
+	endpointChecks := 0
+	dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		joined := name + " " + strings.Join(args, " ")
+		if strings.Contains(joined, "get endpoints victoria-metrics-single-server") {
+			endpointChecks++
+			if endpointChecks == 1 {
+				return "", nil
+			}
+			return "10.42.0.10\n", nil
+		}
+		if strings.Contains(joined, "scale deployment victoria-metrics-single-server") {
+			return "", errors.New(`Error from server (NotFound): deployments.apps "victoria-metrics-single-server" not found`)
+		}
+		if strings.Contains(joined, "scale statefulset victoria-metrics-single-server") {
+			return "", nil
+		}
+		if strings.Contains(joined, "rollout status statefulset/victoria-metrics-single-server") {
+			return "statefulset rolled out", nil
+		}
+		return "", nil
+	}
+	orch.runOverride = dispatch
+	orch.runSensitiveOverride = dispatch
+
+	if err := orch.waitForCriticalServiceEndpoints(context.Background()); err != nil {
+		t.Fatalf("waitForCriticalServiceEndpoints failed: %v", err)
+	}
+	if endpointChecks < 2 {
+		t.Fatalf("expected repeated endpoint checks, got %d", endpointChecks)
+	}
+}

internal/cluster/orchestrator_critical_endpoints.go

@@ -2,12 +2,22 @@ package cluster
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"sort"
 	"strings"
 	"time"
 )
 
+type endpointResource struct {
+	Subsets []struct {
+		Addresses []struct {
+			IP string `json:"ip"`
+		} `json:"addresses"`
+	} `json:"subsets"`
+}
+
 // waitForCriticalServiceEndpoints runs one orchestration or CLI step.
 // Signature: (o *Orchestrator) waitForCriticalServiceEndpoints(ctx context.Context) error.
 // Why: some externally-healthy services (like Grafana) still require backend
@@ -25,8 +35,10 @@ func (o *Orchestrator) waitForCriticalServiceEndpoints(ctx context.Context) erro
 	lastFailure := "unknown"
 	lastLogged := time.Time{}
 	lastHealAttempt := time.Time{}
+	lastRecycleAttempt := time.Time{}
 
 	for {
+		o.maybeAutoRecycleStuckPods(ctx, &lastRecycleAttempt)
 		ready, detail, failedNamespace, failedService, err := o.criticalServiceEndpointsReady(ctx)
 		if err != nil {
 			lastFailure = err.Error()
@@ -44,6 +56,16 @@ func (o *Orchestrator) waitForCriticalServiceEndpoints(ctx context.Context) erro
 			lastHealAttempt = now
 			healed, healErr := o.maybeHealCriticalEndpointBackends(ctx, failedNamespace, failedService)
 			if healErr != nil {
+				probeRepairs, probeErr := o.maybeRepairCriticalBackendStartupProbes(ctx, failedNamespace, failedService)
+				if probeErr != nil {
+					o.log.Printf("warning: critical endpoint startup-probe repair failed for %s/%s: %v", failedNamespace, failedService, probeErr)
+				}
+				if len(probeRepairs) > 0 {
+					sort.Strings(probeRepairs)
+					repairDetail := fmt.Sprintf("repaired critical endpoint startup probes: %s", joinLimited(probeRepairs, 8))
+					o.log.Printf("%s", repairDetail)
+					o.noteStartupAutoHeal(repairDetail)
+				}
 				o.log.Printf("warning: critical endpoint backend auto-heal failed for %s/%s: %v", failedNamespace, failedService, healErr)
 			}
 			if len(healed) > 0 {
@@ -120,7 +142,7 @@ func (o *Orchestrator) maybeHealCriticalEndpointBackends(ctx context.Context, na
 			}
 			return healed, fmt.Errorf("scale %s/%s/%s to 1: %w", namespace, kind, service, err)
 		}
-		if err := o.waitWorkloadReady(ctx, workload); err != nil {
+		if err := o.workloadRolloutReadyOnce(ctx, workload); err != nil {
 			if !isNotFoundErr(err) {
 				return healed, err
 			}
@@ -131,6 +153,83 @@ func (o *Orchestrator) maybeHealCriticalEndpointBackends(ctx context.Context, na
 	return healed, nil
 }
 
+// workloadRolloutReadyOnce runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) workloadRolloutReadyOnce(ctx context.Context, w startupWorkload) error.
+// Why: endpoint backend healing should issue one bounded readiness nudge and
+// let the outer endpoint wait enforce the configured convergence deadline.
+func (o *Orchestrator) workloadRolloutReadyOnce(ctx context.Context, w startupWorkload) error {
+	_, err := o.kubectl(
+		ctx,
+		45*time.Second,
+		"-n",
+		w.Namespace,
+		"rollout",
+		"status",
+		fmt.Sprintf("%s/%s", w.Kind, w.Name),
+		"--timeout=30s",
+	)
+	return err
+}
+
+// healUnreadyConfiguredServiceBackends runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) healUnreadyConfiguredServiceBackends(ctx context.Context) ([]string, error).
+// Why: the daemon's post-start loop should be able to repair configured service
+// backends, including Mailu, without rerunning the whole startup workflow.
+func (o *Orchestrator) healUnreadyConfiguredServiceBackends(ctx context.Context) ([]string, error) {
+	healed := []string{}
+	if len(o.cfg.Startup.CriticalServiceEndpoints) > 0 {
+		errs := []string{}
+		for _, entry := range o.cfg.Startup.CriticalServiceEndpoints {
+			namespace, service, err := parseCriticalServiceEndpoint(entry)
+			if err != nil {
+				errs = append(errs, err.Error())
+				continue
+			}
+			count, err := o.endpointAddressCount(ctx, namespace, service)
+			if err != nil && !isNotFoundErr(err) {
+				errs = append(errs, fmt.Sprintf("query endpoints %s/%s: %v", namespace, service, err))
+				continue
+			}
+			if err == nil && count > 0 {
+				continue
+			}
+			detail := fmt.Sprintf("%s/%s endpoints=0", namespace, service)
+			if err != nil {
+				detail = fmt.Sprintf("%s/%s not found", namespace, service)
+			}
+			o.log.Printf("warning: configured critical endpoint unhealthy (%s); attempting backend repair", detail)
+			items, healErr := o.maybeHealCriticalEndpointBackends(ctx, namespace, service)
+			healed = append(healed, items...)
+			if healErr != nil {
+				probeItems, probeErr := o.maybeRepairCriticalBackendStartupProbes(ctx, namespace, service)
+				healed = append(healed, probeItems...)
+				if probeErr != nil {
+					errs = append(errs, fmt.Sprintf("heal critical endpoint %s/%s (%s): %v; startup-probe repair: %v", namespace, service, detail, healErr, probeErr))
+					continue
+				}
+				if o.cfg.Startup.AutoRecycleStuckPods {
+					o.bestEffort("recycle stuck pods after configured service backend repair failure", func() error {
+						return o.recycleStuckControllerPods(ctx)
+					})
+				}
+				if len(probeItems) == 0 {
+					errs = append(errs, fmt.Sprintf("heal critical endpoint %s/%s (%s): %v", namespace, service, detail, healErr))
+				}
+			}
+		}
+		if len(errs) > 0 {
+			return healed, errors.New(strings.Join(errs, "; "))
+		}
+	}
+
+	tcpHealed, err := o.healFailedTCPServiceBackends(ctx)
+	healed = append(healed, tcpHealed...)
+	if err != nil {
+		return healed, err
+	}
+	return healed, nil
+}
+
 // endpointAddressCount runs one orchestration or CLI step.
 // Signature: (o *Orchestrator) endpointAddressCount(ctx context.Context, namespace string, service string) (int, error).
 // Why: endpoint address counts provide an objective service-backend readiness
@@ -145,12 +244,43 @@ func (o *Orchestrator) endpointAddressCount(ctx context.Context, namespace strin
 		"endpoints",
 		service,
 		"-o",
-		`jsonpath={range .subsets[*].addresses[*]}{.ip}{"\n"}{end}`,
+		"json",
 	)
 	if err != nil {
 		return 0, err
 	}
-	return len(lines(out)), nil
+	cleaned := stripKubectlWarnings(out)
+	if strings.HasPrefix(cleaned, "{") {
+		var endpoint endpointResource
+		if err := json.Unmarshal([]byte(cleaned), &endpoint); err != nil {
+			return 0, fmt.Errorf("decode endpoints %s/%s: %w", namespace, service, err)
+		}
+		count := 0
+		for _, subset := range endpoint.Subsets {
+			for _, address := range subset.Addresses {
+				if strings.TrimSpace(address.IP) != "" {
+					count++
+				}
+			}
+		}
+		return count, nil
+	}
+	return len(lines(cleaned)), nil
+}
+
+// stripKubectlWarnings runs one orchestration or CLI step.
+// Signature: stripKubectlWarnings(out string) string.
+// Why: kubectl writes deprecation warnings to stderr; command execution combines
+// streams, so readiness parsers must not mistake warning lines for health data.
+func stripKubectlWarnings(out string) string {
+	kept := []string{}
+	for _, line := range strings.Split(out, "\n") {
+		if strings.HasPrefix(strings.TrimSpace(line), "Warning:") {
+			continue
+		}
+		kept = append(kept, line)
+	}
+	return strings.TrimSpace(strings.Join(kept, "\n"))
 }
 
 // parseCriticalServiceEndpoint runs one orchestration or CLI step.

internal/cluster/orchestrator_critical_probe_repair.go

@@ -0,0 +1,144 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+)
+
+type criticalBackendWorkload struct {
+	Spec struct {
+		Template struct {
+			Spec struct {
+				Containers []criticalBackendContainer `json:"containers"`
+			} `json:"spec"`
+		} `json:"template"`
+	} `json:"spec"`
+}
+
+type criticalBackendContainer struct {
+	Name          string         `json:"name"`
+	LivenessProbe map[string]any `json:"livenessProbe"`
+	StartupProbe  map[string]any `json:"startupProbe"`
+}
+
+// maybeRepairCriticalBackendStartupProbes runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) maybeRepairCriticalBackendStartupProbes(ctx context.Context, namespace string, service string) ([]string, error).
+// Why: critical backends that are slow after outage recovery can be killed by
+// liveness probes before they ever become ready; adding a startup probe lets
+// Kubernetes distinguish slow boot from dead process without app-specific code.
+func (o *Orchestrator) maybeRepairCriticalBackendStartupProbes(ctx context.Context, namespace string, service string) ([]string, error) {
+	if !o.cfg.Startup.CriticalServiceStartupProbeRepair {
+		return nil, nil
+	}
+	namespace = strings.TrimSpace(namespace)
+	service = strings.TrimSpace(service)
+	if namespace == "" || service == "" {
+		return nil, nil
+	}
+	repaired := []string{}
+	errs := []string{}
+	for _, kind := range []string{"deployment", "statefulset"} {
+		items, err := o.repairWorkloadStartupProbes(ctx, namespace, kind, service)
+		if err != nil {
+			if isNotFoundErr(err) {
+				continue
+			}
+			errs = append(errs, err.Error())
+			continue
+		}
+		repaired = append(repaired, items...)
+	}
+	if len(errs) > 0 {
+		return repaired, errors.New(strings.Join(errs, "; "))
+	}
+	return repaired, nil
+}
+
+// repairWorkloadStartupProbes runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) repairWorkloadStartupProbes(ctx context.Context, namespace string, kind string, name string) ([]string, error).
+// Why: the startup-probe patch is deliberately limited to an existing
+// deployment/statefulset with a liveness probe and no startup probe.
+func (o *Orchestrator) repairWorkloadStartupProbes(ctx context.Context, namespace string, kind string, name string) ([]string, error) {
+	out, err := o.kubectl(ctx, 20*time.Second, "-n", namespace, "get", kind, name, "-o", "json")
+	if err != nil {
+		if strings.TrimSpace(out) != "" {
+			return nil, fmt.Errorf("%w: %s", err, strings.TrimSpace(out))
+		}
+		return nil, err
+	}
+	var workload criticalBackendWorkload
+	if err := json.Unmarshal([]byte(out), &workload); err != nil {
+		return nil, fmt.Errorf("decode %s/%s/%s for startup probe repair: %w", namespace, kind, name, err)
+	}
+
+	containers := []map[string]any{}
+	threshold := o.cfg.Startup.CriticalServiceStartupProbeThreshold
+	if threshold <= 0 {
+		threshold = 30
+	}
+	for _, container := range workload.Spec.Template.Spec.Containers {
+		containerName := strings.TrimSpace(container.Name)
+		if containerName == "" || len(container.LivenessProbe) == 0 || len(container.StartupProbe) > 0 {
+			continue
+		}
+		startupProbe := cloneMap(container.LivenessProbe)
+		startupProbe["failureThreshold"] = threshold
+		if _, ok := startupProbe["successThreshold"]; !ok {
+			startupProbe["successThreshold"] = 1
+		}
+		containers = append(containers, map[string]any{
+			"name":         containerName,
+			"startupProbe": startupProbe,
+		})
+	}
+	if len(containers) == 0 {
+		return nil, nil
+	}
+
+	now := time.Now().UTC().Format(time.RFC3339)
+	patch := map[string]any{
+		"spec": map[string]any{
+			"template": map[string]any{
+				"metadata": map[string]any{
+					"annotations": map[string]string{
+						"ananke.bstein.dev/startup-probe-repair-at":     now,
+						"ananke.bstein.dev/startup-probe-repair-reason": "critical-service-backend-unready",
+					},
+				},
+				"spec": map[string]any{
+					"containers": containers,
+				},
+			},
+		},
+	}
+	rawPatch, err := json.Marshal(patch)
+	if err != nil {
+		return nil, fmt.Errorf("encode startup probe repair patch: %w", err)
+	}
+	if _, err := o.kubectl(ctx, 25*time.Second, "-n", namespace, "patch", kind, name, "--type=strategic", "-p", string(rawPatch)); err != nil {
+		return nil, fmt.Errorf("patch startup probes on %s/%s/%s: %w", namespace, kind, name, err)
+	}
+	o.log.Printf("repaired startup probes on critical backend %s/%s/%s containers=%d", namespace, kind, name, len(containers))
+	o.noteStartupAutoHeal(fmt.Sprintf("repaired startup probes on %s/%s/%s", namespace, kind, name))
+	return []string{namespace + "/" + kind + "/" + name}, nil
+}
+
+// cloneMap runs one orchestration or CLI step.
+// Signature: cloneMap(in map[string]any) map[string]any.
+// Why: Kubernetes probe objects are patch payloads; cloning avoids mutating the
+// decoded workload fixture while changing the generated startup probe threshold.
+func cloneMap(in map[string]any) map[string]any {
+	out := map[string]any{}
+	for key, value := range in {
+		if nested, ok := value.(map[string]any); ok {
+			out[key] = cloneMap(nested)
+			continue
+		}
+		out[key] = value
+	}
+	return out
+}

internal/cluster/orchestrator_critical_vault.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"time"
 )
@@ -162,24 +161,38 @@ func (o *Orchestrator) waitWorkloadReady(ctx context.Context, w startupWorkload)
 		return o.waitVaultReady(ctx, w)
 	}
 
-	timeout := "240s"
+	deadline := time.Now().Add(7 * time.Minute)
-	if w.Kind == "statefulset" {
+	var lastErr error
-		timeout = "360s"
+	for time.Now().Before(deadline) {
+		o.bestEffort("recycle stuck critical workload pods", func() error { return o.recycleStuckControllerPods(ctx) })
+		_, err := o.kubectl(
+			ctx,
+			45*time.Second,
+			"-n",
+			w.Namespace,
+			"rollout",
+			"status",
+			fmt.Sprintf("%s/%s", w.Kind, w.Name),
+			"--timeout=30s",
+		)
+		if err == nil {
+			return nil
+		}
+		if isNotFoundErr(err) {
+			return err
+		}
+		lastErr = err
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(5 * time.Second):
+		}
 	}
-	_, err := o.kubectl(
+	if lastErr != nil {
-		ctx,
+		return fmt.Errorf("wait ready %s/%s/%s: %w", w.Namespace, w.Kind, w.Name, lastErr)
-		7*time.Minute,
-		"-n",
-		w.Namespace,
-		"rollout",
-		"status",
-		fmt.Sprintf("%s/%s", w.Kind, w.Name),
-		"--timeout="+timeout,
-	)
-	if err != nil {
-		return fmt.Errorf("wait ready %s/%s/%s: %w", w.Namespace, w.Kind, w.Name, err)
 	}
-	return nil
+	return fmt.Errorf("wait ready %s/%s/%s: timeout", w.Namespace, w.Kind, w.Name)
 }
 
 // waitVaultReady runs one orchestration or CLI step.
@@ -227,6 +240,31 @@ func (o *Orchestrator) waitVaultReady(ctx context.Context, w startupWorkload) er
 	return fmt.Errorf("wait ready %s/%s/%s: timeout", w.Namespace, w.Kind, w.Name)
 }
 
+// ensureVaultUnsealedWhenRunnable runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) ensureVaultUnsealedWhenRunnable(ctx context.Context) (bool, string, error).
+// Why: lets startup defer vault unseal until the pod is actually runnable, while
+// keeping the direct unseal helper strict for explicit recovery paths and tests.
+func (o *Orchestrator) ensureVaultUnsealedWhenRunnable(ctx context.Context) (bool, string, error) {
+	if o.runner.DryRun {
+		return false, "", nil
+	}
+
+	phase, err := o.kubectl(ctx, 15*time.Second, "-n", "vault", "get", "pod", "vault-0", "-o", "jsonpath={.status.phase}")
+	if err != nil {
+		if isNotFoundErr(err) {
+			return true, "vault-0 pod is not present yet; deferring unseal until critical workload recovery", nil
+		}
+		return false, "", fmt.Errorf("vault pod phase check failed: %w", err)
+	}
+
+	trimmedPhase := strings.TrimSpace(phase)
+	if trimmedPhase != "Running" {
+		return true, fmt.Sprintf("vault-0 pod phase is %q; deferring unseal until critical workload recovery", trimmedPhase), nil
+	}
+
+	return false, "", o.ensureVaultUnsealed(ctx)
+}
+
 // ensureVaultUnsealed runs one orchestration or CLI step.
 // Signature: (o *Orchestrator) ensureVaultUnsealed(ctx context.Context) error.
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
@@ -283,28 +321,6 @@ func (o *Orchestrator) ensureVaultUnsealed(ctx context.Context) error {
 	return fmt.Errorf("vault remained sealed after 5 auto-unseal attempts")
 }
 
-// vaultSealed runs one orchestration or CLI step.
-// Signature: (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error).
-// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
-func (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error) {
-	out, err := o.kubectl(
-		ctx,
-		25*time.Second,
-		"-n", "vault",
-		"exec", "vault-0", "--",
-		"sh", "-lc",
-		"VAULT_ADDR=http://127.0.0.1:8200 vault status -format=json 2>/dev/null || true",
-	)
-	if err != nil {
-		return false, fmt.Errorf("vault status check failed: %w", err)
-	}
-	sealed, err := parseVaultSealed(out)
-	if err != nil {
-		return false, fmt.Errorf("parse vault status: %w", err)
-	}
-	return sealed, nil
-}
-
 // parseVaultSealed runs one orchestration or CLI step.
 // Signature: parseVaultSealed(raw string) (bool, error).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
@@ -429,43 +445,3 @@ func (o *Orchestrator) readVaultUnsealKeyFile() (string, error) {
 	}
 	return key, nil
 }
-
-// workloadReady runs one orchestration or CLI step.
-// Signature: (o *Orchestrator) workloadReady(ctx context.Context, w startupWorkload) (bool, error).
-// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
-func (o *Orchestrator) workloadReady(ctx context.Context, w startupWorkload) (bool, error) {
-	out, err := o.kubectl(
-		ctx,
-		20*time.Second,
-		"-n",
-		w.Namespace,
-		"get",
-		w.Kind,
-		w.Name,
-		"-o",
-		"jsonpath={.status.readyReplicas}",
-	)
-	if err != nil {
-		return false, err
-	}
-	raw := strings.TrimSpace(out)
-	if raw == "" || raw == "<no value>" {
-		return false, nil
-	}
-	n, err := strconv.Atoi(raw)
-	if err != nil {
-		return false, fmt.Errorf("parse readyReplicas %q: %w", raw, err)
-	}
-	return n >= 1, nil
-}
-
-// isNotFoundErr runs one orchestration or CLI step.
-// Signature: isNotFoundErr(err error) bool.
-// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
-func isNotFoundErr(err error) bool {
-	if err == nil {
-		return false
-	}
-	msg := strings.ToLower(err.Error())
-	return strings.Contains(msg, "not found") || strings.Contains(msg, "(notfound)")
-}

internal/cluster/orchestrator_drain.go

@@ -3,6 +3,7 @@ package cluster
 import (
 	"context"
 	"fmt"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -145,6 +146,140 @@ func (o *Orchestrator) uncordonWorkers(ctx context.Context, workers []string) er
 	return nil
 }
 
+// ensureLonghornEncryptedHostPrereqs runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) ensureLonghornEncryptedHostPrereqs(ctx context.Context, workers []string) ([]string, error).
+// Why: encrypted Longhorn PVCs fail at kubelet mount time when a storage host
+// lacks host cryptsetup; startup must quarantine those nodes before workloads
+// are scheduled there.
+func (o *Orchestrator) ensureLonghornEncryptedHostPrereqs(ctx context.Context, workers []string) ([]string, error) {
+	longhornHosts, err := o.longhornHostNodes(ctx)
+	if err != nil {
+		return workers, err
+	}
+	if len(longhornHosts) == 0 {
+		return workers, nil
+	}
+
+	ignored := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
+	exempt := makeStringSet(o.cfg.Startup.LonghornCryptsetupExemptNodes)
+	unsafe := map[string]struct{}{}
+	var errs []string
+	nodes := make([]string, 0, len(longhornHosts))
+	for node := range longhornHosts {
+		nodes = append(nodes, node)
+	}
+	sort.Strings(nodes)
+	for _, node := range nodes {
+		if _, skip := ignored[node]; skip {
+			continue
+		}
+		if _, skip := exempt[node]; skip {
+			o.log.Printf("skip cryptsetup preflight on longhorn host %s: configured exemption", node)
+			continue
+		}
+		if !o.sshManaged(node) {
+			o.log.Printf("warning: keeping longhorn host %s cordoned because encrypted-volume prerequisites cannot be verified without SSH management", node)
+			if cordonErr := o.cordonNodeForMissingCryptsetup(ctx, node); cordonErr != nil {
+				errs = append(errs, fmt.Sprintf("%s cordon after unverifiable cryptsetup prerequisite: %v", node, cordonErr))
+			}
+			unsafe[node] = struct{}{}
+			continue
+		}
+		if checkErr := o.ensureHostCryptsetup(ctx, node); checkErr != nil {
+			o.log.Printf("warning: keeping longhorn host %s cordoned after cryptsetup prerequisite check failed: %v", node, checkErr)
+			if cordonErr := o.cordonNodeForMissingCryptsetup(ctx, node); cordonErr != nil {
+				errs = append(errs, fmt.Sprintf("%s cordon after cryptsetup prerequisite failure: %v", node, cordonErr))
+			}
+			unsafe[node] = struct{}{}
+			continue
+		}
+	}
+
+	guarded := make([]string, 0, len(workers))
+	for _, worker := range workers {
+		node := strings.TrimSpace(worker)
+		if node == "" {
+			continue
+		}
+		if _, blocked := unsafe[node]; blocked {
+			continue
+		}
+		guarded = append(guarded, node)
+	}
+	if len(errs) > 0 {
+		return guarded, fmt.Errorf("%s", strings.Join(errs, "; "))
+	}
+	return guarded, nil
+}
+
+// uncordonLonghornCryptsetupExemptNodes runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) uncordonLonghornCryptsetupExemptNodes(ctx context.Context) error.
+// Why: special Longhorn hosts that only run non-encrypted local workloads, such
+// as Veles/Oceanus, must recover from stale cordons without weakening the
+// encrypted-volume guard for normal storage workers.
+func (o *Orchestrator) uncordonLonghornCryptsetupExemptNodes(ctx context.Context) error {
+	exempt := makeStringSet(o.cfg.Startup.LonghornCryptsetupExemptNodes)
+	if len(exempt) == 0 {
+		return nil
+	}
+	longhornHosts, err := o.longhornHostNodes(ctx)
+	if err != nil {
+		return err
+	}
+	ignored := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
+	nodes := make([]string, 0, len(exempt))
+	for node := range exempt {
+		if _, skip := ignored[node]; skip {
+			continue
+		}
+		if _, ok := longhornHosts[node]; ok {
+			nodes = append(nodes, node)
+		}
+	}
+	sort.Strings(nodes)
+	if len(nodes) == 0 {
+		return nil
+	}
+	o.log.Printf("uncordon longhorn cryptsetup-exempt hosts: %s", strings.Join(nodes, ","))
+	o.noteStartupAutoHeal(fmt.Sprintf("uncordoned longhorn cryptsetup-exempt hosts: %s", strings.Join(nodes, ",")))
+	return o.uncordonWorkers(ctx, nodes)
+}
+
+// longhornHostNodes runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) longhornHostNodes(ctx context.Context) (map[string]struct{}, error).
+// Why: the live node label captures storage hosts that may not be in Ananke's
+// static worker list, so startup quarantine decisions should follow the
+// cluster's actual scheduling surface.
+func (o *Orchestrator) longhornHostNodes(ctx context.Context) (map[string]struct{}, error) {
+	out, err := o.kubectl(ctx, 20*time.Second,
+		"get", "nodes",
+		"-l", "longhorn-host=true",
+		"-o", "jsonpath={range .items[*]}{.metadata.name}{'\\n'}{end}",
+	)
+	if err != nil {
+		return nil, fmt.Errorf("query longhorn host nodes: %w", err)
+	}
+	nodes := map[string]struct{}{}
+	for _, line := range lines(out) {
+		node := strings.TrimSpace(line)
+		if node != "" {
+			nodes[node] = struct{}{}
+		}
+	}
+	if len(nodes) > 0 {
+		return nodes, nil
+	}
+	for node, labels := range o.cfg.Startup.RequiredNodeLabels {
+		if strings.EqualFold(strings.TrimSpace(labels["longhorn-host"]), "true") {
+			name := strings.TrimSpace(node)
+			if name != "" {
+				nodes[name] = struct{}{}
+			}
+		}
+	}
+	return nodes, nil
+}
+
 // stopWorkers runs one orchestration or CLI step.
 // Signature: (o *Orchestrator) stopWorkers(ctx context.Context, workers []string).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

internal/cluster/orchestrator_fluxhealth.go

@@ -143,6 +143,8 @@ func (o *Orchestrator) fluxHealthReady(ctx context.Context) (bool, string, error
 		return false, "", fmt.Errorf("decode flux kustomizations: %w", err)
 	}
 	ignored := makeStringSet(o.cfg.Startup.IgnoreFluxKustomizations)
+	required := o.startupRequiredFluxKustomizations()
+	requiredSeen := map[string]struct{}{}
 	notReady := []string{}
 	for _, ks := range list.Items {
 		ns := strings.TrimSpace(ks.Metadata.Namespace)
@@ -154,6 +156,12 @@ func (o *Orchestrator) fluxHealthReady(ctx context.Context) (bool, string, error
 		if ks.Spec.Suspend {
 			continue
 		}
+		if len(required) > 0 {
+			if _, ok := required[full]; !ok {
+				continue
+			}
+			requiredSeen[full] = struct{}{}
+		}
 		if _, ok := ignored[full]; ok {
 			continue
 		}
@@ -173,10 +181,25 @@ func (o *Orchestrator) fluxHealthReady(ctx context.Context) (bool, string, error
 		}
 		notReady = append(notReady, fmt.Sprintf("%s(%s)", full, reason))
 	}
+	if len(required) > 0 {
+		missing := []string{}
+		for full := range required {
+			if _, ok := requiredSeen[full]; !ok {
+				missing = append(missing, full+"(missing)")
+			}
+		}
+		if len(missing) > 0 {
+			sort.Strings(missing)
+			notReady = append(notReady, missing...)
+		}
+	}
 	if len(notReady) > 0 {
 		sort.Strings(notReady)
 		return false, "not ready: " + joinLimited(notReady, 6), nil
 	}
+	if len(required) > 0 {
+		return true, fmt.Sprintf("required kustomizations ready=%d", len(requiredSeen)), nil
+	}
 	return true, fmt.Sprintf("all kustomizations ready=%d", len(list.Items)), nil
 }
 

internal/cluster/orchestrator_hardening_test.go

@@ -0,0 +1,455 @@
+package cluster
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"log"
+	"net"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret runs one orchestration or CLI step.
+// Signature: TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret(t *testing.T).
+// Why: host repair must recover from password-required sudo while keeping the
+// sudo password out of command arguments and logs.
+func TestHostPrivilegeUsesVaultBackedSudoWithoutLeakingSecret(t *testing.T) {
+	secret := "correct horse battery staple"
+	var logs bytes.Buffer
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{
+			HostSudoSecretNamespace:    "ops",
+			HostSudoSecretNameTemplate: "sudo-{node}",
+			HostSudoSecretPasswordKey:  "password",
+		},
+	}, nil)
+	orch.log = log.New(&logs, "", 0)
+
+	orch.SetCommandOverrides(func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		joined := strings.Join(args, " ")
+		switch {
+		case name == "ssh" && strings.Contains(joined, "sudo -n /usr/bin/systemctl --version"):
+			return "sudo: a password is required", errors.New("exit status 1")
+		case name == "kubectl" && strings.Contains(joined, "-n ops get secret sudo-titan-05 -o json"):
+			return `{"data":{"password":"` + base64.StdEncoding.EncodeToString([]byte(secret)) + `"}}`, nil
+		default:
+			return "", nil
+		}
+	}, nil)
+
+	inputSeen := ""
+	commandSeen := ""
+	orch.SetSSHInputOverride(func(_ context.Context, _ time.Duration, node string, command string, input string) (string, error) {
+		if node != "titan-05" {
+			t.Fatalf("unexpected node %q", node)
+		}
+		commandSeen = command
+		inputSeen = input
+		return "systemd 255", nil
+	})
+
+	if err := orch.preflightHostPrivilege(context.Background(), "titan-05"); err != nil {
+		t.Fatalf("preflightHostPrivilege failed: %v", err)
+	}
+	if !strings.Contains(commandSeen, "sudo -S -p '' /usr/bin/systemctl --version") {
+		t.Fatalf("expected sudo stdin command, got %q", commandSeen)
+	}
+	if inputSeen != secret+"\n" {
+		t.Fatalf("expected sudo password on stdin only")
+	}
+	if strings.Contains(logs.String(), secret) || strings.Contains(commandSeen, secret) {
+		t.Fatalf("sudo password leaked through logs or command")
+	}
+}
+
+// TestHostPrivilegeMissingSecretReportsUnavailable runs one orchestration or CLI step.
+// Signature: TestHostPrivilegeMissingSecretReportsUnavailable(t *testing.T).
+// Why: missing Vault/Kubernetes sudo material should be a classified blocker,
+// not an opaque loop of failed host commands.
+func TestHostPrivilegeMissingSecretReportsUnavailable(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{
+			match: matchContains("ssh", "sudo -n /usr/bin/systemctl --version"),
+			out:   "sudo: a password is required",
+			err:   errors.New("exit status 1"),
+		},
+	})
+	err := orch.preflightHostPrivilege(context.Background(), "titan-05")
+	if err == nil || !strings.Contains(err.Error(), "host-privilege-unavailable") {
+		t.Fatalf("expected host-privilege-unavailable, got %v", err)
+	}
+}
+
+// TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel runs one orchestration or CLI step.
+// Signature: TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel(t *testing.T).
+// Why: a Kubernetes Ready node with a Longhorn Node CR should regain missing
+// manager selector labels instead of leaving PVC workloads blocked.
+func TestReconcileLonghornKubernetesReadinessRestoresManagerSelectorLabel(t *testing.T) {
+	labeled := false
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{
+			match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"),
+			out:   `{"items":[{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`,
+		},
+		{
+			match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager", "-o", "json"),
+			out:   `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true"}}}}`,
+		},
+		{
+			match: matchContains("kubectl", "get", "nodes", "-o", "json"),
+			out:   `{"items":[{"metadata":{"name":"titan-22","labels":{"kubernetes.io/hostname":"titan-22"}},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`,
+		},
+		{
+			match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json", "-l", "longhorn-host=true"),
+			out:   `{"items":[]}`,
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "label", "node", "titan-22", "--overwrite", "longhorn-host=true")(name, args) {
+					return false
+				}
+				labeled = true
+				return true
+			},
+		},
+	})
+
+	repaired, err := orch.reconcileLonghornKubernetesReadiness(context.Background())
+	if err != nil {
+		t.Fatalf("reconcileLonghornKubernetesReadiness failed: %v", err)
+	}
+	if repaired != 1 || !labeled {
+		t.Fatalf("expected one Longhorn label repair, repaired=%d labeled=%v", repaired, labeled)
+	}
+}
+
+// TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter runs one orchestration or CLI step.
+// Signature: TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter(t *testing.T).
+// Why: stale RWO cleanup may force-delete sidecar-only owners but must not clear
+// an API object while a live application container still mounts the PVC.
+func TestStaleRWOPVCOwnerDecisionsDistinguishSidecarAndLiveWriter(t *testing.T) {
+	old := time.Now().Add(-30 * time.Minute).UTC().Format(time.RFC3339)
+	pvcJSON := `{"items":[{"metadata":{"namespace":"finance","name":"firefly-storage"},"spec":{"accessModes":["ReadWriteOnce"],"volumeName":"pvc-1"}}]}`
+	eventsJSON := `{"items":[{"involvedObject":{"kind":"Pod","namespace":"finance","name":"firefly-new"},"type":"Warning","reason":"FailedAttachVolume","message":"Multi-Attach error for volume pvc-1: Volume is already used by pod(s) firefly-old"}]}`
+
+	t.Run("sidecar-only force delete", func(t *testing.T) {
+		podsJSON := `{"items":[` +
+			staleRWOPodJSON("firefly-old", old, "titan-04", true, false) + `,` +
+			replacementRWOPodJSON("firefly-new", "titan-06") +
+			`]}`
+		orch := staleRWOOrchestrator(t, podsJSON, pvcJSON, eventsJSON)
+
+		var pods podList
+		if err := jsonUnmarshal(podsJSON, &pods); err != nil {
+			t.Fatalf("decode pods: %v", err)
+		}
+		decisions, err := orch.staleRWOPVCOwnerDecisions(context.Background(), pods, 5*time.Minute)
+		if err != nil {
+			t.Fatalf("staleRWOPVCOwnerDecisions failed: %v", err)
+		}
+		decision := decisions["finance/firefly-old"]
+		if !decision.ForceDelete || decision.Unsafe {
+			t.Fatalf("expected sidecar-only force delete decision, got %#v", decision)
+		}
+	})
+
+	t.Run("live writer unsafe", func(t *testing.T) {
+		podsJSON := `{"items":[` +
+			staleRWOPodJSON("firefly-old", old, "titan-04", true, true) + `,` +
+			replacementRWOPodJSON("firefly-new", "titan-06") +
+			`]}`
+		orch := staleRWOOrchestrator(t, podsJSON, pvcJSON, eventsJSON)
+
+		var pods podList
+		if err := jsonUnmarshal(podsJSON, &pods); err != nil {
+			t.Fatalf("decode pods: %v", err)
+		}
+		decisions, err := orch.staleRWOPVCOwnerDecisions(context.Background(), pods, 5*time.Minute)
+		if err != nil {
+			t.Fatalf("staleRWOPVCOwnerDecisions failed: %v", err)
+		}
+		decision := decisions["finance/firefly-old"]
+		if !decision.Unsafe || decision.ForceDelete {
+			t.Fatalf("expected live writer unsafe decision, got %#v", decision)
+		}
+	})
+}
+
+// TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS runs one orchestration or CLI step.
+// Signature: TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS(t *testing.T).
+// Why: DNS-caused image pulls should be reported as registry/DNS blockers rather
+// than recycled as if deleting the pod could fix name resolution.
+func TestImagePullDNSBlockerReasonsClassifiesDockerHubDNS(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{
+			match: matchContains("kubectl", "get", "events", "-A", "-o", "json"),
+			out:   `{"items":[{"involvedObject":{"kind":"Pod","namespace":"logging","name":"oauth2"},"type":"Warning","reason":"Failed","message":"Failed to pull image: lookup registry-1.docker.io: Try again"}]}`,
+		},
+	})
+	reasons, err := orch.imagePullDNSBlockerReasons(context.Background())
+	if err != nil {
+		t.Fatalf("imagePullDNSBlockerReasons failed: %v", err)
+	}
+	if got := reasons["logging/oauth2"]; got != "ImagePullDNSBlocker:registry-1.docker.io" {
+		t.Fatalf("unexpected image pull DNS reason %q", got)
+	}
+}
+
+// TestVaultSealedFallsBackToHTTPHealthWhenExecKilled runs one orchestration or CLI step.
+// Signature: TestVaultSealedFallsBackToHTTPHealthWhenExecKilled(t *testing.T).
+// Why: a transient killed kubectl exec should not block startup when Vault HTTP
+// health proves the pod is initialized and unsealed.
+func TestVaultSealedFallsBackToHTTPHealthWhenExecKilled(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{
+			match: matchContains("kubectl", "-n", "vault", "exec", "vault-0"),
+			err:   errors.New("signal: killed"),
+		},
+		{
+			match: matchContains("kubectl", "-n", "vault", "get", "--raw", "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health"),
+			out:   `{"initialized":true,"sealed":false,"standby":false}`,
+		},
+	})
+	sealed, err := orch.vaultSealed(context.Background())
+	if err != nil || sealed {
+		t.Fatalf("expected HTTP fallback to report unsealed, sealed=%v err=%v", sealed, err)
+	}
+}
+
+// TestTCPServiceChecklistReadyChecksMailProtocolBanner runs one orchestration or CLI step.
+// Signature: TestTCPServiceChecklistReadyChecksMailProtocolBanner(t *testing.T).
+// Why: Mailu is not covered by HTTP ingress checks, so Ananke needs direct
+// protocol probes that validate a TCP greeting and command response.
+func TestTCPServiceChecklistReadyChecksMailProtocolBanner(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen: %v", err)
+	}
+	defer listener.Close()
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		conn, acceptErr := listener.Accept()
+		if acceptErr != nil {
+			return
+		}
+		defer conn.Close()
+		_, _ = conn.Write([]byte("220 mail.bstein.dev ESMTP ready\r\n"))
+		_, _ = bufio.NewReader(conn).ReadString('\n')
+		_, _ = conn.Write([]byte("221 2.0.0 Bye\r\n"))
+	}()
+
+	host, port, err := net.SplitHostPort(listener.Addr().String())
+	if err != nil {
+		t.Fatalf("split listener address: %v", err)
+	}
+	check := config.TCPServiceChecklistCheck{
+		Name:           "mail-smtp",
+		Host:           host,
+		Port:           atoiForTest(t, port),
+		Send:           "QUIT\r\n",
+		ExpectContains: "ESMTP",
+		TimeoutSeconds: 2,
+	}
+	orch := buildOrchestratorWithStubs(t, config.Config{}, nil)
+	ok, detail := orch.tcpServiceCheckReady(context.Background(), check)
+	if !ok {
+		t.Fatalf("expected TCP mail check to pass, detail=%s", detail)
+	}
+	<-done
+}
+
+// TestServiceChecklistReadyIncludesTCPFailures runs one orchestration or CLI step.
+// Signature: TestServiceChecklistReadyIncludesTCPFailures(t *testing.T).
+// Why: startup should fail clearly when a configured mail protocol endpoint is
+// down even if there are no HTTP checks in that config.
+func TestServiceChecklistReadyIncludesTCPFailures(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{
+			TCPServiceChecklist: []config.TCPServiceChecklistCheck{
+				{
+					Name:           "mail-smtp",
+					Host:           "127.0.0.1",
+					Port:           1,
+					ExpectContains: "ESMTP",
+					TimeoutSeconds: 1,
+				},
+			},
+		},
+	}, nil)
+	ok, detail := orch.serviceChecklistReady(context.Background())
+	if ok {
+		t.Fatalf("expected TCP-only checklist to fail")
+	}
+	if !strings.Contains(detail, "tcp mail-smtp") {
+		t.Fatalf("expected TCP failure detail, got %q", detail)
+	}
+}
+
+// TestEndpointAddressCountIgnoresKubectlWarnings runs one orchestration or CLI step.
+// Signature: TestEndpointAddressCountIgnoresKubectlWarnings(t *testing.T).
+// Why: Kubernetes endpoint deprecation warnings must not make an empty service
+// look ready by contributing a fake output line.
+func TestEndpointAddressCountIgnoresKubectlWarnings(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{
+			match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "endpoints", "mailu-admin", "-o", "json"),
+			out:   "Warning: v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice\n{\"subsets\":[]}",
+		},
+	})
+	count, err := orch.endpointAddressCount(context.Background(), "mailu-mailserver", "mailu-admin")
+	if err != nil {
+		t.Fatalf("endpointAddressCount failed: %v", err)
+	}
+	if count != 0 {
+		t.Fatalf("expected empty endpoint count after warning strip, got %d", count)
+	}
+}
+
+// TestCriticalBackendStartupProbeRepairCopiesLivenessProbe runs one orchestration or CLI step.
+// Signature: TestCriticalBackendStartupProbeRepairCopiesLivenessProbe(t *testing.T).
+// Why: slow critical backends should get a bounded startup grace without
+// changing app-specific liveness semantics or requiring a Mailu special case.
+func TestCriticalBackendStartupProbeRepairCopiesLivenessProbe(t *testing.T) {
+	var patchPayload string
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{
+			CriticalServiceStartupProbeRepair:    true,
+			CriticalServiceStartupProbeThreshold: 24,
+		},
+	}, []commandStub{
+		{
+			match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "deployment", "mailu-admin", "-o", "json"),
+			out: `{"spec":{"template":{"spec":{"containers":[{"name":"admin","livenessProbe":{"httpGet":{"path":"/ping","port":"http"},"periodSeconds":10,"failureThreshold":3}},` +
+				`{"name":"sidecar"}]}}}}`,
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "mailu-mailserver", "patch", "deployment", "mailu-admin", "--type=strategic", "-p")(name, args) {
+					return false
+				}
+				for i, arg := range args {
+					if arg == "-p" && i+1 < len(args) {
+						patchPayload = args[i+1]
+					}
+				}
+				return true
+			},
+		},
+		{
+			match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "statefulset", "mailu-admin", "-o", "json"),
+			err:   errors.New("not found"),
+		},
+	})
+
+	repaired, err := orch.maybeRepairCriticalBackendStartupProbes(context.Background(), "mailu-mailserver", "mailu-admin")
+	if err != nil {
+		t.Fatalf("maybeRepairCriticalBackendStartupProbes failed: %v", err)
+	}
+	if len(repaired) != 1 || repaired[0] != "mailu-mailserver/deployment/mailu-admin" {
+		t.Fatalf("unexpected repaired list: %v", repaired)
+	}
+	if patchPayload == "" {
+		t.Fatalf("expected strategic merge patch payload")
+	}
+	if !strings.Contains(patchPayload, `"startupProbe"`) || !strings.Contains(patchPayload, `"failureThreshold":24`) || !strings.Contains(patchPayload, `"path":"/ping"`) {
+		t.Fatalf("unexpected startup-probe patch payload: %s", patchPayload)
+	}
+}
+
+// TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet runs one orchestration or CLI step.
+// Signature: TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet(t *testing.T).
+// Why: Mailu backends are Deployments; a kubectl NotFound message on the
+// StatefulSet probe path must not turn a successful/no-op Deployment check into
+// an auto-heal failure.
+func TestCriticalBackendStartupProbeRepairIgnoresMissingStatefulSet(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{
+			CriticalServiceStartupProbeRepair:    true,
+			CriticalServiceStartupProbeThreshold: 24,
+		},
+	}, []commandStub{
+		{
+			match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "deployment", "mailu-rspamd", "-o", "json"),
+			out:   `{"spec":{"template":{"spec":{"containers":[{"name":"rspamd","livenessProbe":{"httpGet":{"path":"/","port":"http"}},"startupProbe":{"httpGet":{"path":"/","port":"http"}}}]}}}}`,
+		},
+		{
+			match: matchContains("kubectl", "-n", "mailu-mailserver", "get", "statefulset", "mailu-rspamd", "-o", "json"),
+			out:   `Error from server (NotFound): statefulsets.apps "mailu-rspamd" not found`,
+			err:   errors.New("exit status 1"),
+		},
+	})
+	repaired, err := orch.maybeRepairCriticalBackendStartupProbes(context.Background(), "mailu-mailserver", "mailu-rspamd")
+	if err != nil {
+		t.Fatalf("expected missing statefulset to be ignored, got %v", err)
+	}
+	if len(repaired) != 0 {
+		t.Fatalf("expected no repair for existing startup probe, got %v", repaired)
+	}
+}
+
+// atoiForTest runs one orchestration or CLI step.
+// Signature: atoiForTest(t *testing.T, raw string) int.
+// Why: TCP listener fixtures need the kernel-selected port as a typed config
+// value without obscuring test failures.
+func atoiForTest(t *testing.T, raw string) int {
+	t.Helper()
+	port, err := strconv.Atoi(raw)
+	if err != nil {
+		t.Fatalf("parse port %q: %v", raw, err)
+	}
+	return port
+}
+
+// staleRWOOrchestrator runs one orchestration or CLI step.
+// Signature: staleRWOOrchestrator(t *testing.T, podsJSON string, pvcJSON string, eventsJSON string) *Orchestrator.
+// Why: stale RWO tests need a tiny fake API surface for pods, PVCs, and events
+// without depending on a live Kubernetes cluster.
+func staleRWOOrchestrator(t *testing.T, podsJSON string, pvcJSON string, eventsJSON string) *Orchestrator {
+	t.Helper()
+	return buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: eventsJSON},
+		{match: matchContains("kubectl", "get", "pvc", "-A", "-o", "json"), out: pvcJSON},
+		{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: podsJSON},
+	})
+}
+
+// staleRWOPodJSON runs one orchestration or CLI step.
+// Signature: staleRWOPodJSON(name string, deletedAt string, node string, sidecarRunning bool, appRunning bool) string.
+// Why: the sidecar-only and live-writer fixtures differ only in container
+// status, so one builder keeps the safety distinction obvious.
+func staleRWOPodJSON(name string, deletedAt string, node string, sidecarRunning bool, appRunning bool) string {
+	statuses := []string{}
+	if sidecarRunning {
+		statuses = append(statuses, `{"name":"vault-agent","state":{"running":{"startedAt":"`+deletedAt+`"}}}`)
+	}
+	if appRunning {
+		statuses = append(statuses, `{"name":"firefly","state":{"running":{"startedAt":"`+deletedAt+`"}}}`)
+	}
+	return `{"metadata":{"namespace":"finance","name":"` + name + `","creationTimestamp":"` + deletedAt + `","deletionTimestamp":"` + deletedAt + `","ownerReferences":[{"kind":"ReplicaSet","name":"firefly"}]},"spec":{"nodeName":"` + node + `","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"firefly-storage"}},{"name":"vault-secret"}],"containers":[{"name":"firefly","volumeMounts":[{"name":"data","mountPath":"/var/www/html/storage"}]},{"name":"vault-agent","volumeMounts":[{"name":"vault-secret","mountPath":"/vault/secrets"}]}]},"status":{"phase":"Running","containerStatuses":[` + strings.Join(statuses, ",") + `]}}`
+}
+
+// replacementRWOPodJSON runs one orchestration or CLI step.
+// Signature: replacementRWOPodJSON(name string, node string) string.
+// Why: stale-owner tests need a controller sibling blocked in Pending on a
+// different node while sharing the same RWO claim.
+func replacementRWOPodJSON(name string, node string) string {
+	return `{"metadata":{"namespace":"finance","name":"` + name + `","creationTimestamp":"` + time.Now().Add(-10*time.Minute).UTC().Format(time.RFC3339) + `","ownerReferences":[{"kind":"ReplicaSet","name":"firefly"}]},"spec":{"nodeName":"` + node + `","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"firefly-storage"}}],"containers":[{"name":"firefly","volumeMounts":[{"name":"data","mountPath":"/var/www/html/storage"}]}]},"status":{"phase":"Pending","containerStatuses":[{"name":"firefly","state":{"waiting":{"reason":"ContainerCreating"}}}]}}`
+}
+
+// jsonUnmarshal runs one orchestration or CLI step.
+// Signature: jsonUnmarshal(raw string, target any) error.
+// Why: local JSON fixtures should use the same decoder semantics as Kubernetes
+// command output parsing.
+func jsonUnmarshal(raw string, target any) error {
+	return json.Unmarshal([]byte(raw), target)
+}

internal/cluster/orchestrator_host_privilege.go

@@ -0,0 +1,331 @@
+package cluster
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+type hostPrivilegedAction string
+
+const (
+	hostActionSudoPreflight     hostPrivilegedAction = "sudo-preflight"
+	hostActionK3sAgentShow      hostPrivilegedAction = "k3s-agent-show"
+	hostActionK3sAgentIsActive  hostPrivilegedAction = "k3s-agent-is-active"
+	hostActionK3sAgentRestart   hostPrivilegedAction = "k3s-agent-restart-no-block"
+	hostActionHostReboot        hostPrivilegedAction = "host-reboot"
+	hostActionInstallCryptsetup hostPrivilegedAction = "install-cryptsetup-bin"
+	hostActionModprobeDMCrypt   hostPrivilegedAction = "modprobe-dm-crypt"
+	hostActionCrictlPods        hostPrivilegedAction = "crictl-pods"
+	hostActionCrictlPs          hostPrivilegedAction = "crictl-ps"
+	hostActionCrictlInspect     hostPrivilegedAction = "crictl-inspect"
+	hostActionCrictlStop        hostPrivilegedAction = "crictl-stop"
+)
+
+type hostPrivilegeError struct {
+	Class  string
+	Node   string
+	Action hostPrivilegedAction
+	Detail string
+}
+
+// Error runs one orchestration or CLI step.
+// Signature: (e hostPrivilegeError) Error() string.
+// Why: host repair failures need stable operator-facing classes without
+// embedding secret material or raw command input.
+func (e hostPrivilegeError) Error() string {
+	parts := []string{strings.TrimSpace(e.Class)}
+	if strings.TrimSpace(e.Node) != "" {
+		parts = append(parts, "node="+strings.TrimSpace(e.Node))
+	}
+	if e.Action != "" {
+		parts = append(parts, "action="+string(e.Action))
+	}
+	if strings.TrimSpace(e.Detail) != "" {
+		parts = append(parts, strings.TrimSpace(e.Detail))
+	}
+	return strings.Join(parts, " ")
+}
+
+type hostSudoSecret struct {
+	Data map[string]string `json:"data"`
+}
+
+// preflightHostPrivilege verifies that Ananke can run harmless sudo on a node.
+// Signature: (o *Orchestrator) preflightHostPrivilege(ctx context.Context, node string) error.
+// Why: host repair should fail with a clear privilege class before an outage
+// path needs a package install, k3s-agent restart, runtime inspection, or reboot.
+func (o *Orchestrator) preflightHostPrivilege(ctx context.Context, node string) error {
+	_, err := o.runHostPrivilegedAction(ctx, node, hostActionSudoPreflight, 12*time.Second)
+	return err
+}
+
+// runHostPrivilegedAction executes one allowlisted sudo action on a managed host.
+// Signature: (o *Orchestrator) runHostPrivilegedAction(ctx context.Context, node string, action hostPrivilegedAction, timeout time.Duration, args ...string) (string, error).
+// Why: recovery must not turn Vault-backed sudo into arbitrary remote root
+// execution; every privileged operation is classified, bounded, and auditable.
+func (o *Orchestrator) runHostPrivilegedAction(ctx context.Context, node string, action hostPrivilegedAction, timeout time.Duration, args ...string) (string, error) {
+	node = strings.TrimSpace(node)
+	if node == "" {
+		return "", hostPrivilegeError{Class: "host-privilege-unavailable", Action: action, Detail: "node is empty"}
+	}
+	if !o.sshManaged(node) {
+		return "", hostPrivilegeError{Class: "host-privilege-unavailable", Node: node, Action: action, Detail: "node is not SSH-managed"}
+	}
+	commandArgs, err := hostPrivilegedCommandArgs(action, args...)
+	if err != nil {
+		return "", err
+	}
+	if timeout <= 0 {
+		timeout = o.hostPrivilegedCommandTimeout()
+	}
+
+	passwordless := sudoRemoteCommand(false, commandArgs)
+	out, err := o.sshWithTimeout(ctx, node, passwordless, timeout)
+	if err == nil {
+		o.logHostPrivilegedAction(node, action, "passwordless-sudo")
+		return out, nil
+	}
+	if !sudoMayNeedPassword(out, err) {
+		return out, classifyHostPrivilegeFailure(node, action, err, out)
+	}
+
+	password, secretClass, secretErr := o.hostSudoPassword(ctx, node)
+	if secretErr != nil {
+		return "", hostPrivilegeError{
+			Class:  "host-privilege-unavailable",
+			Node:   node,
+			Action: action,
+			Detail: fmt.Sprintf("%s: %v", secretClass, secretErr),
+		}
+	}
+	passwordBacked := sudoRemoteCommand(true, commandArgs)
+	out, err = o.sshWithInput(ctx, node, passwordBacked, password+"\n", timeout)
+	if err != nil {
+		return out, classifyHostPrivilegeFailure(node, action, err, out)
+	}
+	o.logHostPrivilegedAction(node, action, "vault-backed-sudo")
+	return out, nil
+}
+
+// hostPrivilegedCommandTimeout runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) hostPrivilegedCommandTimeout() time.Duration.
+// Why: every privileged host command must have a bounded default timeout even
+// when a focused unit test builds a partial config.
+func (o *Orchestrator) hostPrivilegedCommandTimeout() time.Duration {
+	seconds := o.cfg.Startup.HostPrivilegedCommandTimeoutSec
+	if seconds <= 0 {
+		seconds = 90
+	}
+	return time.Duration(seconds) * time.Second
+}
+
+// logHostPrivilegedAction runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) logHostPrivilegedAction(node string, action hostPrivilegedAction, mode string).
+// Why: audit logs should record the action class and sudo mode while omitting
+// command stdin, passwords, and secret contents.
+func (o *Orchestrator) logHostPrivilegedAction(node string, action hostPrivilegedAction, mode string) {
+	o.log.Printf("host privileged action completed node=%s action=%s mode=%s", node, action, mode)
+}
+
+// hostPrivilegedCommandArgs runs one orchestration or CLI step.
+// Signature: hostPrivilegedCommandArgs(action hostPrivilegedAction, args ...string) ([]string, error).
+// Why: the Vault-backed sudo path must stay on a strict allowlist instead of
+// accepting arbitrary remote root commands.
+func hostPrivilegedCommandArgs(action hostPrivilegedAction, args ...string) ([]string, error) {
+	switch action {
+	case hostActionSudoPreflight:
+		return []string{"/usr/bin/systemctl", "--version"}, nil
+	case hostActionK3sAgentShow:
+		return []string{"systemctl", "show", "k3s-agent", "--property=ActiveState,SubState,Result,MainPID", "--no-pager"}, nil
+	case hostActionK3sAgentIsActive:
+		return []string{"systemctl", "is-active", "k3s-agent"}, nil
+	case hostActionK3sAgentRestart:
+		return []string{"systemctl", "--no-block", "restart", "k3s-agent"}, nil
+	case hostActionHostReboot:
+		return []string{"systemctl", "reboot"}, nil
+	case hostActionInstallCryptsetup:
+		return []string{
+			"env",
+			"DEBIAN_FRONTEND=noninteractive",
+			"sh",
+			"-lc",
+			"apt-get update && apt-get install -y --no-install-recommends cryptsetup-bin && (command -v cryptsetup >/dev/null 2>&1 || test -x /usr/sbin/cryptsetup || test -x /usr/bin/cryptsetup) && echo __ANANKE_CRYPTSETUP_INSTALLED__",
+		}, nil
+	case hostActionModprobeDMCrypt:
+		return []string{"modprobe", "dm_crypt"}, nil
+	case hostActionCrictlPods:
+		return []string{"crictl", "pods", "-o", "json"}, nil
+	case hostActionCrictlPs:
+		return []string{"crictl", "ps", "-a", "-o", "json"}, nil
+	case hostActionCrictlInspect:
+		if len(args) != 1 || strings.TrimSpace(args[0]) == "" || strings.ContainsAny(args[0], " \t\r\n;&|`$<>") {
+			return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "crictl inspect requires one safe container id"}
+		}
+		return []string{"crictl", "inspect", strings.TrimSpace(args[0])}, nil
+	case hostActionCrictlStop:
+		if len(args) != 1 || strings.TrimSpace(args[0]) == "" || strings.ContainsAny(args[0], " \t\r\n;&|`$<>") {
+			return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "crictl stop requires one safe container id"}
+		}
+		return []string{"crictl", "stop", strings.TrimSpace(args[0])}, nil
+	default:
+		return nil, hostPrivilegeError{Class: "host-command-not-allowed", Action: action, Detail: "action is not allowlisted"}
+	}
+}
+
+// sudoRemoteCommand runs one orchestration or CLI step.
+// Signature: sudoRemoteCommand(passwordBacked bool, commandArgs []string) string.
+// Why: passwordless and password-backed sudo share one command builder so
+// quoting and prompt suppression stay consistent.
+func sudoRemoteCommand(passwordBacked bool, commandArgs []string) string {
+	parts := []string{"sudo"}
+	if passwordBacked {
+		parts = append(parts, "-S", "-p", "")
+	} else {
+		parts = append(parts, "-n")
+	}
+	parts = append(parts, commandArgs...)
+	return shellJoin(parts)
+}
+
+// shellJoin runs one orchestration or CLI step.
+// Signature: shellJoin(parts []string) string.
+// Why: remote SSH commands need readable simple words and safe quoting for
+// shell-sensitive fragments such as package-manager scripts.
+func shellJoin(parts []string) string {
+	out := make([]string, 0, len(parts))
+	for _, part := range parts {
+		out = append(out, shellQuoteIfNeeded(part))
+	}
+	return strings.Join(out, " ")
+}
+
+// shellQuoteIfNeeded runs one orchestration or CLI step.
+// Signature: shellQuoteIfNeeded(part string) string.
+// Why: audit-friendly commands should avoid unnecessary quotes while still
+// protecting whitespace, quotes, and metacharacters.
+func shellQuoteIfNeeded(part string) string {
+	if part == "" {
+		return shellQuote(part)
+	}
+	for _, r := range part {
+		if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') {
+			continue
+		}
+		switch r {
+		case '_', '-', '.', '/', '=', ':':
+			continue
+		default:
+			return shellQuote(part)
+		}
+	}
+	return part
+}
+
+// sudoMayNeedPassword runs one orchestration or CLI step.
+// Signature: sudoMayNeedPassword(out string, err error) bool.
+// Why: Ananke should try Vault-backed sudo only for sudo privilege failures, not
+// for unrelated SSH or command errors.
+func sudoMayNeedPassword(out string, err error) bool {
+	full := strings.ToLower(strings.TrimSpace(out + " " + fmt.Sprint(err)))
+	needles := []string{
+		"a password is required",
+		"password is required",
+		"sudo: a terminal is required",
+		"no tty present",
+		"may not run sudo",
+		"sudo denied",
+		"is not in the sudoers file",
+	}
+	for _, needle := range needles {
+		if strings.Contains(full, needle) {
+			return true
+		}
+	}
+	return false
+}
+
+// classifyHostPrivilegeFailure runs one orchestration or CLI step.
+// Signature: classifyHostPrivilegeFailure(node string, action hostPrivilegedAction, err error, out string) error.
+// Why: host repair needs distinct timeout, auth, SSH, and command-failure
+// classes so status output tells the operator what kind of help is needed.
+func classifyHostPrivilegeFailure(node string, action hostPrivilegedAction, err error, out string) error {
+	full := strings.ToLower(strings.TrimSpace(out + " " + fmt.Sprint(err)))
+	class := "host-command-failed"
+	switch {
+	case strings.Contains(full, "context deadline exceeded"), strings.Contains(full, "timed out"), strings.Contains(full, "timeout"):
+		class = "host-command-timeout"
+	case strings.Contains(full, "authentication failure"), strings.Contains(full, "incorrect password"), strings.Contains(full, "try again"):
+		class = "host-privilege-auth-failed"
+	case strings.Contains(full, "permission denied"), strings.Contains(full, "publickey"):
+		class = "host-ssh-unavailable"
+	}
+	return hostPrivilegeError{Class: class, Node: node, Action: action, Detail: scrubHostPrivilegeDetail(out, err)}
+}
+
+// scrubHostPrivilegeDetail runs one orchestration or CLI step.
+// Signature: scrubHostPrivilegeDetail(out string, err error) string.
+// Why: host command details should be compact enough for annotations and avoid
+// accidental multi-line or control-character log noise.
+func scrubHostPrivilegeDetail(out string, err error) string {
+	detail := strings.TrimSpace(fmt.Sprint(err))
+	if trimmed := strings.TrimSpace(out); trimmed != "" {
+		if detail != "" {
+			detail += ": "
+		}
+		detail += trimmed
+	}
+	return sanitizeCordonAnnotationValue(detail)
+}
+
+// hostSudoPassword runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) hostSudoPassword(ctx context.Context, node string) (string, string, error).
+// Why: sudo material should come from a configured Vault-synced Kubernetes
+// Secret and never be logged or passed through ordinary command arguments.
+func (o *Orchestrator) hostSudoPassword(ctx context.Context, node string) (string, string, error) {
+	namespace := strings.TrimSpace(o.cfg.Startup.HostSudoSecretNamespace)
+	template := strings.TrimSpace(o.cfg.Startup.HostSudoSecretNameTemplate)
+	key := strings.TrimSpace(o.cfg.Startup.HostSudoSecretPasswordKey)
+	if key == "" {
+		key = "password"
+	}
+	if namespace == "" || template == "" {
+		return "", "secret-lookup-unconfigured", fmt.Errorf("host sudo secret namespace/template is not configured")
+	}
+	secretName := strings.ReplaceAll(template, "{node}", node)
+	secretName = strings.TrimSpace(secretName)
+	if secretName == "" {
+		return "", "secret-lookup-invalid", fmt.Errorf("host sudo secret name resolved empty")
+	}
+
+	out, err := o.kubectl(ctx, 15*time.Second, "-n", namespace, "get", "secret", secretName, "-o", "json")
+	if err != nil {
+		return "", "secret-lookup-failed", err
+	}
+	var secret hostSudoSecret
+	if err := json.Unmarshal([]byte(out), &secret); err != nil {
+		return "", "secret-decode-failed", err
+	}
+	encoded := strings.TrimSpace(secret.Data[key])
+	if encoded == "" {
+		keys := make([]string, 0, len(secret.Data))
+		for k := range secret.Data {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		return "", "secret-key-missing", fmt.Errorf("password key %q missing (available=%s)", key, joinLimited(keys, 4))
+	}
+	decoded, err := base64.StdEncoding.DecodeString(encoded)
+	if err != nil {
+		return "", "secret-decode-failed", err
+	}
+	password := strings.TrimRight(string(decoded), "\r\n")
+	if password == "" {
+		return "", "secret-empty", fmt.Errorf("password key %q is empty", key)
+	}
+	return password, "secret-lookup-ok", nil
+}

internal/cluster/orchestrator_image_pull_credentials.go

@@ -0,0 +1,321 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/state"
+)
+
+const credentialEventMaxAge = 10 * time.Minute
+const credentialRepairCooldown = 30 * time.Minute
+
+type imagePullCredentialDeploymentList struct {
+	Items []struct {
+		Metadata struct {
+			Name   string            `json:"name"`
+			Labels map[string]string `json:"labels"`
+		} `json:"metadata"`
+	} `json:"items"`
+}
+
+// imagePullCredentialBlockerReasons classifies image pull failures caused by
+// registry auth or missing imagePullSecret material.
+// Signature: (o *Orchestrator) imagePullCredentialBlockerReasons(ctx context.Context) (map[string]string, error).
+// Why: deleting a pod with bad registry credentials does not refresh Vault/CSI
+// secret material; the secret sync path needs a direct repair signal.
+func (o *Orchestrator) imagePullCredentialBlockerReasons(ctx context.Context) (map[string]string, error) {
+	eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query events for image-pull credential scan: %w", err)
+	}
+	reasons := map[string]string{}
+	if strings.TrimSpace(eventsOut) == "" {
+		return reasons, nil
+	}
+	var events eventList
+	if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
+		return nil, fmt.Errorf("decode events for image-pull credential scan: %w", err)
+	}
+	if len(events.Items) == 0 {
+		return reasons, nil
+	}
+	podsOut, err := o.kubectl(ctx, 30*time.Second, "get", "pods", "-A", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query current pods for image-pull credential scan: %w", err)
+	}
+	var pods podList
+	if err := json.Unmarshal([]byte(podsOut), &pods); err != nil {
+		return nil, fmt.Errorf("decode current pods for image-pull credential scan: %w", err)
+	}
+	blocked := map[string]string{}
+	for _, pod := range pods.Items {
+		if podHasCurrentImagePullFailure(pod) {
+			blocked[pod.Metadata.Namespace+"/"+pod.Metadata.Name] = pod.Metadata.UID
+		}
+	}
+	for _, event := range events.Items {
+		if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
+			continue
+		}
+		reason := strings.TrimSpace(event.Reason)
+		message := strings.TrimSpace(event.Message)
+		if !imagePullCredentialEvent(reason, message) {
+			continue
+		}
+		namespace := strings.TrimSpace(event.InvolvedObject.Namespace)
+		if namespace == "" {
+			namespace = strings.TrimSpace(event.Metadata.Namespace)
+		}
+		name := strings.TrimSpace(event.InvolvedObject.Name)
+		if namespace == "" || name == "" {
+			continue
+		}
+		// Retained events outlive pods and successful pulls. Require current state,
+		// exact pod identity, and recent evidence before changing a sync helper.
+		uid := blocked[namespace+"/"+name]
+		observed := eventLastObservedAt(event)
+		if uid == "" || uid != event.InvolvedObject.UID || observed.IsZero() ||
+			time.Since(observed) > credentialEventMaxAge || time.Until(observed) > time.Minute {
+			continue
+		}
+		reasons[namespace+"/"+name] = "ImagePullCredentialBlocker:" + imagePullCredentialFailureClass(reason, message)
+	}
+	return reasons, nil
+}
+
+// healImagePullCredentialSync restarts namespace-local Vault sync deployments
+// when pods are blocked on registry credentials.
+// Signature: (o *Orchestrator) healImagePullCredentialSync(ctx context.Context) ([]string, error).
+// Why: Secrets Store CSI secretObjects are materialized by mounted workloads; a
+// restarted vault-sync deployment is the bounded repair that refreshes pull
+// secrets such as harbor-regcred without mutating application deployments.
+func (o *Orchestrator) healImagePullCredentialSync(ctx context.Context) ([]string, error) {
+	if o.runner.DryRun {
+		return nil, nil
+	}
+	blockers, err := o.imagePullCredentialBlockerReasons(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if len(blockers) == 0 {
+		return nil, nil
+	}
+
+	namespaces := map[string]struct{}{}
+	for key := range blockers {
+		namespace, _, ok := strings.Cut(key, "/")
+		if ok && strings.TrimSpace(namespace) != "" {
+			namespaces[namespace] = struct{}{}
+		}
+	}
+	orderedNamespaces := make([]string, 0, len(namespaces))
+	for namespace := range namespaces {
+		orderedNamespaces = append(orderedNamespaces, namespace)
+	}
+	sort.Strings(orderedNamespaces)
+
+	repaired := []string{}
+	errs := []string{}
+	for _, namespace := range orderedNamespaces {
+		namespaceRepairs, restartErr := o.restartVaultSyncDeployments(ctx, namespace)
+		repaired = append(repaired, namespaceRepairs...)
+		if restartErr != nil {
+			errs = append(errs, restartErr.Error())
+		}
+	}
+	sort.Strings(repaired)
+	if len(errs) > 0 {
+		return repaired, errors.New(strings.Join(errs, "; "))
+	}
+	return repaired, nil
+}
+
+// restartVaultSyncDeployments runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) restartVaultSyncDeployments(ctx context.Context, namespace string) ([]string, error).
+// Why: namespaces using Vault/CSI secret material have tiny sync deployments
+// named or labeled vault-sync; restarting those nudges secretObject rotation.
+func (o *Orchestrator) restartVaultSyncDeployments(ctx context.Context, namespace string) ([]string, error) {
+	o.credentialRepairMu.Lock()
+	defer o.credentialRepairMu.Unlock()
+	out, err := o.kubectl(ctx, 20*time.Second, "-n", namespace, "get", "deployment", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query deployments in %s for image-pull credential repair: %w", namespace, err)
+	}
+	var deployments imagePullCredentialDeploymentList
+	if err := json.Unmarshal([]byte(out), &deployments); err != nil {
+		return nil, fmt.Errorf("decode deployments in %s for image-pull credential repair: %w", namespace, err)
+	}
+
+	repaired := []string{}
+	found := false
+	for _, deployment := range deployments.Items {
+		name := strings.TrimSpace(deployment.Metadata.Name)
+		if name == "" || !vaultSyncDeployment(name, deployment.Metadata.Labels) {
+			continue
+		}
+		found = true
+		allowed, err := o.reserveCredentialRepair(namespace + "/deployment/" + name)
+		if err != nil {
+			return repaired, err
+		}
+		if !allowed {
+			continue
+		}
+		if _, err := o.kubectl(ctx, 25*time.Second, "-n", namespace, "rollout", "restart", "deployment", name); err != nil {
+			return repaired, fmt.Errorf("restart %s/deployment/%s for image-pull credential repair: %w", namespace, name, err)
+		}
+		if _, err := o.kubectl(ctx, 75*time.Second, "-n", namespace, "rollout", "status", "deployment/"+name, "--timeout=60s"); err != nil {
+			return repaired, fmt.Errorf("wait for %s/deployment/%s after image-pull credential repair: %w", namespace, name, err)
+		}
+		repaired = append(repaired, namespace+"/deployment/"+name)
+	}
+	if !found {
+		return nil, fmt.Errorf("image-pull credential blocker in namespace %s but no vault-sync deployment was found", namespace)
+	}
+	return repaired, nil
+}
+
+// imagePullCredentialEvent runs one orchestration or CLI step.
+// Signature: imagePullCredentialEvent(reason string, message string) bool.
+// Why: image-pull auth detection must include kubelet pull-secret events while
+// staying separate from DNS and ordinary transient pull backoff.
+func imagePullCredentialEvent(reason string, message string) bool {
+	normalizedReason := strings.ToLower(strings.TrimSpace(reason))
+	if normalizedReason == "failedtoretrieveimagepullsecret" {
+		return true
+	}
+	if normalizedReason != "failed" &&
+		normalizedReason != "failedpull" &&
+		normalizedReason != "errimagepull" &&
+		normalizedReason != "imagepullbackoff" {
+		return false
+	}
+	return imagePullMessageHasCredentialFailure(message)
+}
+
+// imagePullMessageHasCredentialFailure runs one orchestration or CLI step.
+// Signature: imagePullMessageHasCredentialFailure(message string) bool.
+// Why: credential blockers need a tight predicate so missing images and registry
+// DNS outages keep their own repair paths.
+func imagePullMessageHasCredentialFailure(message string) bool {
+	lower := strings.ToLower(strings.TrimSpace(message))
+	if lower == "" || imagePullMessageHasDNSFailure(message) {
+		return false
+	}
+	return strings.Contains(lower, "failed to retrieve image pull secret") ||
+		strings.Contains(lower, "failedtoretrieveimagepullsecret") ||
+		strings.Contains(lower, "unable to retrieve some image pull secrets") ||
+		(strings.Contains(lower, "image pull secret") && strings.Contains(lower, "not found")) ||
+		(strings.Contains(lower, "pull secret") && strings.Contains(lower, "not found")) ||
+		(strings.Contains(lower, "secret ") && strings.Contains(lower, " not found") && strings.Contains(lower, "pull")) ||
+		strings.Contains(lower, "no basic auth credentials") ||
+		strings.Contains(lower, "unauthorized") ||
+		strings.Contains(lower, "authentication required") ||
+		strings.Contains(lower, "authorization failed") ||
+		strings.Contains(lower, "failed to authorize") ||
+		strings.Contains(lower, "invalid username/password") ||
+		strings.Contains(lower, "401 unauthorized") ||
+		strings.Contains(lower, "403 forbidden") ||
+		strings.Contains(lower, "pull access denied") ||
+		strings.Contains(lower, "requested access to the resource is denied") ||
+		strings.Contains(lower, "failed to fetch anonymous token")
+}
+
+// imagePullCredentialFailureClass runs one orchestration or CLI step.
+// Signature: imagePullCredentialFailureClass(reason string, message string) string.
+// Why: compact blocker classes keep logs and status useful without copying long
+// kubelet event messages.
+func imagePullCredentialFailureClass(reason string, message string) string {
+	lowerReason := strings.ToLower(strings.TrimSpace(reason))
+	lower := strings.ToLower(strings.TrimSpace(message))
+	switch {
+	case lowerReason == "failedtoretrieveimagepullsecret",
+		strings.Contains(lower, "failed to retrieve image pull secret"),
+		strings.Contains(lower, "unable to retrieve some image pull secrets"),
+		strings.Contains(lower, "image pull secret") && strings.Contains(lower, "not found"),
+		strings.Contains(lower, "pull secret") && strings.Contains(lower, "not found"),
+		strings.Contains(lower, "secret ") && strings.Contains(lower, " not found") && strings.Contains(lower, "pull"):
+		return "missing-pull-secret"
+	case strings.Contains(lower, "no basic auth credentials"):
+		return "no-basic-auth"
+	case strings.Contains(lower, "invalid username/password"):
+		return "invalid-credentials"
+	case strings.Contains(lower, "unauthorized") || strings.Contains(lower, "authentication required"):
+		return "unauthorized"
+	case strings.Contains(lower, "forbidden") || strings.Contains(lower, "authorization failed") || strings.Contains(lower, "failed to authorize") || strings.Contains(lower, "failed to fetch anonymous token"):
+		return "authorization-failed"
+	case strings.Contains(lower, "pull access denied") || strings.Contains(lower, "requested access to the resource is denied"):
+		return "pull-access-denied"
+	default:
+		return "registry-credential-error"
+	}
+}
+
+// vaultSyncDeployment runs one orchestration or CLI step.
+// Signature: vaultSyncDeployment(name string, labels map[string]string) bool.
+// Why: different apps may prefix their sync deployment names, but they consistently
+// identify the narrow Vault sync helper with a vault-sync name or label.
+func vaultSyncDeployment(name string, labels map[string]string) bool {
+	if strings.Contains(strings.ToLower(strings.TrimSpace(name)), "vault-sync") {
+		return true
+	}
+	for key, value := range labels {
+		if strings.Contains(strings.ToLower(strings.TrimSpace(key)), "vault-sync") ||
+			strings.Contains(strings.ToLower(strings.TrimSpace(value)), "vault-sync") {
+			return true
+		}
+	}
+	return false
+}
+
+// podHasCurrentImagePullFailure returns whether a live pod still needs a pull.
+// Completed, deleting, and recovered pods cannot justify a credential repair.
+// Signature: podHasCurrentImagePullFailure(pod podResource) bool.
+// Why: historical warnings must not trigger repairs for recovered pods.
+func podHasCurrentImagePullFailure(pod podResource) bool {
+	if pod.Metadata.DeletionTimestamp != nil || pod.Status.Phase == "Succeeded" || pod.Status.Phase == "Failed" {
+		return false
+	}
+	for _, statuses := range [][]podContainerStatus{pod.Status.InitContainerStatuses, pod.Status.ContainerStatuses} {
+		for _, status := range statuses {
+			if wait := status.State.Waiting; wait != nil && (wait.Reason == "ErrImagePull" || wait.Reason == "ImagePullBackOff") {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// reserveCredentialRepair records an attempt before mutation and returns false
+// during the cooldown. The existing run history preserves it over daemon restarts.
+// A failed or timed-out rollout also consumes the cooldown to prevent churn.
+// Signature: (o *Orchestrator) reserveCredentialRepair(target string) (bool, error).
+// Why: a cooldown must survive daemon restarts and failed rollouts.
+func (o *Orchestrator) reserveCredentialRepair(target string) (bool, error) {
+	if o.store == nil {
+		return false, errors.New("credential repair requires persistent run history")
+	}
+	records, err := o.store.Load()
+	if err != nil {
+		return false, fmt.Errorf("read credential repair history: %w", err)
+	}
+	for _, record := range records {
+		if record.Action == "image-pull-credential-repair" && record.Reason == target && time.Since(record.StartedAt) < credentialRepairCooldown {
+			return false, nil
+		}
+	}
+	now := time.Now()
+	if err := o.store.Append(state.RunRecord{ID: now.UTC().Format(time.RFC3339Nano), Action: "image-pull-credential-repair", Reason: target, StartedAt: now, EndedAt: now}); err != nil {
+		return false, fmt.Errorf("record credential repair attempt: %w", err)
+	}
+	return true, nil
+}

internal/cluster/orchestrator_image_pull_credentials_test.go

@@ -0,0 +1,230 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestImagePullCredentialBlockerReasonsClassifiesHarborAuth runs one
+// orchestration or CLI step.
+// Signature: TestImagePullCredentialBlockerReasonsClassifiesHarborAuth(t *testing.T).
+// Why: Harbor auth failures should drive secret-sync repair instead of pod
+// recycling or DNS remediation.
+func TestImagePullCredentialBlockerReasonsClassifiesHarborAuth(t *testing.T) {
+	events := `{"items":[` +
+		`{"involvedObject":{"kind":"Pod","namespace":"veles","name":"veles-backend"},"type":"Warning","reason":"Failed","message":"Failed to pull image \"registry.bstein.dev/veles/veles-backend:0.5.25\": no basic auth credentials"},` +
+		`{"metadata":{"namespace":"veles"},"involvedObject":{"kind":"Pod","name":"veles-frontend"},"type":"Warning","reason":"FailedToRetrieveImagePullSecret","message":"Unable to retrieve some image pull secrets (harbor-regcred); attempting to pull the image may not succeed."},` +
+		`{"involvedObject":{"kind":"Pod","namespace":"logging","name":"oauth2"},"type":"Warning","reason":"Failed","message":"Failed to pull image: lookup registry-1.docker.io: Try again"}` +
+		`]}`
+	events, pods := currentCredentialFixture(t, events)
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: pods},
+	})
+
+	reasons, err := orch.imagePullCredentialBlockerReasons(context.Background())
+	if err != nil {
+		t.Fatalf("imagePullCredentialBlockerReasons failed: %v", err)
+	}
+	if got := reasons["veles/veles-backend"]; got != "ImagePullCredentialBlocker:no-basic-auth" {
+		t.Fatalf("unexpected backend credential reason %q", got)
+	}
+	if got := reasons["veles/veles-frontend"]; got != "ImagePullCredentialBlocker:missing-pull-secret" {
+		t.Fatalf("unexpected frontend credential reason %q", got)
+	}
+	if _, ok := reasons["logging/oauth2"]; ok {
+		t.Fatalf("DNS image-pull blocker must not be classified as credential failure")
+	}
+}
+
+// TestHealImagePullCredentialSyncRestartsVaultSyncDeployment runs one
+// orchestration or CLI step.
+// Signature: TestHealImagePullCredentialSyncRestartsVaultSyncDeployment(t *testing.T).
+// Why: a namespace with registry credential blockers and a Vault CSI sync helper
+// should get that sync helper rolled instead of application pods deleted.
+func TestHealImagePullCredentialSyncRestartsVaultSyncDeployment(t *testing.T) {
+	events := `{"items":[{"involvedObject":{"kind":"Pod","namespace":"veles","name":"veles-backend"},"type":"Warning","reason":"Failed","message":"Failed to pull image \"registry.bstein.dev/veles/veles-backend:0.5.25\": no basic auth credentials"}]}`
+	deployments := `{"items":[` +
+		`{"metadata":{"name":"veles-backend","labels":{"app.kubernetes.io/name":"veles-backend"}}},` +
+		`{"metadata":{"name":"veles-vault-sync","labels":{"app.kubernetes.io/component":"vault-sync"}}}` +
+		`]}`
+	restarted := false
+	rolledOut := false
+	events, pods := currentCredentialFixture(t, events)
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: pods},
+		{match: matchContains("kubectl", "-n", "veles", "get", "deployment", "-o", "json"), out: deployments},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "veles", "rollout", "restart", "deployment", "veles-vault-sync")(name, args) {
+					return false
+				}
+				restarted = true
+				return true
+			},
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "veles", "rollout", "status", "deployment/veles-vault-sync", "--timeout=60s")(name, args) {
+					return false
+				}
+				rolledOut = true
+				return true
+			},
+		},
+	})
+
+	repaired, err := orch.healImagePullCredentialSync(context.Background())
+	if err != nil {
+		t.Fatalf("healImagePullCredentialSync failed: %v", err)
+	}
+	if strings.Join(repaired, ",") != "veles/deployment/veles-vault-sync" {
+		t.Fatalf("unexpected image-pull credential repair list: %#v", repaired)
+	}
+	if !restarted || !rolledOut {
+		t.Fatalf("expected vault-sync deployment restart and rollout wait, restarted=%v rolledOut=%v", restarted, rolledOut)
+	}
+}
+
+// currentCredentialFixture gives legacy fixtures current pod identities and states.
+// Signature: currentCredentialFixture(t *testing.T, raw string) (string, string).
+// Why: event-only fixtures cannot demonstrate a currently blocked pull.
+func currentCredentialFixture(t *testing.T, raw string) (string, string) {
+	t.Helper()
+	var events eventList
+	if err := json.Unmarshal([]byte(raw), &events); err != nil {
+		t.Fatal(err)
+	}
+	pods := podList{}
+	for i := range events.Items {
+		event := &events.Items[i]
+		event.LastTimestamp = time.Now()
+		event.InvolvedObject.UID = fmt.Sprintf("pod-%d", i)
+		pod := podResource{}
+		pod.Metadata.Namespace = event.InvolvedObject.Namespace
+		if pod.Metadata.Namespace == "" {
+			pod.Metadata.Namespace = event.Metadata.Namespace
+		}
+		pod.Metadata.Name = event.InvolvedObject.Name
+		pod.Metadata.UID = event.InvolvedObject.UID
+		pod.Status.Phase = "Pending"
+		pod.Status.ContainerStatuses = []podContainerStatus{{State: podContainerState{Waiting: &podContainerWaitingState{Reason: "ImagePullBackOff"}}}}
+		pods.Items = append(pods.Items, pod)
+	}
+	e, _ := json.Marshal(events)
+	p, _ := json.Marshal(pods)
+	return string(e), string(p)
+}
+
+// TestCredentialRecoveryRequiresCurrentEvidence rejects warnings for healed or replaced pods.
+// Signature: TestCredentialRecoveryRequiresCurrentEvidence(t *testing.T).
+// Why: only fresh evidence for the current pod may trigger mutation.
+func TestCredentialRecoveryRequiresCurrentEvidence(t *testing.T) {
+	for _, scenario := range []string{"current", "init", "stale", "missing-time", "future", "missing-pod", "replaced", "missing-uid", "running", "deleting", "succeeded", "failed", "query-error", "bad-pods", "empty-events"} {
+		t.Run(scenario, func(t *testing.T) {
+			raw, podsRaw := currentCredentialFixture(t, `{"items":[{"type":"Warning","reason":"Failed","message":"unauthorized","involvedObject":{"kind":"Pod","namespace":"apps","name":"test"}}]}`)
+			var events eventList
+			var pods podList
+			_ = json.Unmarshal([]byte(raw), &events)
+			_ = json.Unmarshal([]byte(podsRaw), &pods)
+			switch scenario {
+			case "init":
+				pods.Items[0].Status.InitContainerStatuses = pods.Items[0].Status.ContainerStatuses
+				pods.Items[0].Status.ContainerStatuses = nil
+			case "stale":
+				events.Items[0].LastTimestamp = time.Now().Add(-11 * time.Minute)
+			case "missing-time":
+				events.Items[0].LastTimestamp = time.Time{}
+			case "future":
+				events.Items[0].LastTimestamp = time.Now().Add(time.Hour)
+			case "missing-pod":
+				pods.Items = nil
+			case "replaced":
+				pods.Items[0].Metadata.UID = "replacement"
+			case "missing-uid":
+				events.Items[0].InvolvedObject.UID = ""
+			case "running":
+				pods.Items[0].Status.ContainerStatuses = nil
+				pods.Items[0].Status.Phase = "Running"
+			case "deleting":
+				now := time.Now()
+				pods.Items[0].Metadata.DeletionTimestamp = &now
+			case "succeeded":
+				pods.Items[0].Status.Phase = "Succeeded"
+			case "failed":
+				pods.Items[0].Status.Phase = "Failed"
+			}
+			e, _ := json.Marshal(events)
+			p, _ := json.Marshal(pods)
+			raw, podsRaw = string(e), string(p)
+			var queryErr error
+			if scenario == "query-error" {
+				queryErr = fmt.Errorf("unavailable")
+			}
+			if scenario == "bad-pods" {
+				podsRaw = "{"
+			}
+			if scenario == "empty-events" {
+				raw = ""
+			}
+			orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+				{match: matchContains("kubectl", "get", "events", "-A"), out: raw},
+				{match: matchContains("kubectl", "get", "pods", "-A"), out: podsRaw, err: queryErr},
+			})
+			reasons, err := orch.imagePullCredentialBlockerReasons(context.Background())
+			if scenario == "query-error" || scenario == "bad-pods" {
+				if err == nil {
+					t.Fatal("expected error")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatal(err)
+			}
+			want := 0
+			if scenario == "current" || scenario == "init" {
+				want = 1
+			}
+			if len(reasons) != want {
+				t.Fatalf("got %v, want %d blockers", reasons, want)
+			}
+		})
+	}
+}
+
+// TestCredentialRepairCooldownSurvivesRestart prevents repeated recovery mutations.
+// Signature: TestCredentialRepairCooldownSurvivesRestart(t *testing.T).
+// Why: a failing helper must not be restarted every recovery cycle.
+func TestCredentialRepairCooldownSurvivesRestart(t *testing.T) {
+	restarts := 0
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "deployment"), out: `{"items":[{"metadata":{"name":"app-vault-sync"}}]}`},
+		{match: func(name string, args []string) bool {
+			if matchContains("kubectl", "rollout", "restart")(name, args) {
+				restarts++
+				return true
+			}
+			return false
+		}},
+		{match: matchContains("kubectl", "rollout", "status"), err: fmt.Errorf("rollout timeout")},
+	})
+	if _, err := orch.restartVaultSyncDeployments(context.Background(), "apps"); err == nil {
+		t.Fatal("expected timeout")
+	}
+	replacement := &Orchestrator{cfg: orch.cfg, runner: orch.runner, store: orch.store, log: orch.log, runOverride: orch.runOverride}
+	repaired, err := replacement.restartVaultSyncDeployments(context.Background(), "apps")
+	if err != nil || len(repaired) != 0 || restarts != 1 {
+		t.Fatalf("repeated repair: %v %v %d", repaired, err, restarts)
+	}
+	replacement.store = nil
+	if _, err := replacement.reserveCredentialRepair("apps/other"); err == nil {
+		t.Fatal("missing state must fail closed")
+	}
+}

internal/cluster/orchestrator_image_pull_dns.go

@@ -0,0 +1,88 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// imagePullDNSBlockerReasons classifies image pull failures caused by DNS.
+// Signature: (o *Orchestrator) imagePullDNSBlockerReasons(ctx context.Context) (map[string]string, error).
+// Why: deleting an unchanged ImagePullBackOff pod does not repair registry DNS;
+// Ananke should report the blocker and wait for DNS/registry health to change.
+func (o *Orchestrator) imagePullDNSBlockerReasons(ctx context.Context) (map[string]string, error) {
+	eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query events for image-pull DNS scan: %w", err)
+	}
+	reasons := map[string]string{}
+	if strings.TrimSpace(eventsOut) == "" {
+		return reasons, nil
+	}
+	var events eventList
+	if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
+		return nil, fmt.Errorf("decode events for image-pull DNS scan: %w", err)
+	}
+	for _, event := range events.Items {
+		if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
+			continue
+		}
+		reason := strings.TrimSpace(event.Reason)
+		if reason != "Failed" && reason != "FailedPull" && reason != "ErrImagePull" && reason != "ImagePullBackOff" {
+			continue
+		}
+		message := strings.TrimSpace(event.Message)
+		if !imagePullMessageHasDNSFailure(message) {
+			continue
+		}
+		key := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name)
+		if key == "/" {
+			continue
+		}
+		reasons[key] = "ImagePullDNSBlocker:" + imagePullRegistryHost(message)
+	}
+	return reasons, nil
+}
+
+// imagePullMessageHasDNSFailure runs one orchestration or CLI step.
+// Signature: imagePullMessageHasDNSFailure(message string) bool.
+// Why: image pull recycling should be suppressed only for resolver failures,
+// leaving ordinary pull/auth errors on their existing paths.
+func imagePullMessageHasDNSFailure(message string) bool {
+	lower := strings.ToLower(message)
+	if !strings.Contains(lower, "lookup ") {
+		return false
+	}
+	return strings.Contains(lower, "try again") ||
+		strings.Contains(lower, "no such host") ||
+		strings.Contains(lower, "server misbehaving") ||
+		strings.Contains(lower, "temporary failure") ||
+		strings.Contains(lower, "i/o timeout")
+}
+
+// imagePullRegistryHost runs one orchestration or CLI step.
+// Signature: imagePullRegistryHost(message string) string.
+// Why: operator status should group DNS blockers by registry host instead of
+// repeating full kubelet event text for every pod.
+func imagePullRegistryHost(message string) string {
+	lower := strings.ToLower(message)
+	idx := strings.Index(lower, "lookup ")
+	if idx < 0 {
+		return "unknown-registry"
+	}
+	rest := strings.TrimSpace(message[idx+len("lookup "):])
+	fields := strings.Fields(rest)
+	if len(fields) == 0 {
+		return "unknown-registry"
+	}
+	host := strings.Trim(fields[0], `"'[]():,`)
+	if host == "" {
+		return "unknown-registry"
+	}
+	return host
+}

internal/cluster/orchestrator_ingress.go

@@ -19,6 +19,10 @@ func (o *Orchestrator) ensureRequiredNodeLabels(ctx context.Context) error {
 	if o.runner.DryRun || len(o.cfg.Startup.RequiredNodeLabels) == 0 {
 		return nil
 	}
+	if o.cfg.Startup.RequiredNodeLabelsMode == "validate" {
+		return o.validateRequiredNodeLabels(ctx)
+	}
+	ignored := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
 	nodes := make([]string, 0, len(o.cfg.Startup.RequiredNodeLabels))
 	for node := range o.cfg.Startup.RequiredNodeLabels {
 		node = strings.TrimSpace(node)
@@ -28,6 +32,10 @@ func (o *Orchestrator) ensureRequiredNodeLabels(ctx context.Context) error {
 	}
 	sort.Strings(nodes)
 	for _, node := range nodes {
+		if _, skip := ignored[node]; skip {
+			o.log.Printf("skipping required node labels for ignored unavailable node %s", node)
+			continue
+		}
 		labels := o.cfg.Startup.RequiredNodeLabels[node]
 		if len(labels) == 0 {
 			continue
@@ -55,6 +63,11 @@ func (o *Orchestrator) ensureRequiredNodeLabels(ctx context.Context) error {
 			continue
 		}
 		if _, err := o.kubectl(ctx, 25*time.Second, args...); err != nil {
+			if isNotFoundErr(err) && !o.startupNodeStrictlyRequired(node) {
+				o.log.Printf("warning: skipping required labels for absent non-core node %s: %v", node, err)
+				o.noteStartupAutoHeal(fmt.Sprintf("skipped required node labels for absent non-core node %s", node))
+				continue
+			}
 			return fmt.Errorf("ensure required node labels on %s (%s): %w", node, strings.Join(pairs, ", "), err)
 		}
 		o.log.Printf("ensured required labels on node %s: %s", node, strings.Join(pairs, ", "))
@@ -62,6 +75,83 @@ func (o *Orchestrator) ensureRequiredNodeLabels(ctx context.Context) error {
 	return nil
 }
 
+type nodeLabelItem struct {
+	Metadata struct {
+		Labels map[string]string `json:"labels"`
+	} `json:"metadata"`
+}
+
+// validateRequiredNodeLabels runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) validateRequiredNodeLabels(ctx context.Context) error.
+// Why: lets Terraform own steady-state labels while Ananke still reports drift
+// during startup and post-start health checks.
+func (o *Orchestrator) validateRequiredNodeLabels(ctx context.Context) error {
+	ignored := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
+	nodes := make([]string, 0, len(o.cfg.Startup.RequiredNodeLabels))
+	for node := range o.cfg.Startup.RequiredNodeLabels {
+		node = strings.TrimSpace(node)
+		if node != "" {
+			nodes = append(nodes, node)
+		}
+	}
+	sort.Strings(nodes)
+
+	drift := []string{}
+	for _, node := range nodes {
+		if _, skip := ignored[node]; skip {
+			o.log.Printf("skipping required node label validation for ignored unavailable node %s", node)
+			continue
+		}
+		required := o.cfg.Startup.RequiredNodeLabels[node]
+		if len(required) == 0 {
+			continue
+		}
+		keys := make([]string, 0, len(required))
+		for key := range required {
+			key = strings.TrimSpace(key)
+			if key != "" {
+				keys = append(keys, key)
+			}
+		}
+		sort.Strings(keys)
+		if len(keys) == 0 {
+			continue
+		}
+
+		out, err := o.kubectl(ctx, 25*time.Second, "get", "node", node, "-o", "json")
+		if err != nil {
+			if isNotFoundErr(err) && !o.startupNodeStrictlyRequired(node) {
+				o.log.Printf("warning: skipping required label validation for absent non-core node %s: %v", node, err)
+				o.noteStartupAutoHeal(fmt.Sprintf("skipped required node label validation for absent non-core node %s", node))
+				continue
+			}
+			return fmt.Errorf("validate required node labels on %s: %w", node, err)
+		}
+		var item nodeLabelItem
+		if err := json.Unmarshal([]byte(out), &item); err != nil {
+			return fmt.Errorf("decode node labels for %s: %w", node, err)
+		}
+		for _, key := range keys {
+			want := strings.TrimSpace(required[key])
+			if want == "" {
+				continue
+			}
+			got := strings.TrimSpace(item.Metadata.Labels[key])
+			if got != want {
+				if got == "" {
+					got = "<missing>"
+				}
+				drift = append(drift, fmt.Sprintf("%s %s want=%s got=%s", node, key, want, got))
+			}
+		}
+	}
+	if len(drift) > 0 {
+		return fmt.Errorf("required node label drift: %s", strings.Join(drift, "; "))
+	}
+	o.log.Printf("validated required node labels on %d node(s)", len(nodes))
+	return nil
+}
+
 // waitForStartupConvergence runs one orchestration or CLI step.
 // Signature: (o *Orchestrator) waitForStartupConvergence(ctx context.Context) error.
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

internal/cluster/orchestrator_lifecycle.go

@@ -11,6 +11,8 @@ import (
 	"scm.bstein.dev/bstein/ananke/internal/state"
 )
 
+var etcdRestorePostStartDelay = 10 * time.Second
+
 // Startup runs one orchestration or CLI step.
 // Signature: (o *Orchestrator) Startup(ctx context.Context, opts StartupOptions) (err error).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
@@ -37,6 +39,7 @@ func (o *Orchestrator) Startup(ctx context.Context, opts StartupOptions) (err er
 		return invErr
 	}
 	o.noteStartupCheck("node-inventory", true, "inventory/user/port validation passed")
+	o.maybeRunEarlyVaultUnseal(ctx)
 	o.setStartupPhase("preflight-node-reachability", "waiting for ssh reachability across configured inventory")
 	if reachErr := o.waitForNodeInventoryReachability(ctx); reachErr != nil {
 		o.noteStartupCheck("node-inventory-reachability", false, reachErr.Error())
@@ -44,12 +47,17 @@ func (o *Orchestrator) Startup(ctx context.Context, opts StartupOptions) (err er
 	}
 	o.noteStartupCheck("node-inventory-reachability", true, "all expected nodes responded over SSH")
 	resumedFlux := false
+	allowFailedStartupFluxResume := false
 	defer func() {
 		if o.runner.DryRun || err == nil || resumedFlux {
 			return
 		}
-		o.log.Printf("warning: startup failed before normal flux resume; attempting best-effort recovery resume")
 		o.bestEffort("restore scaled workloads after failed startup", func() error { return o.restoreScaledApps(ctx) })
+		if !allowFailedStartupFluxResume {
+			o.log.Printf("warning: startup failed before recovery-safe flux resume; leaving flux suspension state unchanged")
+			return
+		}
+		o.log.Printf("warning: startup failed during flux resume; attempting best-effort recovery resume")
 		o.bestEffort("resume flux after failed startup", func() error { return o.resumeFluxAndReconcile(ctx) })
 	}()
 
@@ -179,6 +187,9 @@ func (o *Orchestrator) Startup(ctx context.Context, opts StartupOptions) (err er
 		}
 	}
 	o.noteStartupCheck("kubernetes-api", true, "kubernetes api reachable")
+	if err := o.runStartupVaultUnsealGate(ctx); err != nil {
+		return err
+	}
 	if err := o.ensureRequiredNodeLabels(ctx); err != nil {
 		return err
 	}
@@ -206,11 +217,43 @@ func (o *Orchestrator) Startup(ctx context.Context, opts StartupOptions) (err er
 	}
 	o.log.Printf("startup workers=%s", strings.Join(workers, ","))
 	o.setStartupPhase("worker-start", "starting and uncordoning worker nodes")
+	runtimeWedgedNodes := []string{}
+	o.bestEffort("quarantine container-runtime-wedged nodes", func() error {
+		nodes, err := o.quarantineContainerRuntimeWedgeNodesFromCluster(ctx)
+		if err != nil {
+			return err
+		}
+		runtimeWedgedNodes = nodes
+		return nil
+	})
+	if len(runtimeWedgedNodes) > 0 {
+		ignored := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
+		for _, node := range runtimeWedgedNodes {
+			if _, ok := ignored[node]; ok {
+				continue
+			}
+			o.cfg.Startup.IgnoreUnavailableNodes = append(o.cfg.Startup.IgnoreUnavailableNodes, node)
+			ignored[node] = struct{}{}
+		}
+		workers = filterIgnoredNodes(workers, o.cfg.Startup.IgnoreUnavailableNodes)
+		o.log.Printf("startup workers after container-runtime quarantine=%s", strings.Join(workers, ","))
+	}
 	if o.cfg.Startup.ReconcileAccessOnBoot {
 		o.bestEffort("reconcile worker access", func() error { return o.reconcileNodeAccess(ctx, workers) })
 	}
 	o.startWorkers(ctx, workers)
-	o.bestEffort("uncordon workers", func() error { return o.uncordonWorkers(ctx, workers) })
+	workersToUncordon := workers
+	o.bestEffort("longhorn encrypted host prerequisites", func() error {
+		guardedWorkers, err := o.ensureLonghornEncryptedHostPrereqs(ctx, workers)
+		if err == nil {
+			workersToUncordon = guardedWorkers
+		}
+		return err
+	})
+	o.bestEffort("uncordon workers", func() error { return o.uncordonWorkers(ctx, workersToUncordon) })
+	o.bestEffort("uncordon longhorn cryptsetup-exempt hosts", func() error {
+		return o.uncordonLonghornCryptsetupExemptNodes(ctx)
+	})
 	sshCheckNodes := append([]string{}, o.cfg.ControlPlanes...)
 	sshCheckNodes = append(sshCheckNodes, workers...)
 	if err := o.waitForNodeSSHAuth(ctx, sshCheckNodes); err != nil {
@@ -288,6 +331,7 @@ func (o *Orchestrator) Startup(ctx context.Context, opts StartupOptions) (err er
 
 	o.noteStartupCheckState("flux-resume-reconcile", "running", "resuming flux and waiting for reconcile")
 	o.setStartupPhase("flux-resume", "resuming flux controllers and reconciling kustomizations")
+	allowFailedStartupFluxResume = true
 	if err := o.resumeFluxAndReconcile(ctx); err != nil {
 		o.noteStartupCheck("flux-resume-reconcile", false, err.Error())
 		return err
@@ -391,7 +435,7 @@ func (o *Orchestrator) EtcdRestore(ctx context.Context, opts EtcdRestoreOptions)
 	if _, err := o.ssh(ctx, controlPlane, "sudo systemctl start k3s || true"); err != nil {
 		return fmt.Errorf("failed to start k3s on restore control plane %s: %w", controlPlane, err)
 	}
-	time.Sleep(10 * time.Second)
+	time.Sleep(etcdRestorePostStartDelay)
 	for _, cp := range o.cfg.ControlPlanes {
 		cp := cp
 		if cp == controlPlane {
@@ -404,90 +448,3 @@ func (o *Orchestrator) EtcdRestore(ctx context.Context, opts EtcdRestoreOptions)
 	}
 	return nil
 }
-
-// Shutdown runs one orchestration or CLI step.
-// Signature: (o *Orchestrator) Shutdown(ctx context.Context, opts ShutdownOptions) (err error).
-// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
-func (o *Orchestrator) Shutdown(ctx context.Context, opts ShutdownOptions) (err error) {
-	unlock, err := state.AcquireLock(o.cfg.State.LockPath)
-	if err != nil {
-		return err
-	}
-	defer unlock()
-	if invErr := o.validateNodeInventory(); invErr != nil {
-		return invErr
-	}
-
-	record := state.RunRecord{
-		ID:        fmt.Sprintf("shutdown-%d", time.Now().UnixNano()),
-		Action:    "shutdown",
-		Reason:    opts.Reason,
-		DryRun:    o.runner.DryRun,
-		StartedAt: time.Now().UTC(),
-	}
-	defer o.finalizeRecord(&record, &err)
-	if !o.runner.DryRun {
-		if writeErr := state.MustWriteIntent(o.cfg.State.IntentPath, state.IntentShuttingDown, opts.Reason, "shutdown"); writeErr != nil {
-			return fmt.Errorf("set shutdown intent: %w", writeErr)
-		}
-		defer func() {
-			final := state.IntentShuttingDown
-			if err == nil {
-				final = state.IntentShutdownComplete
-			}
-			if writeErr := state.MustWriteIntent(o.cfg.State.IntentPath, final, opts.Reason, "shutdown"); writeErr != nil {
-				o.log.Printf("warning: write shutdown completion intent failed: %v", writeErr)
-			}
-		}()
-	}
-
-	workers, err := o.effectiveWorkers(ctx)
-	if err != nil {
-		return err
-	}
-	o.log.Printf("shutdown control-planes=%s workers=%s", strings.Join(o.cfg.ControlPlanes, ","), strings.Join(workers, ","))
-
-	o.reportFluxSource(ctx, "")
-
-	skipEtcd := opts.SkipEtcdSnapshot || o.cfg.Shutdown.SkipEtcdSnapshot
-	if !skipEtcd {
-		o.bestEffort("etcd snapshot", func() error {
-			return o.takeEtcdSnapshot(ctx, o.cfg.ControlPlanes[0])
-		})
-	}
-
-	o.bestEffort("suspend flux", func() error { return o.patchFluxSuspendAll(ctx, true) })
-	o.bestEffort("scale down apps", func() error { return o.scaleDownApps(ctx) })
-
-	skipDrain := opts.SkipDrain || o.cfg.Shutdown.SkipDrain
-	if !skipDrain {
-		o.bestEffort("drain workers", func() error { return o.drainWorkers(ctx, workers) })
-	}
-
-	shutdownMode := strings.TrimSpace(opts.Mode)
-	effectiveMode, modeErr := normalizeShutdownMode(shutdownMode)
-	if modeErr != nil {
-		return modeErr
-	}
-	o.log.Printf("shutdown execution mode=%s (requested=%q)", effectiveMode, shutdownMode)
-
-	o.stopWorkers(ctx, workers)
-	o.stopControlPlanes(ctx, o.cfg.ControlPlanes)
-	o.log.Printf("shutdown flow complete")
-	return nil
-}
-
-// normalizeShutdownMode runs one orchestration or CLI step.
-// Signature: normalizeShutdownMode(raw string) (string, error).
-// Why: keeps shutdown behavior explicit and safe by allowing only cluster-only
-// semantics while preserving compatibility with legacy "config" callers.
-func normalizeShutdownMode(raw string) (string, error) {
-	switch strings.TrimSpace(raw) {
-	case "", "config", "cluster-only":
-		return "cluster-only", nil
-	case "poweroff":
-		return "", fmt.Errorf("shutdown mode %q has been removed; ananke no longer powers off hosts", raw)
-	default:
-		return "", fmt.Errorf("unsupported shutdown mode %q (expected config|cluster-only)", raw)
-	}
-}

internal/cluster/orchestrator_longhorn_reconcile.go

@@ -0,0 +1,265 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+type longhornNodeList struct {
+	Items []longhornNode `json:"items"`
+}
+
+type longhornNode struct {
+	Metadata struct {
+		Name string `json:"name"`
+	} `json:"metadata"`
+	Spec struct {
+		AllowScheduling *bool `json:"allowScheduling"`
+	} `json:"spec"`
+	Status struct {
+		Conditions []longhornCondition `json:"conditions"`
+	} `json:"status"`
+}
+
+type longhornCondition struct {
+	Type    string `json:"type"`
+	Status  string `json:"status"`
+	Reason  string `json:"reason"`
+	Message string `json:"message"`
+}
+
+type daemonSetResource struct {
+	Metadata struct {
+		Namespace string `json:"namespace"`
+		Name      string `json:"name"`
+	} `json:"metadata"`
+	Spec struct {
+		Selector struct {
+			MatchLabels map[string]string `json:"matchLabels"`
+		} `json:"selector"`
+	} `json:"spec"`
+}
+
+// reconcileLonghornKubernetesReadiness repairs safe Kubernetes/Longhorn drift.
+// Signature: (o *Orchestrator) reconcileLonghornKubernetesReadiness(ctx context.Context) (int, error).
+// Why: Kubernetes Ready=True is not enough for PVC scheduling when Longhorn's
+// manager, labels, and node conditions disagree with the Kubernetes node model.
+func (o *Orchestrator) reconcileLonghornKubernetesReadiness(ctx context.Context) (int, error) {
+	longhornNodes, err := o.queryLonghornNodes(ctx)
+	if err != nil {
+		return 0, err
+	}
+	if len(longhornNodes.Items) == 0 {
+		return 0, nil
+	}
+	managerDS, err := o.queryLonghornManagerDaemonSet(ctx)
+	if err != nil {
+		return 0, err
+	}
+	selector := normalizeSelectorLabels(managerDS.Spec.Selector.MatchLabels)
+	if len(selector) == 0 {
+		return 0, fmt.Errorf("longhorn manager daemonset has no matchLabels selector")
+	}
+	nodes, err := o.queryReadyNodes(ctx)
+	if err != nil {
+		return 0, err
+	}
+	kubeNodes := map[string]nodeReadyItem{}
+	for _, node := range nodes.Items {
+		name := strings.TrimSpace(node.Metadata.Name)
+		if name != "" {
+			kubeNodes[name] = node
+		}
+	}
+	managerPods, err := o.longhornManagerPodsByNode(ctx, selector)
+	if err != nil {
+		return 0, err
+	}
+
+	repaired := 0
+	errs := []string{}
+	for _, lhNode := range longhornNodes.Items {
+		name := strings.TrimSpace(lhNode.Metadata.Name)
+		if name == "" {
+			continue
+		}
+		kubeNode, ok := kubeNodes[name]
+		if !ok {
+			continue
+		}
+		if _, hasManager := managerPods[name]; hasManager {
+			continue
+		}
+		readyCond := longhornConditionByType(lhNode, "Ready")
+		if readyCond == nil || strings.EqualFold(strings.TrimSpace(readyCond.Status), "True") {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(readyCond.Reason), "ManagerPodMissing") {
+			continue
+		}
+		missing := missingSelectorLabels(kubeNode.Metadata.Labels, selector)
+		if len(missing) == 0 {
+			o.log.Printf("warning: Longhorn manager pod missing on node=%s but node already matches selector; leaving for DaemonSet scheduler reconciliation", name)
+			continue
+		}
+
+		labels := make([]string, 0, len(missing))
+		for key, value := range missing {
+			labels = append(labels, key+"="+value)
+		}
+		sort.Strings(labels)
+		args := append([]string{"label", "node", name, "--overwrite"}, labels...)
+		o.log.Printf("warning: restoring Longhorn manager selector labels on node=%s labels=%s reason=LonghornNodeExistsAndManagerPodMissing", name, strings.Join(labels, ","))
+		if _, err := o.kubectl(ctx, 25*time.Second, args...); err != nil {
+			errs = append(errs, fmt.Sprintf("%s restore Longhorn selector labels: %v", name, err))
+			continue
+		}
+		repaired++
+		o.noteStartupAutoHeal(fmt.Sprintf("restored Longhorn manager selector labels on %s: %s", name, strings.Join(labels, ",")))
+	}
+	if len(errs) > 0 {
+		return repaired, errors.New(strings.Join(errs, "; "))
+	}
+	return repaired, nil
+}
+
+// queryLonghornNodes runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) queryLonghornNodes(ctx context.Context) (longhornNodeList, error).
+// Why: Longhorn/Kubernetes drift reconciliation starts from Longhorn's own node
+// model rather than assuming Kubernetes Ready is sufficient for storage.
+func (o *Orchestrator) queryLonghornNodes(ctx context.Context) (longhornNodeList, error) {
+	out, err := o.kubectl(ctx, 30*time.Second, "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json")
+	if err != nil {
+		if isNotFoundErr(err) {
+			return longhornNodeList{}, nil
+		}
+		return longhornNodeList{}, fmt.Errorf("query longhorn nodes: %w", err)
+	}
+	if strings.TrimSpace(out) == "" {
+		return longhornNodeList{}, nil
+	}
+	var nodes longhornNodeList
+	if err := json.Unmarshal([]byte(out), &nodes); err != nil {
+		return longhornNodeList{}, fmt.Errorf("decode longhorn nodes: %w", err)
+	}
+	return nodes, nil
+}
+
+// queryLonghornManagerDaemonSet runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) queryLonghornManagerDaemonSet(ctx context.Context) (daemonSetResource, error).
+// Why: manager selector labels are installation policy, so repair must read
+// the live DaemonSet selector instead of hard-coding a Titan label.
+func (o *Orchestrator) queryLonghornManagerDaemonSet(ctx context.Context) (daemonSetResource, error) {
+	out, err := o.kubectl(ctx, 20*time.Second, "-n", "longhorn-system", "get", "daemonset", "longhorn-manager", "-o", "json")
+	if err != nil {
+		return daemonSetResource{}, fmt.Errorf("query longhorn manager daemonset: %w", err)
+	}
+	var ds daemonSetResource
+	if err := json.Unmarshal([]byte(out), &ds); err != nil {
+		return daemonSetResource{}, fmt.Errorf("decode longhorn manager daemonset: %w", err)
+	}
+	return ds, nil
+}
+
+// longhornManagerPodsByNode runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) longhornManagerPodsByNode(ctx context.Context, selector map[string]string) (map[string]struct{}, error).
+// Why: manager-pod presence is the postcondition for selector repair and
+// distinguishes label drift from a healthy Longhorn node.
+func (o *Orchestrator) longhornManagerPodsByNode(ctx context.Context, selector map[string]string) (map[string]struct{}, error) {
+	args := []string{"-n", "longhorn-system", "get", "pods", "-o", "json"}
+	if len(selector) > 0 {
+		args = append(args, "-l", selectorLabelString(selector))
+	}
+	out, err := o.kubectl(ctx, 20*time.Second, args...)
+	if err != nil {
+		return nil, fmt.Errorf("query longhorn manager pods: %w", err)
+	}
+	var pods podList
+	if err := json.Unmarshal([]byte(out), &pods); err != nil {
+		return nil, fmt.Errorf("decode longhorn manager pods: %w", err)
+	}
+	byNode := map[string]struct{}{}
+	for _, pod := range pods.Items {
+		node := strings.TrimSpace(pod.Spec.NodeName)
+		if node == "" {
+			continue
+		}
+		if podMatchesLabels(pod.Metadata.Labels, selector) {
+			byNode[node] = struct{}{}
+		}
+	}
+	return byNode, nil
+}
+
+// normalizeSelectorLabels runs one orchestration or CLI step.
+// Signature: normalizeSelectorLabels(labels map[string]string) map[string]string.
+// Why: empty selector fragments should not produce accidental node label writes.
+func normalizeSelectorLabels(labels map[string]string) map[string]string {
+	out := map[string]string{}
+	for key, value := range labels {
+		key = strings.TrimSpace(key)
+		value = strings.TrimSpace(value)
+		if key != "" && value != "" {
+			out[key] = value
+		}
+	}
+	return out
+}
+
+// selectorLabelString runs one orchestration or CLI step.
+// Signature: selectorLabelString(labels map[string]string) string.
+// Why: kubectl label selectors need stable ordering for deterministic tests and
+// readable logs.
+func selectorLabelString(labels map[string]string) string {
+	parts := make([]string, 0, len(labels))
+	for key, value := range labels {
+		parts = append(parts, key+"="+value)
+	}
+	sort.Strings(parts)
+	return strings.Join(parts, ",")
+}
+
+// podMatchesLabels runs one orchestration or CLI step.
+// Signature: podMatchesLabels(labels map[string]string, selector map[string]string) bool.
+// Why: Longhorn manager pod checks should use the DaemonSet selector rather
+// than assuming a specific app label.
+func podMatchesLabels(labels map[string]string, selector map[string]string) bool {
+	for key, value := range selector {
+		if labels == nil || strings.TrimSpace(labels[key]) != value {
+			return false
+		}
+	}
+	return true
+}
+
+// missingSelectorLabels runs one orchestration or CLI step.
+// Signature: missingSelectorLabels(labels map[string]string, selector map[string]string) map[string]string.
+// Why: label-drift repair should write exactly the manager selector labels that
+// are absent or wrong on the node.
+func missingSelectorLabels(labels map[string]string, selector map[string]string) map[string]string {
+	missing := map[string]string{}
+	for key, value := range selector {
+		if labels == nil || strings.TrimSpace(labels[key]) != value {
+			missing[key] = value
+		}
+	}
+	return missing
+}
+
+// longhornConditionByType runs one orchestration or CLI step.
+// Signature: longhornConditionByType(node longhornNode, condType string) *longhornCondition.
+// Why: Longhorn node condition parsing is shared by readiness and schedulability
+// checks and should remain case-insensitive.
+func longhornConditionByType(node longhornNode, condType string) *longhornCondition {
+	for i := range node.Status.Conditions {
+		if strings.EqualFold(strings.TrimSpace(node.Status.Conditions[i].Type), condType) {
+			return &node.Status.Conditions[i]
+		}
+	}
+	return nil
+}

internal/cluster/orchestrator_longhorn_recovery.go

@@ -0,0 +1,266 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// repairEncryptedVolumeMountPrereqs runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) repairEncryptedVolumeMountPrereqs(ctx context.Context, pods podList, grace time.Duration) (map[string]string, error).
+// Why: encrypted Longhorn volume mounts depend on host cryptsetup. After node
+// rebuilds or partial OS recovery, Kubernetes may be ready while kubelet cannot
+// mount encrypted PVCs; installing the missing host tool and recycling the
+// controller-owned pod lets kubelet retry the same volume safely.
+func (o *Orchestrator) repairEncryptedVolumeMountPrereqs(ctx context.Context, pods podList, grace time.Duration) (map[string]string, error) {
+	eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query events for encrypted volume mount scan: %w", err)
+	}
+	var events eventList
+	if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
+		return nil, fmt.Errorf("decode events for encrypted volume mount scan: %w", err)
+	}
+
+	podsByKey := map[string]podResource{}
+	for _, pod := range pods.Items {
+		ns := strings.TrimSpace(pod.Metadata.Namespace)
+		name := strings.TrimSpace(pod.Metadata.Name)
+		node := strings.TrimSpace(pod.Spec.NodeName)
+		if ns == "" || name == "" || node == "" {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(pod.Status.Phase), "Pending") {
+			continue
+		}
+		if !podControllerOwned(pod) {
+			continue
+		}
+		if !pod.Metadata.CreationTimestamp.IsZero() && time.Since(pod.Metadata.CreationTimestamp) < grace {
+			continue
+		}
+		podsByKey[ns+"/"+name] = pod
+	}
+	if len(podsByKey) == 0 {
+		return map[string]string{}, nil
+	}
+
+	repairedNodes := map[string]bool{}
+	reasons := map[string]string{}
+	for _, event := range events.Items {
+		if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
+			continue
+		}
+		if strings.TrimSpace(event.Reason) != "FailedMount" {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
+			continue
+		}
+		key := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name)
+		pod, ok := podsByKey[key]
+		if !ok {
+			continue
+		}
+		lastSeen := eventLastObservedAt(event)
+		if !lastSeen.IsZero() && !pod.Metadata.CreationTimestamp.IsZero() && lastSeen.Before(pod.Metadata.CreationTimestamp) {
+			continue
+		}
+		message := strings.ToLower(strings.TrimSpace(event.Message))
+		if !strings.Contains(message, "cryptsetup") || !strings.Contains(message, "no such file or directory") {
+			continue
+		}
+		node := strings.TrimSpace(pod.Spec.NodeName)
+		if node == "" || !o.sshManaged(node) {
+			o.log.Printf("warning: encrypted volume mount blocked on unmanaged node %s for pod %s", node, key)
+			continue
+		}
+		if repaired, ok := repairedNodes[node]; ok {
+			if repaired {
+				reasons[key] = "EncryptedVolumeCryptsetupRepaired:" + node
+			}
+			continue
+		}
+		if err := o.ensureHostCryptsetup(ctx, node); err != nil {
+			repairedNodes[node] = false
+			o.log.Printf("warning: cryptsetup prerequisite repair failed on %s for pod %s: %v", node, key, err)
+			if cordonErr := o.cordonNodeForMissingCryptsetup(ctx, node); cordonErr != nil {
+				o.log.Printf("warning: cordon failed after cryptsetup repair failure on %s for pod %s: %v", node, key, cordonErr)
+				continue
+			}
+			reasons[key] = "EncryptedVolumeCryptsetupNodeCordoned:" + node
+			continue
+		}
+		repairedNodes[node] = true
+		reasons[key] = "EncryptedVolumeCryptsetupRepaired:" + node
+	}
+	return reasons, nil
+}
+
+// ensureHostCryptsetup runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) ensureHostCryptsetup(ctx context.Context, node string) error.
+// Why: kubelet's encrypted Longhorn mount helper shells into the host namespace,
+// so the package must exist on the node host, not merely inside a workload pod.
+func (o *Orchestrator) ensureHostCryptsetup(ctx context.Context, node string) error {
+	checkCommand := strings.Join([]string{
+		"set -eu",
+		"if command -v cryptsetup >/dev/null 2>&1 || [ -x /usr/sbin/cryptsetup ] || [ -x /usr/bin/cryptsetup ]; then echo __ANANKE_CRYPTSETUP_PRESENT__; exit 0; fi",
+		"if ! command -v apt-get >/dev/null 2>&1; then echo __ANANKE_CRYPTSETUP_NO_APT__; exit 42; fi",
+		"echo __ANANKE_CRYPTSETUP_MISSING__",
+		"exit 41",
+	}, "; ")
+	out, err := o.sshWithTimeout(ctx, node, checkCommand, 25*time.Second)
+	if err == nil && strings.Contains(out, "__ANANKE_CRYPTSETUP_PRESENT__") {
+		o.log.Printf("ensured cryptsetup prerequisite on %s: __ANANKE_CRYPTSETUP_PRESENT__", node)
+		return nil
+	}
+	if strings.Contains(out, "__ANANKE_CRYPTSETUP_NO_APT__") {
+		return fmt.Errorf("install cryptsetup-bin: apt-get is unavailable on host")
+	}
+
+	out, err = o.runHostPrivilegedAction(ctx, node, hostActionInstallCryptsetup, 5*time.Minute)
+	if err != nil {
+		return fmt.Errorf("install cryptsetup-bin: %w (output=%s)", err, strings.TrimSpace(out))
+	}
+	if strings.Contains(out, "__ANANKE_CRYPTSETUP_INSTALLED__") {
+		o.log.Printf("ensured cryptsetup prerequisite on %s: __ANANKE_CRYPTSETUP_INSTALLED__", node)
+		o.noteStartupAutoHeal(fmt.Sprintf("installed cryptsetup on %s", node))
+		return nil
+	}
+	verifyOut, verifyErr := o.sshWithTimeout(ctx, node, checkCommand, 25*time.Second)
+	if verifyErr != nil || !strings.Contains(verifyOut, "__ANANKE_CRYPTSETUP_PRESENT__") {
+		return fmt.Errorf("cryptsetup-bin install completed but verification failed: %w (output=%s)", verifyErr, strings.TrimSpace(verifyOut))
+	}
+	o.log.Printf("ensured cryptsetup prerequisite on %s: __ANANKE_CRYPTSETUP_INSTALLED__", node)
+	o.noteStartupAutoHeal(fmt.Sprintf("installed cryptsetup on %s", node))
+	return nil
+}
+
+// cordonNodeForMissingCryptsetup runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) cordonNodeForMissingCryptsetup(ctx context.Context, node string) error.
+// Why: when host package repair is not permitted, cordoning is the safest
+// automatic fallback: it prevents new encrypted-volume pods from landing on a
+// node kubelet cannot mount from, while leaving existing workloads and storage
+// objects untouched.
+func (o *Orchestrator) cordonNodeForMissingCryptsetup(ctx context.Context, node string) error {
+	if err := o.cordonNodeWithLease(ctx, node, cordonReasonMissingCryptsetup, "encrypted Longhorn volume mount failed because host cryptsetup is missing"); err != nil {
+		return err
+	}
+	o.log.Printf("cordoned node %s after encrypted volume cryptsetup prerequisite failure", node)
+	o.noteStartupAutoHeal(fmt.Sprintf("cordoned %s after missing cryptsetup blocked encrypted volume mount", node))
+	return nil
+}
+
+// longhornAttachBlockedPodReasons runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) longhornAttachBlockedPodReasons(ctx context.Context, pods podList, grace time.Duration) (map[string]string, error).
+// Why: after a power event, Kubernetes can schedule a Longhorn-backed pod onto a
+// node Longhorn still marks unready. Recycling the unattached Pending pod lets
+// the scheduler pick a Longhorn-ready node without touching Longhorn data-plane
+// objects.
+func (o *Orchestrator) longhornAttachBlockedPodReasons(ctx context.Context, pods podList, grace time.Duration) (map[string]string, error) {
+	unreadyNodes, err := o.longhornUnreadyNodes(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if len(unreadyNodes) == 0 {
+		return map[string]string{}, nil
+	}
+
+	eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query events for longhorn attach-blocked pod scan: %w", err)
+	}
+	var events eventList
+	if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
+		return nil, fmt.Errorf("decode events for longhorn attach-blocked pod scan: %w", err)
+	}
+
+	podsByKey := map[string]podResource{}
+	for _, pod := range pods.Items {
+		ns := strings.TrimSpace(pod.Metadata.Namespace)
+		name := strings.TrimSpace(pod.Metadata.Name)
+		node := strings.TrimSpace(pod.Spec.NodeName)
+		if ns == "" || name == "" || node == "" {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(pod.Status.Phase), "Pending") {
+			continue
+		}
+		if _, unready := unreadyNodes[node]; !unready {
+			continue
+		}
+		if !podControllerOwned(pod) {
+			continue
+		}
+		if !pod.Metadata.CreationTimestamp.IsZero() && time.Since(pod.Metadata.CreationTimestamp) < grace {
+			continue
+		}
+		podsByKey[ns+"/"+name] = pod
+	}
+	if len(podsByKey) == 0 {
+		return map[string]string{}, nil
+	}
+
+	reasons := map[string]string{}
+	for _, event := range events.Items {
+		if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
+			continue
+		}
+		if strings.TrimSpace(event.Reason) != "FailedAttachVolume" {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
+			continue
+		}
+		key := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name)
+		pod, ok := podsByKey[key]
+		if !ok {
+			continue
+		}
+		lastSeen := eventLastObservedAt(event)
+		if !lastSeen.IsZero() && !pod.Metadata.CreationTimestamp.IsZero() && lastSeen.Before(pod.Metadata.CreationTimestamp) {
+			continue
+		}
+		node := strings.TrimSpace(pod.Spec.NodeName)
+		message := strings.ToLower(strings.TrimSpace(event.Message))
+		if !strings.Contains(message, "longhorn-backend") || !strings.Contains(message, "failed for volume") {
+			continue
+		}
+		if !strings.Contains(message, "node "+strings.ToLower(node)+" is not ready") {
+			continue
+		}
+		reasons[key] = "LonghornAttachBlockedOnUnreadyNode:" + node
+	}
+	return reasons, nil
+}
+
+// longhornUnreadyNodes runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) longhornUnreadyNodes(ctx context.Context) (map[string]struct{}, error).
+// Why: Longhorn node readiness can lag or intentionally differ from Kubernetes
+// node readiness; attach recovery must use Longhorn's view for safety.
+func (o *Orchestrator) longhornUnreadyNodes(ctx context.Context) (map[string]struct{}, error) {
+	out, err := o.kubectl(ctx, 30*time.Second,
+		"-n", "longhorn-system",
+		"get", "nodes.longhorn.io",
+		"-o", "jsonpath={range .items[*]}{.metadata.name}{'\\t'}{range .status.conditions[?(@.type==\"Ready\")]}{.status}{end}{'\\n'}{end}",
+	)
+	if err != nil {
+		if isNotFoundErr(err) {
+			return map[string]struct{}{}, nil
+		}
+		return nil, fmt.Errorf("query longhorn node readiness: %w", err)
+	}
+	unready := map[string]struct{}{}
+	for _, line := range lines(out) {
+		fields := strings.Fields(line)
+		if len(fields) < 2 {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(fields[1]), "True") {
+			unready[strings.TrimSpace(fields[0])] = struct{}{}
+		}
+	}
+	return unready, nil
+}

internal/cluster/orchestrator_node_reachability.go

@@ -30,7 +30,7 @@ func (o *Orchestrator) waitForNodeInventoryReachability(ctx context.Context) err
 	ignored := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
 	targets := make([]string, 0, len(o.inventoryNodesForValidation()))
 	seen := map[string]struct{}{}
-	for _, node := range o.inventoryNodesForValidation() {
+	for _, node := range startupRequiredNodes(o.inventoryNodesForValidation(), o.cfg.Startup.NodeInventoryReachRequiredNodes) {
 		node = strings.TrimSpace(node)
 		if node == "" {
 			continue

internal/cluster/orchestrator_node_runtime_recovery.go

@@ -0,0 +1,162 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// recoverManagedNodeRuntime runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) recoverManagedNodeRuntime(ctx context.Context, node string, alreadyUnschedulable bool, reason string) error.
+// Why: kubelet/containerd wedges need one bounded escalation ladder: cordon,
+// non-blocking k3s-agent restart, health proof, and optional controlled reboot.
+func (o *Orchestrator) recoverManagedNodeRuntime(ctx context.Context, node string, alreadyUnschedulable bool, reason string) error {
+	node = strings.TrimSpace(node)
+	if node == "" {
+		return fmt.Errorf("node is empty")
+	}
+	if !alreadyUnschedulable {
+		if err := o.cordonNodeWithLease(ctx, node, cordonReasonKubeletProxy, reason); err != nil {
+			return fmt.Errorf("cordon before kubelet restart: %w", err)
+		}
+	}
+
+	o.log.Printf("warning: detected node runtime repair signal on %s; issuing non-blocking k3s-agent restart reason=%s", node, sanitizeCordonAnnotationValue(reason))
+	if _, err := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentRestart, 30*time.Second); err != nil {
+		if !alreadyUnschedulable {
+			o.bestEffort("uncordon node after failed kubelet proxy repair", func() error {
+				return o.uncordonAndClearCordonLease(ctx, node, cordonReasonKubeletProxy)
+			})
+		}
+		return fmt.Errorf("restart k3s-agent: %w", err)
+	}
+
+	readyErr := o.waitForManagedNodeRuntimeReady(ctx, node, o.nodeRuntimeRestartWait())
+	if readyErr == nil {
+		return o.releaseRuntimeRecoveryCordon(ctx, node, alreadyUnschedulable)
+	}
+
+	showOut, showErr := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentShow, 20*time.Second)
+	if showErr != nil {
+		return fmt.Errorf("%v; k3s-agent state check failed: %w", readyErr, showErr)
+	}
+	if !k3sAgentStateRequiresReboot(showOut) {
+		return readyErr
+	}
+	if !o.cfg.Startup.HostRepairAllowReboot {
+		return fmt.Errorf("node runtime requires controlled reboot but host_repair_allow_reboot is false; k3s-agent state=%s", summarizeSystemctlShow(showOut))
+	}
+
+	o.log.Printf("warning: escalating node runtime recovery to controlled reboot node=%s state=%s", node, summarizeSystemctlShow(showOut))
+	if _, err := o.runHostPrivilegedAction(ctx, node, hostActionHostReboot, 30*time.Second); err != nil {
+		return fmt.Errorf("controlled reboot command failed: %w", err)
+	}
+	if err := o.waitForManagedNodeRuntimeReady(ctx, node, o.nodeRuntimeRebootWait()); err != nil {
+		return fmt.Errorf("node did not recover after controlled reboot: %w", err)
+	}
+	return o.releaseRuntimeRecoveryCordon(ctx, node, alreadyUnschedulable)
+}
+
+// releaseRuntimeRecoveryCordon runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) releaseRuntimeRecoveryCordon(ctx context.Context, node string, alreadyUnschedulable bool) error.
+// Why: nodes Ananke cordoned for repair should return to service after proof of
+// recovery, while pre-existing cordons remain under their original owner.
+func (o *Orchestrator) releaseRuntimeRecoveryCordon(ctx context.Context, node string, alreadyUnschedulable bool) error {
+	if alreadyUnschedulable {
+		return nil
+	}
+	if err := o.uncordonAndClearCordonLease(ctx, node, cordonReasonKubeletProxy); err != nil {
+		return fmt.Errorf("uncordon after kubelet proxy repair: %w", err)
+	}
+	return nil
+}
+
+// nodeRuntimeRestartWait runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) nodeRuntimeRestartWait() time.Duration.
+// Why: k3s-agent restart recovery needs a bounded wait even when tests build a
+// partial startup config.
+func (o *Orchestrator) nodeRuntimeRestartWait() time.Duration {
+	seconds := o.cfg.Startup.NodeRuntimeRestartWaitSeconds
+	if seconds <= 0 {
+		seconds = 140
+	}
+	return time.Duration(seconds) * time.Second
+}
+
+// nodeRuntimeRebootWait runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) nodeRuntimeRebootWait() time.Duration.
+// Why: controlled reboot recovery has a longer but still finite postcondition
+// window for SSH, systemd, kubelet, and Kubernetes readiness.
+func (o *Orchestrator) nodeRuntimeRebootWait() time.Duration {
+	seconds := o.cfg.Startup.NodeRuntimeRebootWaitSeconds
+	if seconds <= 0 {
+		seconds = 420
+	}
+	return time.Duration(seconds) * time.Second
+}
+
+// waitForManagedNodeRuntimeReady runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) waitForManagedNodeRuntimeReady(ctx context.Context, node string, wait time.Duration) error.
+// Why: runtime repair is not complete until Kubernetes Ready, kubelet proxy, and
+// k3s-agent service state all agree the node is usable.
+func (o *Orchestrator) waitForManagedNodeRuntimeReady(ctx context.Context, node string, wait time.Duration) error {
+	if wait <= 0 {
+		wait = 140 * time.Second
+	}
+	kubeWait := wait - 20*time.Second
+	if kubeWait < 10*time.Second {
+		kubeWait = 10 * time.Second
+	}
+	if _, err := o.kubectl(ctx, wait, "wait", "node/"+node, "--for=condition=Ready", "--timeout="+fmt.Sprintf("%.0fs", kubeWait.Seconds())); err != nil {
+		return fmt.Errorf("wait Ready after k3s-agent restart: %w", err)
+	}
+	healthy, err := o.kubeletProxyHealthy(ctx, node)
+	if !healthy {
+		return fmt.Errorf("proxy still broken after k3s-agent restart: %v", err)
+	}
+	out, err := o.runHostPrivilegedAction(ctx, node, hostActionK3sAgentIsActive, 12*time.Second)
+	if err != nil {
+		return fmt.Errorf("k3s-agent active check: %w", err)
+	}
+	if strings.TrimSpace(out) != "active" {
+		return fmt.Errorf("k3s-agent is-active returned %q", strings.TrimSpace(out))
+	}
+	return nil
+}
+
+// k3sAgentStateRequiresReboot runs one orchestration or CLI step.
+// Signature: k3sAgentStateRequiresReboot(out string) bool.
+// Why: only the stuck systemd states observed during hard runtime wedges should
+// escalate from non-blocking restart to controlled reboot.
+func k3sAgentStateRequiresReboot(out string) bool {
+	lower := strings.ToLower(out)
+	needles := []string{
+		"activestate=deactivating",
+		"activestate=activating",
+		"substate=stop-sigterm",
+		"substate=stop-sigkill",
+		"substate=final-sigterm",
+		"substate=final-sigkill",
+		"substate=start",
+		"result=timeout",
+	}
+	for _, needle := range needles {
+		if strings.Contains(lower, needle) {
+			return true
+		}
+	}
+	return false
+}
+
+// summarizeSystemctlShow runs one orchestration or CLI step.
+// Signature: summarizeSystemctlShow(out string) string.
+// Why: operator-facing runtime incidents need compact systemd state summaries
+// rather than full multi-line command output.
+func summarizeSystemctlShow(out string) string {
+	lines := lines(out)
+	if len(lines) == 0 {
+		return "unknown"
+	}
+	return sanitizeCordonAnnotationValue(joinLimited(lines, 4))
+}

internal/cluster/orchestrator_pod_recovery.go

@@ -0,0 +1,139 @@
+package cluster
+
+import (
+	"context"
+	"strings"
+	"time"
+)
+
+// staleControllerPodReasons runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) staleControllerPodReasons(ctx context.Context, pods podList, grace time.Duration) (map[string]string, error).
+// Why: after node or kubelet recovery, controller-owned pods can stay in
+// terminal or unknown status even though the node is Ready and a replacement may
+// already be healthy. A normal pod delete lets Kubernetes clean the stale status
+// without touching storage objects or forcing deletion on a partitioned node.
+func (o *Orchestrator) staleControllerPodReasons(ctx context.Context, pods podList, grace time.Duration) (map[string]string, error) {
+	unavailable, err := o.unavailableNodeSet(ctx)
+	if err != nil {
+		return nil, err
+	}
+	reasons := map[string]string{}
+	for _, pod := range pods.Items {
+		ns := strings.TrimSpace(pod.Metadata.Namespace)
+		name := strings.TrimSpace(pod.Metadata.Name)
+		node := strings.TrimSpace(pod.Spec.NodeName)
+		if ns == "" || name == "" || node == "" {
+			continue
+		}
+		phase := strings.TrimSpace(pod.Status.Phase)
+		if !strings.EqualFold(phase, "Unknown") && !strings.EqualFold(phase, "Failed") {
+			continue
+		}
+		if _, badNode := unavailable[node]; badNode {
+			continue
+		}
+		if !podControllerOwned(pod) {
+			continue
+		}
+		if !pod.Metadata.CreationTimestamp.IsZero() && time.Since(pod.Metadata.CreationTimestamp) < grace {
+			continue
+		}
+		reasons[ns+"/"+name] = "StaleControllerPodOnReadyNode:" + node + ":" + phase
+	}
+	return reasons, nil
+}
+
+// staleControllerPodForceDeleteSafe runs one orchestration or CLI step.
+// Signature: staleControllerPodForceDeleteSafe(pod podResource, grace time.Duration) bool.
+// Why: a stale pod already marked for deletion may need force removal after a
+// node outage. Keep that fallback away from PVC-bearing pods so Ananke never
+// risks duplicating a storage writer.
+func staleControllerPodForceDeleteSafe(pod podResource, grace time.Duration) bool {
+	if pod.Metadata.DeletionTimestamp == nil {
+		return false
+	}
+	if time.Since(*pod.Metadata.DeletionTimestamp) < grace {
+		return false
+	}
+	if podUsesPersistentVolumeClaim(pod) {
+		return false
+	}
+	return true
+}
+
+// podUsesPersistentVolumeClaim runs one orchestration or CLI step.
+// Signature: podUsesPersistentVolumeClaim(pod podResource) bool.
+// Why: force-delete recovery is deliberately disallowed for pods with PVCs; the
+// scheduler and storage controller need to settle those normally.
+func podUsesPersistentVolumeClaim(pod podResource) bool {
+	for _, volume := range pod.Spec.Volumes {
+		if volume.PersistentVolumeClaim != nil && strings.TrimSpace(volume.PersistentVolumeClaim.ClaimName) != "" {
+			return true
+		}
+	}
+	return false
+}
+
+// podControllerOwned runs one orchestration or CLI step.
+// Signature: podControllerOwned(p podResource) bool.
+// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
+func podControllerOwned(p podResource) bool {
+	for _, owner := range p.Metadata.OwnerReferences {
+		switch strings.TrimSpace(owner.Kind) {
+		case "ReplicaSet", "StatefulSet", "DaemonSet", "Job":
+			return true
+		}
+	}
+	return false
+}
+
+// stuckContainerReason runs one orchestration or CLI step.
+// Signature: stuckContainerReason(p podResource, reasons map[string]struct{}) string.
+// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
+func stuckContainerReason(p podResource, reasons map[string]struct{}) string {
+	check := func(statuses []podContainerStatus) string {
+		for _, st := range statuses {
+			if st.State.Waiting == nil {
+				continue
+			}
+			reason := strings.TrimSpace(st.State.Waiting.Reason)
+			if reason == "" {
+				continue
+			}
+			if _, ok := reasons[reason]; ok {
+				return reason
+			}
+		}
+		return ""
+	}
+	if reason := check(p.Status.InitContainerStatuses); reason != "" {
+		return reason
+	}
+	return check(p.Status.ContainerStatuses)
+}
+
+// stuckVaultInitReason runs one orchestration or CLI step.
+// Signature: stuckVaultInitReason(p podResource, grace time.Duration) string.
+// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
+func stuckVaultInitReason(p podResource, grace time.Duration) string {
+	if !strings.EqualFold(strings.TrimSpace(p.Status.Phase), "Pending") {
+		return ""
+	}
+	if !strings.EqualFold(strings.TrimSpace(p.Metadata.Annotations["vault.hashicorp.com/agent-inject"]), "true") {
+		return ""
+	}
+	for _, st := range p.Status.InitContainerStatuses {
+		if strings.TrimSpace(st.Name) != "vault-agent-init" || st.State.Running == nil {
+			continue
+		}
+		startedAt := st.State.Running.StartedAt
+		if startedAt.IsZero() {
+			continue
+		}
+		if time.Since(startedAt) < grace {
+			return ""
+		}
+		return "VaultInitStuck"
+	}
+	return ""
+}

internal/cluster/orchestrator_poststart_exec.go

@@ -187,6 +187,81 @@ func (o *Orchestrator) ssh(ctx context.Context, node string, command string) (st
 // Signature: (o *Orchestrator) sshWithTimeout(ctx context.Context, node string, command string, timeout time.Duration) (string, error).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
 func (o *Orchestrator) sshWithTimeout(ctx context.Context, node string, command string, timeout time.Duration) (string, error) {
+	attempts, attemptNames, knownHostsFiles, repairHosts := o.sshCommandAttempts(node, command)
+
+	var lastOut string
+	var lastErr error
+	for i, args := range attempts {
+		out, err := o.run(ctx, timeout, "ssh", args...)
+		if err == nil {
+			if i > 0 {
+				o.log.Printf("warning: ssh %s path failed for %s, using %s path", attemptNames[i-1], node, attemptNames[i])
+			}
+			return out, nil
+		}
+		if sshutil.ShouldAttemptKnownHostsRepair(out, err) {
+			o.log.Printf("warning: ssh failure on %s via %s path may be host-key related; repairing known_hosts and retrying once", node, attemptNames[i])
+			sshutil.RepairKnownHosts(ctx, o.log, knownHostsFiles, repairHosts, o.cfg.SSHPort)
+			retryOut, retryErr := o.run(ctx, timeout, "ssh", args...)
+			if retryErr == nil {
+				return retryOut, nil
+			}
+			out = retryOut
+			err = retryErr
+		}
+		lastOut = out
+		lastErr = err
+		if i < len(attempts)-1 {
+			o.log.Printf("warning: ssh %s path failed for %s: %v; trying %s path", attemptNames[i], node, err, attemptNames[i+1])
+		}
+	}
+	return lastOut, lastErr
+}
+
+// sshWithInput runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) sshWithInput(ctx context.Context, node string, command string, input string, timeout time.Duration) (string, error).
+// Why: Vault-backed sudo must send passwords over stdin, not in the SSH command
+// line or logs, while still using Ananke's configured node route.
+func (o *Orchestrator) sshWithInput(ctx context.Context, node string, command string, input string, timeout time.Duration) (string, error) {
+	if o.sshInputOverride != nil {
+		return o.sshInputOverride(ctx, timeout, node, command, input)
+	}
+	attempts, attemptNames, knownHostsFiles, repairHosts := o.sshCommandAttempts(node, command)
+
+	var lastOut string
+	var lastErr error
+	for i, args := range attempts {
+		out, err := o.runSensitiveWithInput(ctx, timeout, input, "ssh", args...)
+		if err == nil {
+			if i > 0 {
+				o.log.Printf("warning: ssh %s path failed for %s, using %s path", attemptNames[i-1], node, attemptNames[i])
+			}
+			return out, nil
+		}
+		if sshutil.ShouldAttemptKnownHostsRepair(out, err) {
+			o.log.Printf("warning: ssh failure on %s via %s path may be host-key related; repairing known_hosts and retrying once", node, attemptNames[i])
+			sshutil.RepairKnownHosts(ctx, o.log, knownHostsFiles, repairHosts, o.cfg.SSHPort)
+			retryOut, retryErr := o.runSensitiveWithInput(ctx, timeout, input, "ssh", args...)
+			if retryErr == nil {
+				return retryOut, nil
+			}
+			out = retryOut
+			err = retryErr
+		}
+		lastOut = out
+		lastErr = err
+		if i < len(attempts)-1 {
+			o.log.Printf("warning: ssh %s path failed for %s: %v; trying %s path", attemptNames[i], node, err, attemptNames[i+1])
+		}
+	}
+	return lastOut, lastErr
+}
+
+// sshCommandAttempts runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) sshCommandAttempts(node string, command string) ([][]string, []string, []string, []string).
+// Why: ordinary SSH and stdin-backed SSH must use identical host, identity,
+// port, jump-host, and known-host repair behavior.
+func (o *Orchestrator) sshCommandAttempts(node string, command string) ([][]string, []string, []string, []string) {
 	host := node
 	if mapped, ok := o.cfg.SSHNodeHosts[node]; ok && strings.TrimSpace(mapped) != "" {
 		host = strings.TrimSpace(mapped)
@@ -240,34 +315,7 @@ func (o *Orchestrator) sshWithTimeout(ctx context.Context, node string, command
 	direct = append(direct, target, command)
 	attempts = append(attempts, direct)
 	attemptNames = append(attemptNames, "direct")
-
+	return attempts, attemptNames, knownHostsFiles, repairHosts
-	var lastOut string
-	var lastErr error
-	for i, args := range attempts {
-		out, err := o.run(ctx, timeout, "ssh", args...)
-		if err == nil {
-			if i > 0 {
-				o.log.Printf("warning: ssh %s path failed for %s, using %s path", attemptNames[i-1], node, attemptNames[i])
-			}
-			return out, nil
-		}
-		if sshutil.ShouldAttemptKnownHostsRepair(out, err) {
-			o.log.Printf("warning: ssh failure on %s via %s path may be host-key related; repairing known_hosts and retrying once", node, attemptNames[i])
-			sshutil.RepairKnownHosts(ctx, o.log, knownHostsFiles, repairHosts, o.cfg.SSHPort)
-			retryOut, retryErr := o.run(ctx, timeout, "ssh", args...)
-			if retryErr == nil {
-				return retryOut, nil
-			}
-			out = retryOut
-			err = retryErr
-		}
-		lastOut = out
-		lastErr = err
-		if i < len(attempts)-1 {
-			o.log.Printf("warning: ssh %s path failed for %s: %v; trying %s path", attemptNames[i], node, err, attemptNames[i+1])
-		}
-	}
-	return lastOut, lastErr
 }
 
 // run runs one orchestration or CLI step.
@@ -308,6 +356,31 @@ func (o *Orchestrator) runSensitive(ctx context.Context, timeout time.Duration,
 	return trimmed, nil
 }
 
+// runSensitiveWithInput runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) runSensitiveWithInput(ctx context.Context, timeout time.Duration, input string, name string, args ...string) (string, error).
+// Why: sudo passwords and similar secret stdin must not be passed through
+// ordinary command arguments or dry-run logging.
+func (o *Orchestrator) runSensitiveWithInput(ctx context.Context, timeout time.Duration, input string, name string, args ...string) (string, error) {
+	runCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(runCtx, name, args...)
+	cmd.Env = os.Environ()
+	if o.runner.Kubeconfig != "" {
+		cmd.Env = append(cmd.Env, "KUBECONFIG="+o.runner.Kubeconfig)
+	}
+	cmd.Stdin = strings.NewReader(input)
+	out, err := cmd.CombinedOutput()
+	trimmed := strings.TrimSpace(string(out))
+	if err != nil {
+		if trimmed == "" {
+			return "", fmt.Errorf("%s failed: %w", name, err)
+		}
+		return trimmed, fmt.Errorf("%s failed: %w", name, err)
+	}
+	return trimmed, nil
+}
+
 // lines runs one orchestration or CLI step.
 // Signature: lines(in string) []string.
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

internal/cluster/orchestrator_quality_gate_closeout_test.go

@@ -0,0 +1,444 @@
+package cluster
+
+import (
+	"bufio"
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+	"scm.bstein.dev/bstein/ananke/internal/execx"
+	"scm.bstein.dev/bstein/ananke/internal/state"
+)
+
+// TestTCPServiceChecklistAndBackendHealBranches runs one orchestration or CLI step.
+// Signature: TestTCPServiceChecklistAndBackendHealBranches(t *testing.T).
+// Why: TCP health checks drive Mailu recovery paths and need branch coverage for
+// banner matching, unnamed checks, duplicate backend suppression, and heal notes.
+func TestTCPServiceChecklistAndBackendHealBranches(t *testing.T) {
+	addr := startTCPFixture(t, "220 ready\r\n", "250 pong\r\n")
+	host, portText, err := net.SplitHostPort(addr)
+	if err != nil {
+		t.Fatalf("split tcp fixture address: %v", err)
+	}
+	var port int
+	if _, err := fmt.Sscanf(portText, "%d", &port); err != nil {
+		t.Fatalf("parse tcp fixture port: %v", err)
+	}
+
+	cfg := config.Config{
+		Startup: config.Startup{
+			TCPServiceChecklist: []config.TCPServiceChecklistCheck{
+				{Name: "smtp", Host: host, Port: port, Send: "PING\r\n", ExpectContains: "pong", Namespace: "mail", Service: "mailu"},
+			},
+		},
+	}
+	orch := buildOrchestratorWithStubs(t, cfg, nil)
+	ok, detail := orch.tcpServiceChecklistReady(context.Background())
+	if !ok || detail != "tcp-checks=1" {
+		t.Fatalf("expected tcp checklist success, ok=%v detail=%q", ok, detail)
+	}
+
+	unnamed := config.TCPServiceChecklistCheck{Host: host, Port: port, ExpectContains: "missing"}
+	ok, detail = orch.tcpServiceCheckReady(context.Background(), unnamed)
+	if ok || !strings.Contains(detail, `banner missing expected marker "missing"`) {
+		t.Fatalf("expected missing marker detail, ok=%v detail=%q", ok, detail)
+	}
+	orch.cfg.Startup.TCPServiceChecklist = []config.TCPServiceChecklistCheck{unnamed}
+	ok, detail = orch.tcpServiceChecklistReady(context.Background())
+	if ok || !strings.Contains(detail, host+":"+portText) {
+		t.Fatalf("expected unnamed tcp check to use host:port detail, ok=%v detail=%q", ok, detail)
+	}
+
+	healAddr := startTCPFixture(t, "220 ready\r\n", "250 pong\r\n")
+	healHost, healPortText, _ := net.SplitHostPort(healAddr)
+	var healPort int
+	_, _ = fmt.Sscanf(healPortText, "%d", &healPort)
+	cfg.Startup.TCPServiceChecklist = []config.TCPServiceChecklistCheck{
+		{Host: healHost, Port: healPort, Namespace: "", Service: "skip"},
+		{Name: "ok", Host: healHost, Port: healPort, ExpectContains: "ready", Namespace: "mail", Service: "mailu"},
+		{Name: "bad", Host: healHost, Port: healPort, ExpectContains: "absent", Namespace: "mail", Service: "mailu"},
+		{Name: "dup", Host: healHost, Port: healPort, ExpectContains: "absent", Namespace: "mail", Service: "mailu"},
+	}
+	var scaled, rolled bool
+	orch = buildOrchestratorWithStubs(t, cfg, []commandStub{
+		{match: func(name string, args []string) bool {
+			joined := strings.Join(args, " ")
+			if name == "kubectl" && strings.Contains(joined, "-n mail scale deployment mailu --replicas=1") {
+				scaled = true
+				return true
+			}
+			if name == "kubectl" && strings.Contains(joined, "-n mail rollout status deployment/mailu") {
+				rolled = true
+				return true
+			}
+			return false
+		}},
+		{match: matchContains("kubectl", "-n", "mail", "scale", "statefulset", "mailu"), err: errors.New(`Error from server (NotFound): statefulsets.apps "mailu" not found`)},
+	})
+	healed, err := orch.healFailedTCPServiceBackends(context.Background())
+	if err != nil {
+		t.Fatalf("healFailedTCPServiceBackends: %v", err)
+	}
+	if len(healed) != 1 || healed[0] != "mail/deployment/mailu" || !scaled || !rolled {
+		t.Fatalf("unexpected tcp backend heal result healed=%v scaled=%v rolled=%v", healed, scaled, rolled)
+	}
+	lastAttempt := time.Now().Add(-time.Minute)
+	orch.maybeAutoHealTCPServiceBackends(context.Background(), &lastAttempt)
+}
+
+// TestHostPrivilegeFailureAndSecretBranches runs one orchestration or CLI step.
+// Signature: TestHostPrivilegeFailureAndSecretBranches(t *testing.T).
+// Why: host repair depends on stable allowlist, secret, sudo, and failure
+// classes before Ananke can safely restart or reboot managed nodes.
+func TestHostPrivilegeFailureAndSecretBranches(t *testing.T) {
+	cfg := config.Config{
+		SSHManagedNodes: []string{"titan-23"},
+		SSHUser:         "atlas",
+		Startup: config.Startup{
+			HostSudoSecretNamespace:    "host-secrets",
+			HostSudoSecretNameTemplate: "sudo-{node}",
+			HostSudoSecretPasswordKey:  "password",
+		},
+	}
+	orch := buildOrchestratorWithStubs(t, cfg, nil)
+	if _, err := orch.runHostPrivilegedAction(context.Background(), " ", hostActionSudoPreflight, 0); err == nil {
+		t.Fatalf("expected empty node privilege error")
+	}
+	if _, err := orch.runHostPrivilegedAction(context.Background(), "unmanaged", hostActionSudoPreflight, 0); err == nil {
+		t.Fatalf("expected unmanaged node privilege error")
+	}
+	if _, err := orch.runHostPrivilegedAction(context.Background(), "titan-23", hostPrivilegedAction("bad-action"), 0); err == nil {
+		t.Fatalf("expected bad action privilege error")
+	}
+	if got := orch.hostPrivilegedCommandTimeout(); got != 90*time.Second {
+		t.Fatalf("expected default privileged timeout, got %s", got)
+	}
+	orch.cfg.Startup.HostPrivilegedCommandTimeoutSec = 7
+	if got := orch.hostPrivilegedCommandTimeout(); got != 7*time.Second {
+		t.Fatalf("expected configured privileged timeout, got %s", got)
+	}
+
+	actions := []struct {
+		action hostPrivilegedAction
+		args   []string
+		want   string
+	}{
+		{hostActionHostReboot, nil, "systemctl reboot"},
+		{hostActionModprobeDMCrypt, nil, "modprobe dm_crypt"},
+		{hostActionCrictlPods, nil, "crictl pods -o json"},
+		{hostActionCrictlPs, nil, "crictl ps -a -o json"},
+		{hostActionCrictlInspect, []string{"abc123"}, "crictl inspect abc123"},
+		{hostActionCrictlStop, []string{"abc123"}, "crictl stop abc123"},
+	}
+	for _, tc := range actions {
+		got, err := hostPrivilegedCommandArgs(tc.action, tc.args...)
+		if err != nil {
+			t.Fatalf("hostPrivilegedCommandArgs(%s): %v", tc.action, err)
+		}
+		if strings.Join(got, " ") != tc.want {
+			t.Fatalf("hostPrivilegedCommandArgs(%s)=%q want %q", tc.action, strings.Join(got, " "), tc.want)
+		}
+	}
+	for _, tc := range []struct {
+		action hostPrivilegedAction
+		arg    string
+	}{
+		{hostActionCrictlInspect, "bad;id"},
+		{hostActionCrictlStop, ""},
+	} {
+		if _, err := hostPrivilegedCommandArgs(tc.action, tc.arg); err == nil {
+			t.Fatalf("expected unsafe arg rejection for %s", tc.action)
+		}
+	}
+
+	errs := []error{
+		classifyHostPrivilegeFailure("n", hostActionSudoPreflight, context.DeadlineExceeded, ""),
+		classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("try again"), ""),
+		classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("permission denied publickey"), ""),
+		classifyHostPrivilegeFailure("n", hostActionSudoPreflight, errors.New("exit 1"), "line one\nline two"),
+	}
+	wantClasses := []string{"host-command-timeout", "host-privilege-auth-failed", "host-ssh-unavailable", "host-command-failed"}
+	for i, err := range errs {
+		if !strings.Contains(err.Error(), wantClasses[i]) {
+			t.Fatalf("expected %s in %q", wantClasses[i], err)
+		}
+	}
+
+	secretJSON := `{"data":{"password":"` + base64.StdEncoding.EncodeToString([]byte("pw\n")) + `"}}`
+	orch = buildOrchestratorWithStubs(t, cfg, []commandStub{
+		{match: matchContains("ssh", "sudo -n /usr/bin/systemctl --version"), out: "sudo: a password is required", err: errors.New("sudo failed")},
+		{match: matchContains("kubectl", "-n", "host-secrets", "get", "secret", "sudo-titan-23"), out: secretJSON},
+	})
+	orch.SetSSHInputOverride(func(_ context.Context, _ time.Duration, node, command, input string) (string, error) {
+		if node != "titan-23" || !strings.Contains(command, "sudo -S") || input != "pw\n" {
+			t.Fatalf("unexpected password-backed ssh node=%q command=%q input=%q", node, command, input)
+		}
+		return "incorrect password", errors.New("try again")
+	})
+	if _, err := orch.runHostPrivilegedAction(context.Background(), "titan-23", hostActionSudoPreflight, 0); err == nil || !strings.Contains(err.Error(), "host-privilege-auth-failed") {
+		t.Fatalf("expected password-backed auth failure, got %v", err)
+	}
+
+	secretCases := []struct {
+		name  string
+		cfg   config.Config
+		node  string
+		out   string
+		err   error
+		class string
+	}{
+		{name: "unconfigured", cfg: config.Config{}, class: "secret-lookup-unconfigured"},
+		{name: "empty-name", cfg: config.Config{Startup: config.Startup{HostSudoSecretNamespace: "ns", HostSudoSecretNameTemplate: "{node}"}}, node: " ", class: "secret-lookup-invalid"},
+		{name: "query-error", cfg: cfg, err: errors.New("denied"), class: "secret-lookup-failed"},
+		{name: "bad-json", cfg: cfg, out: "{", class: "secret-decode-failed"},
+		{name: "missing-key", cfg: cfg, out: `{"data":{"other":"cHc="}}`, class: "secret-key-missing"},
+		{name: "bad-base64", cfg: cfg, out: `{"data":{"password":"%"}}`, class: "secret-decode-failed"},
+		{name: "empty-password", cfg: cfg, out: `{"data":{"password":"Cg=="}}`, class: "secret-empty"},
+	}
+	for _, tc := range secretCases {
+		orch := buildOrchestratorWithStubs(t, tc.cfg, []commandStub{
+			{match: matchContains("kubectl", "get", "secret"), out: tc.out, err: tc.err},
+		})
+		node := tc.node
+		if node == "" {
+			node = "titan-23"
+		}
+		_, class, err := orch.hostSudoPassword(context.Background(), node)
+		if err == nil || class != tc.class {
+			t.Fatalf("%s: expected class %q error, got class=%q err=%v", tc.name, tc.class, class, err)
+		}
+	}
+}
+
+// TestImagePullDNSAndCredentialBranches runs one orchestration or CLI step.
+// Signature: TestImagePullDNSAndCredentialBranches(t *testing.T).
+// Why: image-pull blockers should be classified instead of blindly recycling
+// pods when DNS or registry credentials are the real outage cause.
+func TestImagePullDNSAndCredentialBranches(t *testing.T) {
+	eventJSON := `{"items":[
+{"type":"Normal","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"skip","name":"normal"},"message":"lookup registry.bstein.dev no such host"},
+{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Service","namespace":"skip","name":"svc"},"message":"lookup registry.bstein.dev no such host"},
+{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"","name":""},"message":"lookup registry.bstein.dev no such host"},
+{"type":"Warning","reason":"Failed","involvedObject":{"kind":"Pod","namespace":"media","name":"app"},"message":"failed to pull image: lookup registry.bstein.dev: no such host"},
+{"type":"Warning","reason":"FailedPull","metadata":{"namespace":"finance"},"involvedObject":{"kind":"Pod","name":"budget"},"message":"unauthorized: authentication required"},
+{"type":"Warning","reason":"FailedToRetrieveImagePullSecret","involvedObject":{"kind":"Pod","namespace":"sso","name":"keycloak"},"message":"unable to retrieve some image pull secrets"}
+]}`
+	eventJSON, podsJSON := currentCredentialFixture(t, eventJSON)
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: podsJSON},
+		{match: matchContains("kubectl", "get", "events", "-A"), out: eventJSON},
+	})
+	dnsReasons, err := orch.imagePullDNSBlockerReasons(context.Background())
+	if err != nil {
+		t.Fatalf("imagePullDNSBlockerReasons: %v", err)
+	}
+	if dnsReasons["media/app"] != "ImagePullDNSBlocker:registry.bstein.dev" {
+		t.Fatalf("unexpected DNS reasons: %#v", dnsReasons)
+	}
+	credentialReasons, err := orch.imagePullCredentialBlockerReasons(context.Background())
+	if err != nil {
+		t.Fatalf("imagePullCredentialBlockerReasons: %v", err)
+	}
+	if credentialReasons["finance/budget"] != "ImagePullCredentialBlocker:unauthorized" || credentialReasons["sso/keycloak"] != "ImagePullCredentialBlocker:missing-pull-secret" {
+		t.Fatalf("unexpected credential reasons: %#v", credentialReasons)
+	}
+	for _, message := range []string{
+		"invalid username/password",
+		"403 forbidden",
+		"pull access denied",
+		"registry returned something strange",
+	} {
+		if got := imagePullCredentialFailureClass("Failed", message); got == "" {
+			t.Fatalf("expected credential class for %q", message)
+		}
+	}
+	if !vaultSyncDeployment("worker", map[string]string{"app": "secret-vault-sync"}) {
+		t.Fatalf("expected vault-sync label value to identify sync deployment")
+	}
+	for _, message := range []string{"no resolver marker here", "lookup ", "lookup []"} {
+		if got := imagePullRegistryHost(message); got != "unknown-registry" {
+			t.Fatalf("expected unknown registry for %q, got %q", message, got)
+		}
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "finance", "get", "deployment"), out: `{"items":[{"metadata":{"name":"finance-vault-sync","labels":{}}},{"metadata":{"name":"app","labels":{}}}]}`},
+		{match: matchContains("kubectl", "-n", "finance", "rollout", "restart", "deployment", "finance-vault-sync"), out: ""},
+		{match: matchContains("kubectl", "-n", "finance", "rollout", "status", "deployment/finance-vault-sync"), out: ""},
+	})
+	repaired, err := orch.restartVaultSyncDeployments(context.Background(), "finance")
+	if err != nil || len(repaired) != 1 || repaired[0] != "finance/deployment/finance-vault-sync" {
+		t.Fatalf("unexpected vault-sync repair repaired=%v err=%v", repaired, err)
+	}
+	for _, tc := range []struct {
+		name string
+		out  string
+		err  error
+	}{
+		{name: "query", err: errors.New("api down")},
+		{name: "decode", out: "{"},
+		{name: "missing", out: `{"items":[{"metadata":{"name":"app","labels":{}}}]}`},
+		{name: "restart", out: `{"items":[{"metadata":{"name":"vault-sync","labels":{}}}]}`, err: errors.New("restart denied")},
+	} {
+		orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+			{match: matchContains("kubectl", "-n", "ns", "get", "deployment"), out: tc.out, err: mapRestartErr(tc.name, "query", tc.err)},
+			{match: matchContains("kubectl", "-n", "ns", "rollout", "restart"), err: mapRestartErr(tc.name, "restart", tc.err)},
+		})
+		if _, err := orch.restartVaultSyncDeployments(context.Background(), "ns"); err == nil {
+			t.Fatalf("%s: expected restartVaultSyncDeployments error", tc.name)
+		}
+	}
+}
+
+// TestLonghornSelectorAndRuntimeRecoveryBranches runs one orchestration or CLI step.
+// Signature: TestLonghornSelectorAndRuntimeRecoveryBranches(t *testing.T).
+// Why: Longhorn manager drift and kubelet proxy recovery are safety-critical
+// repair paths that should stay covered without a live cluster.
+func TestLonghornSelectorAndRuntimeRecoveryBranches(t *testing.T) {
+	if selectorLabelString(map[string]string{"b": "2", "a": "1"}) != "a=1,b=2" {
+		t.Fatalf("selector labels should be sorted")
+	}
+	if podMatchesLabels(nil, map[string]string{"app": "longhorn"}) {
+		t.Fatalf("nil pod labels must not match selector")
+	}
+	if len(missingSelectorLabels(map[string]string{"app": "old"}, map[string]string{"app": "longhorn"})) != 1 {
+		t.Fatalf("expected missing selector label")
+	}
+
+	cfg := config.Config{}
+	var labeled bool
+	orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[
+{"metadata":{"name":""}},
+{"metadata":{"name":"missing-kube"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
+{"metadata":{"name":"has-manager"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
+{"metadata":{"name":"ready"},"status":{"conditions":[{"type":"Ready","status":"True","reason":"ManagerPodMissing"}]}},
+{"metadata":{"name":"other-reason"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"DiskPressure"}]}},
+{"metadata":{"name":"already-labelled"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}},
+{"metadata":{"name":"repair"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}
+]}`},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true","skip":""}}}}`},
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[
+{"metadata":{"name":"has-manager","labels":{}}},
+{"metadata":{"name":"ready","labels":{}}},
+{"metadata":{"name":"other-reason","labels":{}}},
+{"metadata":{"name":"already-labelled","labels":{"longhorn-host":"true"}}},
+{"metadata":{"name":"repair","labels":{}}}
+]}`},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json", "-l", "longhorn-host=true"), out: `{"items":[{"metadata":{"labels":{"longhorn-host":"true"}},"spec":{"nodeName":"has-manager"}},{"metadata":{"labels":{"longhorn-host":"false"}},"spec":{"nodeName":"repair"}},{"metadata":{"labels":{"longhorn-host":"true"}},"spec":{"nodeName":""}}]}`},
+		{match: func(name string, args []string) bool {
+			if name == "kubectl" && strings.Contains(strings.Join(args, " "), "label node repair --overwrite longhorn-host=true") {
+				labeled = true
+				return true
+			}
+			return false
+		}, out: ""},
+	})
+	repaired, err := orch.reconcileLonghornKubernetesReadiness(context.Background())
+	if err != nil || repaired != 1 || !labeled {
+		t.Fatalf("unexpected longhorn reconcile repaired=%d labeled=%v err=%v", repaired, labeled, err)
+	}
+
+	errOrch := buildOrchestratorWithStubs(t, cfg, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[{"metadata":{"name":"repair"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{"longhorn-host":"true"}}}}`},
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"repair","labels":{}}}]}`},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json"), out: `{"items":[]}`},
+		{match: matchContains("kubectl", "label", "node", "repair"), err: errors.New("label denied")},
+	})
+	if repaired, err := errOrch.reconcileLonghornKubernetesReadiness(context.Background()); err == nil || repaired != 0 {
+		t.Fatalf("expected longhorn label error, repaired=%d err=%v", repaired, err)
+	}
+
+	runtimeCfg := config.Config{
+		SSHManagedNodes: []string{"titan-23"},
+		SSHUser:         "atlas",
+		Startup: config.Startup{
+			NodeRuntimeRestartWaitSeconds: 1,
+			NodeRuntimeRebootWaitSeconds:  1,
+			HostRepairAllowReboot:         true,
+		},
+	}
+	orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
+		{match: matchContains("ssh", "sudo -n systemctl --no-block restart k3s-agent"), out: ""},
+		{match: matchContains("kubectl", "wait", "node/titan-23"), err: errors.New("not ready")},
+		{match: matchContains("ssh", "sudo -n systemctl show k3s-agent"), out: "ActiveState=deactivating\nSubState=stop-sigterm"},
+		{match: matchContains("ssh", "sudo -n systemctl reboot"), out: ""},
+	})
+	if err := orch.recoverManagedNodeRuntime(context.Background(), "titan-23", true, "proxy broken"); err == nil || !strings.Contains(err.Error(), "node did not recover after controlled reboot") {
+		t.Fatalf("expected reboot wait failure, got %v", err)
+	}
+	if err := orch.recoverManagedNodeRuntime(context.Background(), " ", true, ""); err == nil {
+		t.Fatalf("expected empty runtime node error")
+	}
+	if !k3sAgentStateRequiresReboot("Result=timeout") || k3sAgentStateRequiresReboot("ActiveState=active") {
+		t.Fatalf("unexpected k3s-agent reboot predicate")
+	}
+	if summarizeSystemctlShow("") != "unknown" || !strings.Contains(summarizeSystemctlShow("ActiveState=activating\nSubState=start"), "ActiveState") {
+		t.Fatalf("unexpected systemctl summary")
+	}
+}
+
+// startTCPFixture runs one orchestration or CLI step.
+// Signature: startTCPFixture(t *testing.T, banner string, reply string) string.
+// Why: TCP checklist tests need a local loopback endpoint that emits the same
+// banner/response shape as SMTP-like protocol probes.
+func startTCPFixture(t *testing.T, banner string, reply string) string {
+	t.Helper()
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatalf("listen tcp fixture: %v", err)
+	}
+	t.Cleanup(func() { _ = listener.Close() })
+	go func() {
+		for {
+			conn, err := listener.Accept()
+			if err != nil {
+				return
+			}
+			go func() {
+				defer conn.Close()
+				_, _ = conn.Write([]byte(banner))
+				_ = conn.SetDeadline(time.Now().Add(2 * time.Second))
+				_, _ = bufio.NewReader(conn).ReadString('\n')
+				_, _ = conn.Write([]byte(reply))
+			}()
+		}
+	}()
+	return listener.Addr().String()
+}
+
+// mapRestartErr runs one orchestration or CLI step.
+// Signature: mapRestartErr(caseName, target string, err error) error.
+// Why: table-driven deployment restart tests need concise error injection
+// without repeating one-off closures for each command phase.
+func mapRestartErr(caseName, target string, err error) error {
+	if caseName == target {
+		return err
+	}
+	return nil
+}
+
+// TestLocalOrchestratorConstructorUsesQuietLogger runs one orchestration or CLI step.
+// Signature: TestLocalOrchestratorConstructorUsesQuietLogger(t *testing.T).
+// Why: keep imports in this closeout file anchored to real construction helpers
+// while preserving package-local test readability.
+func TestLocalOrchestratorConstructorUsesQuietLogger(t *testing.T) {
+	orch := &Orchestrator{
+		cfg:    config.Config{},
+		runner: &execx.Runner{},
+		store:  state.New(t.TempDir() + "/runs.json"),
+		log:    log.New(io.Discard, "", 0),
+	}
+	if orch.log == nil || orch.runner == nil || orch.store == nil {
+		t.Fatalf("expected constructed orchestrator test fixture")
+	}
+}

internal/cluster/orchestrator_quality_gate_final_closeout_test.go

@@ -0,0 +1,214 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestFinalCoverageLeaseDrainAndSSHBranches runs one orchestration or CLI step.
+// Signature: TestFinalCoverageLeaseDrainAndSSHBranches(t *testing.T).
+// Why: close the last local-only safety branches for cordon release, Longhorn
+// host preflight errors, and stdin-backed SSH known-host retry.
+func TestFinalCoverageLeaseDrainAndSSHBranches(t *testing.T) {
+	ctx := context.Background()
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), err: errors.New("api down")},
+	})
+	if _, err := orch.ensureLonghornEncryptedHostPrereqs(ctx, []string{"worker-a"}); err == nil || !strings.Contains(err.Error(), "api down") {
+		t.Fatalf("expected longhorn host query error, got %v", err)
+	}
+
+	future := time.Now().UTC().Add(time.Hour).Format(time.RFC3339)
+	nodesJSON := fmt.Sprintf(`{"items":[
+{"metadata":{"name":"clear-fails","annotations":{%q:"ananke"}},"spec":{"unschedulable":false}},
+{"metadata":{"name":"crypt-pending","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}}
+]}`,
+		anankeCordonOwnerAnnotation,
+		anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonMissingCryptsetup, anankeCordonDeadlineAnnotation, future)
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: nodesJSON},
+		{match: matchContains("kubectl", "annotate", "node", "clear-fails"), err: errors.New("clear denied")},
+		{match: matchContains("ssh", "crypt-pending", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")},
+	})
+	if released, err := orch.enforceRecoveryCordonLeases(ctx); released != 0 || err == nil || !strings.Contains(err.Error(), "clear denied") {
+		t.Fatalf("expected clear-cordon error and pending repair log, released=%d err=%v", released, err)
+	}
+
+	releaseErrOrch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("ssh", "crypt-release", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_PRESENT__"},
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: `{"items":[]}`},
+		{match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`},
+		{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/proxy-release/proxy/healthz"), out: "ok"},
+		{match: matchContains("kubectl", "uncordon"), err: errors.New("uncordon denied")},
+	})
+	for _, tc := range []struct {
+		node   string
+		reason string
+	}{
+		{"crypt-release", cordonReasonMissingCryptsetup},
+		{"runtime-release", cordonReasonRuntimeWedge},
+		{"proxy-release", cordonReasonKubeletProxy},
+	} {
+		if recovered, err := releaseErrOrch.recoverLeasedCordon(ctx, tc.node, map[string]string{anankeCordonReasonAnnotation: tc.reason}); recovered || err == nil || !strings.Contains(err.Error(), "uncordon denied") {
+			t.Fatalf("expected %s uncordon denial, recovered=%v err=%v", tc.reason, recovered, err)
+		}
+	}
+
+	binDir := t.TempDir()
+	countPath := filepath.Join(t.TempDir(), "ssh-count")
+	writeTestExecutable(t, filepath.Join(binDir, "ssh"), `#!/bin/sh
+count="${ANANKE_FAKE_SSH_COUNT:?}"
+n=0
+if [ -f "$count" ]; then n="$(cat "$count")"; fi
+n=$((n + 1))
+printf '%s\n' "$n" > "$count"
+if [ "$n" -eq 1 ]; then
+  printf '%s\n' 'Host key verification failed.' >&2
+  exit 255
+fi
+cat >/dev/null
+printf '%s\n' 'stdin-ok'
+`)
+	writeTestExecutable(t, filepath.Join(binDir, "ssh-keygen"), "#!/bin/sh\nexit 0\n")
+	t.Setenv("ANANKE_FAKE_SSH_COUNT", countPath)
+	t.Setenv("HOME", t.TempDir())
+	t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
+
+	sshDir := t.TempDir()
+	orch = buildOrchestratorWithStubs(t, config.Config{
+		SSHPort:         2222,
+		SSHUser:         "atlas",
+		SSHConfigFile:   filepath.Join(sshDir, "config"),
+		SSHIdentityFile: filepath.Join(sshDir, "id_ed25519"),
+		SSHNodeHosts:    map[string]string{"node-a": "node-a.local"},
+	}, nil)
+	out, err := orch.sshWithInput(ctx, "node-a", "sudo -S true", "pw\n", time.Second)
+	if err != nil || out != "stdin-ok" {
+		t.Fatalf("expected known-host retry success, out=%q err=%v", out, err)
+	}
+	if raw, err := os.ReadFile(countPath); err != nil || strings.TrimSpace(string(raw)) != "2" {
+		t.Fatalf("expected exactly two fake ssh attempts, count=%q err=%v", strings.TrimSpace(string(raw)), err)
+	}
+}
+
+// TestPostStartAutoHealRequestsReconcileAfterImagePullRepair runs one orchestration or CLI step.
+// Signature: TestPostStartAutoHealRequestsReconcileAfterImagePullRepair(t *testing.T).
+// Why: a successful image-pull credential repair should request a Flux reconcile
+// without requiring any real Kubernetes or Flux command.
+func TestPostStartAutoHealRequestsReconcileAfterImagePullRepair(t *testing.T) {
+	ctx := context.Background()
+	eventsJSON := `{"items":[{"type":"Warning","reason":"FailedPull","message":"unauthorized: authentication required","involvedObject":{"kind":"Pod","namespace":"apps","name":"web"}}]}`
+	deploymentsJSON := `{"items":[{"metadata":{"name":"vault-sync","labels":{"app":"vault-sync"}}}]}`
+	eventsJSON, credentialPods := currentCredentialFixture(t, eventsJSON)
+	sawFluxSourceReconcile := false
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[]}`},
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[]}`},
+		{match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"},
+		{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: eventsJSON},
+		{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: credentialPods},
+		{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "-o", "json"), out: deploymentsJSON},
+		{match: matchContains("kubectl", "-n", "apps", "rollout", "restart", "deployment", "vault-sync"), out: ""},
+		{match: matchContains("kubectl", "-n", "apps", "rollout", "status", "deployment/vault-sync"), out: ""},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "flux-system", "annotate", "gitrepository", "flux-system")(name, args) {
+					return false
+				}
+				sawFluxSourceReconcile = true
+				return true
+			},
+		},
+		{match: matchContains("kubectl", "annotate"), out: ""},
+	})
+	if err := orch.postStartAutoHeal(ctx); err != nil {
+		t.Fatalf("expected post-start image-pull repair success, got %v", err)
+	}
+	if !sawFluxSourceReconcile {
+		t.Fatalf("expected post-start auto-heal to request flux source reconcile")
+	}
+}
+
+// TestStartupQuarantinesRuntimeWedgeBeforeSSHAuthGate runs one orchestration or CLI step.
+// Signature: TestStartupQuarantinesRuntimeWedgeBeforeSSHAuthGate(t *testing.T).
+// Why: startup should quarantine proven runtime-wedged workers before later SSH
+// gates so bad schedulers are removed from the active worker set.
+func TestStartupQuarantinesRuntimeWedgeBeforeSSHAuthGate(t *testing.T) {
+	ctx := context.Background()
+	tmpDir := t.TempDir()
+	old := time.Now().Add(-10 * time.Minute).Format(time.RFC3339)
+	now := time.Now().Format(time.RFC3339)
+	podsJSON := fmt.Sprintf(`{"items":[
+{"metadata":{"namespace":"apps","name":"wedged-a","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"worker-bad"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CreateContainerError"}}}]}},
+{"metadata":{"namespace":"apps","name":"wedged-b","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"worker-bad"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"RunContainerError"}}}]}}
+]}`, old, old)
+	eventsJSON := fmt.Sprintf(`{"items":[
+{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"wedged-a"},"message":"context deadline exceeded while creating container"},
+{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"wedged-b"},"message":"failed to reserve container name"}
+]}`, now, now)
+	cfg := config.Config{
+		Workers:         []string{"worker-good", "worker-bad"},
+		SSHManagedNodes: []string{"worker-good", "worker-bad"},
+		SSHNodeHosts: map[string]string{
+			"worker-good": "worker-good.local",
+			"worker-bad":  "worker-bad.local",
+		},
+		SSHUser: "atlas",
+		SSHPort: 22,
+		Startup: config.Startup{
+			APIWaitSeconds:              1,
+			APIPollSeconds:              1,
+			RequireTimeSync:             false,
+			RequireNodeInventoryReach:   false,
+			RequireNodeSSHAuth:          true,
+			ReconcileAccessOnBoot:       false,
+			AutoEtcdRestoreOnAPIFailure: false,
+			StuckPodGraceSeconds:        1,
+		},
+		State: config.State{
+			Dir:            tmpDir,
+			ReportsDir:     filepath.Join(tmpDir, "reports"),
+			RunHistoryPath: filepath.Join(tmpDir, "runs.json"),
+			LockPath:       filepath.Join(tmpDir, "ananke.lock"),
+			IntentPath:     filepath.Join(tmpDir, "intent.json"),
+		},
+	}
+	orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
+		{match: matchContains("kubectl", "version", "--request-timeout=5s"), out: "ok"},
+		{match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"},
+		{match: matchContains("kubectl", "-n", "flux-system", "get", "gitrepository", "flux-system", "-o", "jsonpath={.spec.url}"), out: ""},
+		{match: matchContains("kubectl", "-n", "flux-system", "get", "gitrepository", "flux-system", "-o", "jsonpath={.spec.ref.branch}"), out: ""},
+		{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: podsJSON},
+		{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: eventsJSON},
+		{match: matchContains("kubectl", "annotate", "node", "worker-bad"), out: ""},
+		{match: matchContains("kubectl", "cordon", "worker-bad"), out: ""},
+		{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: ""},
+		{match: matchContains("ssh", "worker-good", "echo __ANANKE_SSH_AUTH_OK__"), err: errors.New("permission denied publickey")},
+	})
+	err := orch.Startup(ctx, StartupOptions{Reason: "runtime quarantine coverage"})
+	if err == nil || !strings.Contains(err.Error(), "ssh auth gate failed") {
+		t.Fatalf("expected controlled ssh auth gate failure after quarantine, got %v", err)
+	}
+	if !containsNode(orch.cfg.Startup.IgnoreUnavailableNodes, "worker-bad") {
+		t.Fatalf("expected runtime-wedged worker to be added to ignore_unavailable_nodes, got %v", orch.cfg.Startup.IgnoreUnavailableNodes)
+	}
+}
+
+// writeTestExecutable writes a temporary command fixture for tests.
+// Signature: writeTestExecutable(t *testing.T, path string, body string).
+// Why: fake ssh/ssh-keygen binaries let retry paths run without touching host
+// SSH state or invoking the real cluster operator tooling.
+func writeTestExecutable(t *testing.T, path string, body string) {
+	t.Helper()
+	if err := os.WriteFile(path, []byte(body), 0o755); err != nil {
+		t.Fatalf("write test executable %s: %v", path, err)
+	}
+}

internal/cluster/orchestrator_quality_gate_more_closeout_test.go

@@ -0,0 +1,480 @@
+package cluster
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestCordonLeaseEnforcementRecoveryCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestCordonLeaseEnforcementRecoveryCloseoutBranches(t *testing.T).
+// Why: lease enforcement must release recovered Ananke cordons while escalating
+// stale or unknown cordons without overriding operator-owned state.
+func TestCordonLeaseEnforcementRecoveryCloseoutBranches(t *testing.T) {
+	now := time.Now().UTC()
+	future := now.Add(time.Hour).Format(time.RFC3339)
+	past := now.Add(-time.Hour).Format(time.RFC3339)
+	oldTaint := now.Add(-2 * time.Hour).Format(time.RFC3339)
+	nodesJSON := fmt.Sprintf(`{"items":[
+{"metadata":{"name":"","annotations":{}}},
+{"metadata":{"name":"uncordoned","annotations":{%q:"ananke"}},"spec":{"unschedulable":false}},
+{"metadata":{"name":"manual","annotations":{}},"spec":{"unschedulable":true,"taints":[{"key":"node.kubernetes.io/unschedulable","timeAdded":%q}]}},
+{"metadata":{"name":"crypt","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}},
+{"metadata":{"name":"runtime","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}},
+{"metadata":{"name":"proxy","annotations":{%q:"ananke",%q:%q,%q:%q}},"spec":{"unschedulable":true}},
+{"metadata":{"name":"unknown","annotations":{%q:"ananke",%q:"mystery",%q:%q}},"spec":{"unschedulable":true}}
+]}`,
+		anankeCordonOwnerAnnotation,
+		oldTaint,
+		anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonMissingCryptsetup, anankeCordonDeadlineAnnotation, future,
+		anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonRuntimeWedge, anankeCordonDeadlineAnnotation, future,
+		anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, cordonReasonKubeletProxy, anankeCordonDeadlineAnnotation, future,
+		anankeCordonOwnerAnnotation, anankeCordonReasonAnnotation, anankeCordonDeadlineAnnotation, past)
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: nodesJSON},
+		{match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_PRESENT__"},
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: `{"items":[]}`},
+		{match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`},
+		{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/proxy/proxy/healthz"), out: "ok"},
+		{match: matchContains("kubectl", "uncordon"), out: ""},
+		{match: matchContains("kubectl", "annotate", "node"), out: ""},
+	})
+	released, err := orch.enforceRecoveryCordonLeases(context.Background())
+	if err == nil || released != 3 {
+		t.Fatalf("expected three recovered cordons plus manual-action error, released=%d err=%v", released, err)
+	}
+	if !strings.Contains(err.Error(), "manual action required") || !strings.Contains(err.Error(), "unknown") {
+		t.Fatalf("expected manual action details, got %v", err)
+	}
+}
+
+// TestCriticalEndpointServiceHealCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestCriticalEndpointServiceHealCloseoutBranches(t *testing.T).
+// Why: configured service backend repair should distinguish healthy endpoints,
+// missing endpoints, backend scale failures, and startup-probe repair success.
+func TestCriticalEndpointServiceHealCloseoutBranches(t *testing.T) {
+	workloadJSON := `{"spec":{"template":{"spec":{"containers":[{"name":"app","livenessProbe":{"httpGet":{"path":"/healthz"}}}]}}}}`
+	cfg := config.Config{Startup: config.Startup{
+		CriticalServiceEndpoints:          []string{"bad-entry", "ready/svc", "missing/svc", "zero/svc", "probe/svc", "error/svc"},
+		CriticalServiceStartupProbeRepair: true,
+	}}
+	orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
+		{match: matchContains("kubectl", "-n", "ready", "get", "endpoints", "svc"), out: `{"subsets":[{"addresses":[{"ip":"10.0.0.10"}]}]}`},
+		{match: matchContains("kubectl", "-n", "missing", "get", "endpoints", "svc"), err: errors.New(`Error from server (NotFound): endpoints "svc" not found`)},
+		{match: matchContains("kubectl", "-n", "zero", "get", "endpoints", "svc"), out: `{"subsets":[]}`},
+		{match: matchContains("kubectl", "-n", "probe", "get", "endpoints", "svc"), out: `{"subsets":[]}`},
+		{match: matchContains("kubectl", "-n", "error", "get", "endpoints", "svc"), err: errors.New("api down")},
+		{match: matchContains("kubectl", "-n", "zero", "scale", "deployment", "svc"), out: ""},
+		{match: matchContains("kubectl", "-n", "zero", "rollout", "status", "deployment/svc"), out: ""},
+		{match: matchContains("kubectl", "-n", "zero", "scale", "statefulset", "svc"), err: errors.New(`Error from server (NotFound): statefulsets.apps "svc" not found`)},
+		{match: matchContains("kubectl", "-n", "missing", "scale"), err: errors.New(`Error from server (NotFound): deployments.apps "svc" not found`)},
+		{match: matchContains("kubectl", "-n", "probe", "scale", "deployment", "svc"), err: errors.New("scale denied")},
+		{match: matchContains("kubectl", "-n", "probe", "get", "deployment", "svc"), out: workloadJSON},
+		{match: matchContains("kubectl", "-n", "probe", "patch", "deployment", "svc"), out: ""},
+		{match: matchContains("kubectl", "-n", "probe", "get", "statefulset", "svc"), err: errors.New(`Error from server (NotFound): statefulsets.apps "svc" not found`)},
+	})
+	healed, err := orch.healUnreadyConfiguredServiceBackends(context.Background())
+	if err == nil {
+		t.Fatalf("expected aggregated critical endpoint errors")
+	}
+	joined := strings.Join(healed, ",")
+	if !strings.Contains(joined, "zero/deployment/svc") || !strings.Contains(joined, "probe/deployment/svc") {
+		t.Fatalf("expected zero and probe repairs, healed=%v", healed)
+	}
+	if !strings.Contains(err.Error(), "bad-entry") || !strings.Contains(err.Error(), "error/svc") {
+		t.Fatalf("expected invalid and query errors, got %v", err)
+	}
+	if _, _, err := parseCriticalServiceEndpoint(" /svc"); err == nil {
+		t.Fatalf("expected empty namespace endpoint parse error")
+	}
+}
+
+// TestLonghornAndStaleRWOAdditionalCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestLonghornAndStaleRWOAdditionalCloseoutBranches(t *testing.T).
+// Why: Longhorn parser errors and stale RWO helper predicates protect storage
+// repair from false positives and should be covered without live objects.
+func TestLonghornAndStaleRWOAdditionalCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), err: errors.New(`Error from server (NotFound): nodes.longhorn.io not found`)},
+	})
+	if nodes, err := orch.queryLonghornNodes(ctx); err != nil || len(nodes.Items) != 0 {
+		t.Fatalf("expected missing Longhorn CRD to be empty, nodes=%v err=%v", nodes, err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: "{"},
+	})
+	if _, err := orch.queryLonghornNodes(ctx); err == nil {
+		t.Fatalf("expected longhorn node decode error")
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: "{"},
+	})
+	if _, err := orch.queryLonghornManagerDaemonSet(ctx); err == nil {
+		t.Fatalf("expected daemonset decode error")
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "pods", "-o", "json"), out: "{"},
+	})
+	if _, err := orch.longhornManagerPodsByNode(ctx, map[string]string{"app": "longhorn"}); err == nil {
+		t.Fatalf("expected manager pod decode error")
+	}
+	if cond := longhornConditionByType(longhornNode{}, "Ready"); cond != nil {
+		t.Fatalf("expected missing condition to return nil")
+	}
+
+	checkCalls := 0
+	orch = buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"titan-23"}}, nil)
+	orch.runOverride = func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
+		joined := strings.Join(args, " ")
+		if name == "ssh" && strings.Contains(joined, "command -v cryptsetup") {
+			checkCalls++
+			if checkCalls == 1 {
+				return "__ANANKE_CRYPTSETUP_MISSING__", errors.New("exit 41")
+			}
+			return "__ANANKE_CRYPTSETUP_PRESENT__", nil
+		}
+		if name == "ssh" && strings.Contains(joined, "apt-get update") {
+			return "installed but no marker", nil
+		}
+		return "", nil
+	}
+	orch.runSensitiveOverride = orch.runOverride
+	if err := orch.ensureHostCryptsetup(ctx, "titan-23"); err != nil {
+		t.Fatalf("expected cryptsetup verify success after install, got %v", err)
+	}
+	if checkCalls < 2 {
+		t.Fatalf("expected install verification check, calls=%d", checkCalls)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "pvc", "-A"), out: "{"},
+	})
+	if _, err := orch.queryPVCs(ctx); err == nil {
+		t.Fatalf("expected pvc decode error")
+	}
+	pvc := persistentVolumeClaimResource{}
+	pvc.Metadata.Namespace = "apps"
+	pvc.Metadata.Name = "data"
+	pvc.Spec.AccessModes = []string{"ReadWriteOncePod"}
+	pod := closeoutPendingControllerPod("apps", "app", "node", "ReplicaSet", "ContainerCreating", time.Now().Add(-time.Hour))
+	if names := podRWOPVCVolumeNames(pod, map[string]persistentVolumeClaimResource{"apps/data": pvc}); names["data"] != "data" {
+		t.Fatalf("expected RWO pod volume mapping, got %v", names)
+	}
+	initPod := podResource{}
+	initPod.Status.InitContainerStatuses = []podContainerStatus{{Name: "init", State: podContainerState{Waiting: &podContainerWaitingState{Reason: "PodInitializing"}}}}
+	if !replacementPodPendingForAttach(initPod) {
+		t.Fatalf("expected init waiting pod to count as attach-pending")
+	}
+	event := eventList{Items: []eventResource{{Type: "Warning", Reason: "SomethingElse", Message: "volume is already used by pod"}}}
+	event.Items[0].InvolvedObject.Kind = "Pod"
+	event.Items[0].InvolvedObject.Namespace = "apps"
+	event.Items[0].InvolvedObject.Name = "app"
+	pod.Metadata.Namespace = "apps"
+	pod.Metadata.Name = "app"
+	if !podHasBlockedAttachEvent(pod, event) {
+		t.Fatalf("expected blocked attach message to match")
+	}
+	if podsShareAnyClaim(podResource{}, pod, map[string]string{"vol": "other"}) {
+		t.Fatalf("unrelated claims should not match")
+	}
+	unsafePod := podResource{}
+	unsafePod.Status.ContainerStatuses = []podContainerStatus{{Name: "ghost", State: podContainerState{Running: &podContainerRunningState{StartedAt: time.Now()}}}}
+	if !runningContainersMountAnyVolume(unsafePod, map[string]string{"data": "data"}) {
+		t.Fatalf("unknown running container should be treated unsafe")
+	}
+}
+
+// TestTCPDrainAndPostStartAdditionalCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestTCPDrainAndPostStartAdditionalCloseoutBranches(t *testing.T).
+// Why: near-threshold TCP, drain, and post-start helpers need local coverage for
+// TLS probing, error throttling, SSH fallback, and diagnostics summaries.
+func TestTCPDrainAndPostStartAdditionalCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	certServer := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte("ok"))
+	}))
+	cert := certServer.TLS.Certificates[0]
+	certServer.Close()
+	listener, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert}})
+	if err != nil {
+		t.Fatalf("listen TLS fixture: %v", err)
+	}
+	t.Cleanup(func() { _ = listener.Close() })
+	go func() {
+		for {
+			conn, err := listener.Accept()
+			if err != nil {
+				return
+			}
+			go func() {
+				defer conn.Close()
+				_, _ = conn.Write([]byte("220 ready\r\n"))
+				_ = conn.SetDeadline(time.Now().Add(2 * time.Second))
+				buf := make([]byte, 32)
+				_, _ = conn.Read(buf)
+				_, _ = conn.Write([]byte("250 tls-pong\r\n"))
+			}()
+		}
+	}()
+	host, portText, err := net.SplitHostPort(listener.Addr().String())
+	if err != nil {
+		t.Fatalf("split TLS server address: %v", err)
+	}
+	var port int
+	if _, err := fmt.Sscanf(portText, "%d", &port); err != nil {
+		t.Fatalf("parse TLS server port: %v", err)
+	}
+	ok, detail := buildOrchestratorWithStubs(t, config.Config{}, nil).tcpServiceCheckReady(ctx, config.TCPServiceChecklistCheck{
+		Host: host, Port: port, TLS: true, InsecureSkipTLS: true, Send: "PING\r\n", ExpectContains: "tls-pong",
+	})
+	if !ok || detail != "connected" {
+		t.Fatalf("expected TLS tcp check success, ok=%v detail=%q", ok, detail)
+	}
+	orch := buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{TCPServiceChecklist: []config.TCPServiceChecklistCheck{{Host: "127.0.0.1", Port: 1, Namespace: "mail", Service: "smtp"}}}}, []commandStub{
+		{match: matchContains("kubectl", "-n", "mail", "scale", "deployment", "smtp"), err: errors.New("scale denied")},
+	})
+	var zero time.Time
+	orch.maybeAutoHealTCPServiceBackends(ctx, &zero)
+	if zero.IsZero() {
+		t.Fatalf("expected tcp heal attempt timestamp")
+	}
+	dry := buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{TCPServiceChecklist: []config.TCPServiceChecklistCheck{{Host: "127.0.0.1", Port: 1}}}}, nil)
+	dry.runner.DryRun = true
+	dry.maybeAutoHealTCPServiceBackends(ctx, nil)
+
+	tmpConfig := t.TempDir() + "/ssh_config"
+	tmpIdentity := t.TempDir() + "/id_ed25519"
+	orch = buildOrchestratorWithStubs(t, config.Config{SSHConfigFile: tmpConfig, SSHIdentityFile: tmpIdentity}, nil)
+	if orch.resolveSSHConfigFile() != tmpConfig || orch.resolveSSHIdentityFile() != tmpIdentity {
+		t.Fatalf("expected explicit ssh config and identity paths")
+	}
+	orch.runSensitiveOverride = nil
+	if out, err := orch.runSensitive(ctx, time.Second, "sh", "-c", "echo noisy; exit 4"); err == nil || out != "noisy" {
+		t.Fatalf("expected runSensitive output-backed error, out=%q err=%v", out, err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "cordon", "worker-a"), err: errors.New("cordon warning")},
+		{match: matchContains("kubectl", "drain", "worker-a"), err: errors.New("drain denied")},
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: "apps api Running ReplicaSet\nkube ds Running DaemonSet\nold done Succeeded Job"},
+	})
+	if err := orch.drainWorkers(ctx, []string{"worker-a"}); err == nil || !strings.Contains(err.Error(), "apps/api") {
+		t.Fatalf("expected drain diagnostics in error, got %v", err)
+	}
+	if diag := orch.drainNodeDiagnostics(ctx, "worker-a"); !strings.Contains(diag, "apps/api") {
+		t.Fatalf("expected blocking pod diagnostic, got %q", diag)
+	}
+}
+
+// TestWorkloadConvergenceAndAutoHealCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestWorkloadConvergenceAndAutoHealCloseoutBranches(t *testing.T).
+// Why: workload convergence and post-start auto-heal are broad coordinators, so
+// focused fixtures keep their skip, warning, and no-op paths from going dark.
+func TestWorkloadConvergenceAndAutoHealCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), err: errors.New("api down")},
+	})
+	if _, _, err := orch.workloadConvergenceReady(ctx); err == nil {
+		t.Fatalf("expected workload controller query error")
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: "{"},
+	})
+	if _, _, err := orch.workloadConvergenceReady(ctx); err == nil {
+		t.Fatalf("expected workload controller decode error")
+	}
+	workloads := `{"items":[
+{"kind":"","metadata":{"namespace":"apps","name":"skip"}},
+{"kind":"Deployment","metadata":{"namespace":"apps","name":"api"},"spec":{"replicas":2},"status":{"readyReplicas":1}},
+{"kind":"StatefulSet","metadata":{"namespace":"ignored","name":"db"},"spec":{"replicas":1},"status":{"readyReplicas":0}},
+{"kind":"DaemonSet","metadata":{"namespace":"apps","name":"agent"},"status":{"desiredNumberScheduled":2,"numberReady":1}}
+]}`
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
+		IgnoreWorkloadNamespaces: []string{"ignored"},
+		IgnoreUnavailableNodes:   []string{"down-a"},
+	}}, []commandStub{
+		{match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: workloads},
+	})
+	ready, detail, err := orch.workloadConvergenceReady(ctx)
+	if err != nil || ready || !strings.Contains(detail, "apps/deployment/api") {
+		t.Fatalf("expected pending deployment detail, ready=%v detail=%q err=%v", ready, detail, err)
+	}
+	if _, _, ok := desiredReady(workloadResource{Kind: "Job"}); ok {
+		t.Fatalf("unsupported workload kind should not report desired/ready")
+	}
+
+	old := time.Now().Add(-10 * time.Minute)
+	podA := closeoutPendingControllerPod("apps", "wedge-a", "bad-node", "ReplicaSet", "CreateContainerError", old)
+	podB := closeoutPendingControllerPod("apps", "wedge-b", "bad-node", "ReplicaSet", "RunContainerError", old)
+	podA.Spec.Volumes = nil
+	podB.Spec.Volumes = nil
+	reasons := map[string]string{"apps/wedge-a": "ContainerRuntimeWedge:bad-node", "apps/wedge-b": "ContainerRuntimeWedge:bad-node"}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "annotate", "node", "bad-node"), out: ""},
+		{match: matchContains("kubectl", "cordon", "bad-node"), out: ""},
+	})
+	if nodes := orch.quarantineContainerRuntimeWedgeNodes(ctx, podList{Items: []podResource{podA, podB}}, reasons, time.Second, nil, nil, nil); len(nodes) != 1 || nodes[0] != "bad-node" {
+		t.Fatalf("expected wedged node quarantine, nodes=%v", nodes)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "annotate", "node", "bad-node"), out: ""},
+		{match: matchContains("kubectl", "cordon", "bad-node"), err: errors.New("cordon denied")},
+	})
+	if nodes := orch.quarantineContainerRuntimeWedgeNodes(ctx, podList{Items: []podResource{podA, podB}}, reasons, time.Second, nil, nil, nil); len(nodes) != 0 {
+		t.Fatalf("cordon failure should not report quarantined nodes, nodes=%v", nodes)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: "{"},
+	})
+	if _, err := orch.quarantineContainerRuntimeWedgeNodesFromCluster(ctx); err == nil {
+		t.Fatalf("expected runtime wedge cluster decode error")
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A"), err: errors.New("events down")},
+	})
+	if _, err := orch.containerRuntimeWedgePodReasons(ctx, podList{Items: []podResource{podA}}, time.Second); err == nil {
+		t.Fatalf("expected runtime wedge event query error")
+	}
+	invalidPod := closeoutPendingControllerPod("", "invalid", "node-a", "ReplicaSet", "CreateContainerError", old)
+	runningPod := closeoutPendingControllerPod("apps", "running", "node-a", "ReplicaSet", "CreateContainerError", old)
+	runningPod.Status.Phase = "Running"
+	noOwnerPod := closeoutPendingControllerPod("apps", "no-owner", "node-a", "", "CreateContainerError", old)
+	noOwnerPod.Metadata.OwnerReferences = nil
+	newPod := closeoutPendingControllerPod("apps", "new", "node-a", "ReplicaSet", "CreateContainerError", time.Now())
+	runtimePod := closeoutPendingControllerPod("apps", "runtime", "node-a", "ReplicaSet", "CreateContainerError", old)
+	runtimeEvents := eventList{Items: []eventResource{
+		{Type: "Normal", Reason: "Failed", Message: "failed to reserve container name"},
+		{Type: "Warning", Reason: "Other", Message: "failed to reserve container name"},
+		{Type: "Warning", Reason: "Failed", Message: "failed to reserve container name"},
+		{Type: "Warning", Reason: "Failed", Message: "failed to reserve container name"},
+		{Type: "Warning", Reason: "Failed", EventTime: old.Add(-time.Minute), Message: "failed to reserve container name"},
+		{Type: "Warning", Reason: "Failed", Message: "ordinary failure"},
+	}}
+	for i := range runtimeEvents.Items {
+		runtimeEvents.Items[i].InvolvedObject.Kind = "Pod"
+		runtimeEvents.Items[i].InvolvedObject.Namespace = "apps"
+		runtimeEvents.Items[i].InvolvedObject.Name = "runtime"
+	}
+	runtimeEvents.Items[2].InvolvedObject.Kind = "Service"
+	runtimeEvents.Items[3].InvolvedObject.Name = "missing"
+	runtimeRaw, err := json.Marshal(runtimeEvents)
+	if err != nil {
+		t.Fatalf("marshal runtime events: %v", err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A"), out: string(runtimeRaw)},
+	})
+	if reasons, err := orch.containerRuntimeWedgePodReasons(ctx, podList{Items: []podResource{invalidPod, runningPod, noOwnerPod, newPod, runtimePod}}, time.Second); err != nil || len(reasons) != 0 {
+		t.Fatalf("expected runtime skip-only reasons, got reasons=%v err=%v", reasons, err)
+	}
+	ignoredPod := closeoutPendingControllerPod("ignored", "pod", "node-a", "ReplicaSet", "CreateContainerError", old)
+	rulePod := closeoutPendingControllerPod("apps", "skip-me", "node-a", "ReplicaSet", "CreateContainerError", old)
+	ignoredNodePod := closeoutPendingControllerPod("apps", "node-skip", "node-skip", "ReplicaSet", "CreateContainerError", old)
+	pvcPod := closeoutPendingControllerPod("apps", "pvc", "node-a", "ReplicaSet", "CreateContainerError", old)
+	noReasonPod := closeoutPendingControllerPod("apps", "no-reason", "node-a", "ReplicaSet", "CreateContainerError", old)
+	reasonMap := map[string]string{
+		"ignored/pod":    "ContainerRuntimeWedge:node-a",
+		"apps/skip-me":   "ContainerRuntimeWedge:node-a",
+		"apps/node-skip": "ContainerRuntimeWedge:node-skip",
+		"apps/no-owner":  "ContainerRuntimeWedge:node-a",
+		"apps/new":       "ContainerRuntimeWedge:node-a",
+		"apps/pvc":       "ContainerRuntimeWedge:node-a",
+	}
+	if nodes := orch.quarantineContainerRuntimeWedgeNodes(ctx, podList{Items: []podResource{invalidPod, noReasonPod, ignoredPod, rulePod, ignoredNodePod, noOwnerPod, newPod, pvcPod}}, reasonMap, time.Second, map[string]struct{}{"ignored": struct{}{}}, map[string]struct{}{"node-skip": struct{}{}}, []workloadIgnoreRule{{Namespace: "apps", Name: "skip-me"}}); len(nodes) != 0 {
+		t.Fatalf("skip-only quarantine should return no nodes, got %v", nodes)
+	}
+
+	stuck := closeoutPendingControllerPod("apps", "crashy", "node-a", "ReplicaSet", "CrashLoopBackOff", old)
+	stuck.Spec.Volumes = nil
+	podsJSON := fmt.Sprintf(`{"items":[{"metadata":{"namespace":"apps","name":"crashy","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"node-a"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CrashLoopBackOff"}}}]}}]}`, old.Format(time.RFC3339))
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: podsJSON},
+		{match: matchContains("kubectl", "get", "events", "-A"), err: errors.New("events down")},
+		{match: matchContains("kubectl", "get", "pvc", "-A"), err: errors.New("pvc down")},
+		{match: matchContains("kubectl", "-n", "apps", "delete", "pod", "crashy"), out: ""},
+	})
+	if err := orch.recycleStuckControllerPods(ctx); err != nil {
+		t.Fatalf("expected stuck pod recycle despite scan warnings: %v", err)
+	}
+
+	auto := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: ""},
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[]}`},
+		{match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"},
+		{match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`},
+	})
+	if err := auto.postStartAutoHeal(ctx); err != nil {
+		t.Fatalf("expected post-start auto-heal no-op success, got %v", err)
+	}
+}
+
+// TestStartupWaitLoopCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestStartupWaitLoopCloseoutBranches(t *testing.T).
+// Why: startup wait loops should cover immediate success and cancellation paths
+// without sleeping or requiring any live Kubernetes objects.
+func TestStartupWaitLoopCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: `{"items":[]}`},
+	})
+	if err := orch.waitForWorkloadConvergence(ctx); err != nil {
+		t.Fatalf("expected workload convergence success: %v", err)
+	}
+	canceled, cancel := context.WithCancel(ctx)
+	cancel()
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
+		WorkloadConvergenceWaitSeconds: 1,
+		WorkloadConvergencePollSeconds: 1,
+	}}, []commandStub{
+		{match: matchContains("kubectl", "get", "deploy,statefulset,daemonset"), out: `{"items":[{"kind":"Deployment","metadata":{"namespace":"apps","name":"api"},"spec":{"replicas":1},"status":{"readyReplicas":0}}]}`},
+	})
+	if err := orch.waitForWorkloadConvergence(canceled); !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected canceled workload convergence, got %v", err)
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, nil)
+	if err := orch.waitForCriticalServiceEndpoints(ctx); err != nil {
+		t.Fatalf("expected critical endpoint success: %v", err)
+	}
+	canceled, cancel = context.WithCancel(ctx)
+	cancel()
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
+		CriticalServiceEndpoints:       []string{"apps/api"},
+		CriticalServiceEndpointWaitSec: 1,
+		CriticalServiceEndpointPollSec: 1,
+	}}, []commandStub{
+		{match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), out: `{"subsets":[]}`},
+	})
+	orch.runner.DryRun = true
+	if err := orch.waitForCriticalServiceEndpoints(canceled); !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected canceled critical endpoint wait, got %v", err)
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, nil)
+	orch.runner.DryRun = true
+	if err := orch.waitForPostStartProbes(ctx); err != nil {
+		t.Fatalf("dry-run post-start probes should pass: %v", err)
+	}
+	canceled, cancel = context.WithCancel(ctx)
+	cancel()
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
+		PostStartProbes:           []string{"https://example.invalid"},
+		PostStartProbeWaitSeconds: 1,
+		PostStartProbePollSeconds: 1,
+	}}, []commandStub{
+		{match: matchContains("curl", "https://example.invalid"), err: errors.New("curl down")},
+	})
+	if err := orch.waitForPostStartProbes(canceled); !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected canceled post-start probe wait, got %v", err)
+	}
+}

internal/cluster/orchestrator_quality_gate_recovery_closeout_test.go

@@ -0,0 +1,370 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestCordonLeaseCriticalEndpointAndProbeCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestCordonLeaseCriticalEndpointAndProbeCloseoutBranches(t *testing.T).
+// Why: recovery cordons and endpoint probe repair carry the cluster-safety
+// contract for automatic startup repairs, so their failure paths need coverage.
+func TestCordonLeaseCriticalEndpointAndProbeCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	orch := buildOrchestratorWithStubs(t, config.Config{}, nil)
+	if err := orch.cordonNodeWithLease(ctx, " ", "", ""); err == nil {
+		t.Fatalf("expected empty cordon node to fail")
+	}
+	if ok := manualActionAlreadyMarked(nil, "action"); ok {
+		t.Fatalf("nil annotations should not already mark action")
+	}
+	if expired, detail := cordonLeaseExpired(map[string]string{}, time.Now()); !expired || detail != "missing deadline" {
+		t.Fatalf("expected missing deadline to expire, expired=%v detail=%q", expired, detail)
+	}
+	if expired, detail := cordonLeaseExpired(map[string]string{anankeCordonDeadlineAnnotation: "bad-time"}, time.Now()); !expired || detail != "bad-time" {
+		t.Fatalf("expected malformed deadline to expire, expired=%v detail=%q", expired, detail)
+	}
+	if since := nodeUnschedulableSince(nodeReadyItem{}); !since.IsZero() {
+		t.Fatalf("expected zero unschedulable timestamp, got %s", since)
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "annotate", "node", "titan-23"), err: errors.New("annotate denied")},
+	})
+	if err := orch.cordonNodeWithLease(ctx, "titan-23", "repair", "detail"); err == nil || !strings.Contains(err.Error(), "annotate cordon lease") {
+		t.Fatalf("expected annotate cordon lease error, got %v", err)
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "cordon", "titan-23"), err: errors.New("cordon denied")},
+		{match: matchContains("kubectl", "annotate", "node", "titan-23"), out: ""},
+	})
+	if err := orch.cordonNodeWithLease(ctx, "titan-23", "repair", "detail"); err == nil || !strings.Contains(err.Error(), "cordon denied") {
+		t.Fatalf("expected cordon failure, got %v", err)
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "annotate", "node", "gone"), err: errors.New(`Error from server (NotFound): nodes "gone" not found`)},
+		{match: matchContains("kubectl", "annotate", "node", "blocked"), err: errors.New("api denied")},
+	})
+	if err := orch.clearCordonLease(ctx, "gone"); err != nil {
+		t.Fatalf("not-found clear should be ignored: %v", err)
+	}
+	if err := orch.clearCordonLease(ctx, "blocked"); err == nil || !strings.Contains(err.Error(), "clear cordon lease") {
+		t.Fatalf("expected clear cordon lease error, got %v", err)
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "uncordon", "titan-23"), out: ""},
+		{match: matchContains("kubectl", "annotate", "node", "titan-23"), err: errors.New("cleanup denied")},
+	})
+	if err := orch.uncordonAndClearCordonLease(ctx, "titan-23", "test"); err != nil {
+		t.Fatalf("lease cleanup warning should not fail uncordon: %v", err)
+	}
+	if _, err := orch.recoverLeasedCordon(ctx, "titan-23", map[string]string{anankeCordonReasonAnnotation: "mystery"}); err == nil {
+		t.Fatalf("expected unknown cordon reason error")
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/titan-23/proxy/healthz"), err: errors.New("proxy down")},
+	})
+	if recovered, err := orch.recoverLeasedCordon(ctx, "titan-23", map[string]string{anankeCordonReasonAnnotation: cordonReasonKubeletProxy}); err == nil || recovered {
+		t.Fatalf("expected unhealthy proxy to keep cordon, recovered=%v err=%v", recovered, err)
+	}
+
+	old := time.Now().Add(-10 * time.Minute)
+	podJSON := fmt.Sprintf(`{"items":[{"metadata":{"namespace":"apps","name":"stuck","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"titan-23"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CreateContainerError"}}}]}}]}`, old.Format(time.RFC3339))
+	eventJSON := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"stuck"},"message":"failed to reserve container name for container app"}]}`, time.Now().Format(time.RFC3339))
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: podJSON},
+		{match: matchContains("kubectl", "get", "events", "-A"), out: eventJSON},
+	})
+	if wedged, err := orch.nodeStillHasRuntimeWedge(ctx, "titan-23"); err != nil || !wedged {
+		t.Fatalf("expected runtime wedge evidence, wedged=%v err=%v", wedged, err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: "{"},
+	})
+	if _, err := orch.nodeStillHasRuntimeWedge(ctx, "titan-23"); err == nil {
+		t.Fatalf("expected bad pod json error")
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: "{"},
+	})
+	if _, err := orch.queryReadyNodes(ctx); err == nil {
+		t.Fatalf("expected bad node json error")
+	}
+
+	endpointsJSON := `{"subsets":[{"addresses":[{"ip":"10.0.0.1"},{"ip":"10.0.0.2"}]},{"addresses":[{"ip":""}]}]}`
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{
+		{match: matchContains("kubectl", "-n", "grafana", "get", "endpoints", "grafana"), out: endpointsJSON},
+		{match: matchContains("kubectl", "-n", "bad", "get", "endpoints", "svc"), out: "{", err: nil},
+	})
+	if count, err := orch.endpointAddressCount(ctx, "grafana", "grafana"); err != nil || count != 2 {
+		t.Fatalf("unexpected endpoint count=%d err=%v", count, err)
+	}
+	if _, err := orch.endpointAddressCount(ctx, "bad", "svc"); err == nil {
+		t.Fatalf("expected endpoint decode error")
+	}
+	if ready, detail, _, _, err := orch.criticalServiceEndpointsReady(ctx); err != nil || !ready || detail != "no critical service endpoints configured" {
+		t.Fatalf("expected empty critical endpoint checklist ready, ready=%v detail=%q err=%v", ready, detail, err)
+	}
+
+	workloadJSON := `{"spec":{"template":{"spec":{"containers":[{"name":"app","livenessProbe":{"httpGet":{"path":"/healthz"}}},{"name":"","livenessProbe":{"tcpSocket":{"port":8080}}},{"name":"has-startup","livenessProbe":{"httpGet":{"path":"/ready"}},"startupProbe":{"httpGet":{"path":"/ready"}}}]}}}}`
+	var patched bool
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{
+		{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: workloadJSON},
+		{match: func(name string, args []string) bool {
+			if name == "kubectl" && strings.Contains(strings.Join(args, " "), "-n apps patch deployment api") {
+				patched = true
+				return true
+			}
+			return false
+		}, out: ""},
+	})
+	repaired, err := orch.repairWorkloadStartupProbes(ctx, "apps", "deployment", "api")
+	if err != nil || len(repaired) != 1 || repaired[0] != "apps/deployment/api" || !patched {
+		t.Fatalf("unexpected startup probe repair repaired=%v patched=%v err=%v", repaired, patched, err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{
+		{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: "{", err: nil},
+	})
+	if _, err := orch.repairWorkloadStartupProbes(ctx, "apps", "deployment", "api"); err == nil {
+		t.Fatalf("expected workload decode error")
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{
+		{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: workloadJSON},
+		{match: matchContains("kubectl", "-n", "apps", "patch", "deployment", "api"), err: errors.New("patch denied")},
+	})
+	if _, err := orch.repairWorkloadStartupProbes(ctx, "apps", "deployment", "api"); err == nil {
+		t.Fatalf("expected workload patch error")
+	}
+	if repaired, err := orch.maybeRepairCriticalBackendStartupProbes(ctx, " ", "api"); err != nil || repaired != nil {
+		t.Fatalf("blank critical backend repair should no-op, repaired=%v err=%v", repaired, err)
+	}
+}
+
+// TestLonghornRuntimeStaleRWOAndPostStartCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestLonghornRuntimeStaleRWOAndPostStartCloseoutBranches(t *testing.T).
+// Why: storage handoff, host prerequisite repair, runtime readiness, and
+// post-start shell helpers are high-risk automation seams that should stay
+// covered by local fixtures.
+func TestLonghornRuntimeStaleRWOAndPostStartCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	old := time.Now().Add(-10 * time.Minute)
+	pod := closeoutPendingControllerPod("apps", "crypt", "titan-23", "ReplicaSet", "CreateContainerError", old)
+	events := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"crypt"},"message":"MountVolume.SetUp failed: cryptsetup: no such file or directory"}]}`, time.Now().Format(time.RFC3339))
+	cryptsetupCalls := 0
+	orch := buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"titan-23"}}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A"), out: events},
+		{match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")},
+		{match: matchContains("kubectl", "annotate", "node", "titan-23"), out: ""},
+		{match: matchContains("kubectl", "cordon", "titan-23"), out: ""},
+	})
+	reasons, err := orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{pod}}, time.Second)
+	if err != nil || reasons["apps/crypt"] != "EncryptedVolumeCryptsetupNodeCordoned:titan-23" {
+		t.Fatalf("unexpected encrypted mount repair reasons=%v err=%v", reasons, err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"titan-23"}}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A"), out: events},
+		{match: func(name string, args []string) bool {
+			if name == "ssh" && strings.Contains(strings.Join(args, " "), "apt-get update") {
+				cryptsetupCalls++
+				return true
+			}
+			return false
+		}, out: "__ANANKE_CRYPTSETUP_INSTALLED__"},
+		{match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_MISSING__", err: errors.New("exit 41")},
+	})
+	reasons, err = orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{pod, pod}}, time.Second)
+	if err != nil || reasons["apps/crypt"] != "EncryptedVolumeCryptsetupRepaired:titan-23" || cryptsetupCalls != 1 {
+		t.Fatalf("unexpected repaired cryptsetup reasons=%v calls=%d err=%v", reasons, cryptsetupCalls, err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A"), out: "{"},
+	})
+	if _, err := orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{pod}}, time.Second); err == nil {
+		t.Fatalf("expected encrypted mount event decode error")
+	}
+
+	longhornNodes := "titan-23\tFalse\nready\tTrue\npartial\n"
+	attachEvents := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc-1: node titan-23 is not ready: longhorn-backend unavailable"}]}`, time.Now().Format(time.RFC3339))
+	attachPod := closeoutPendingControllerPod("apps", "attach", "titan-23", "ReplicaSet", "ContainerCreating", old)
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: longhornNodes},
+		{match: matchContains("kubectl", "get", "events", "-A"), out: attachEvents},
+	})
+	blocked, err := orch.longhornAttachBlockedPodReasons(ctx, podList{Items: []podResource{attachPod}}, time.Second)
+	if err != nil || blocked["apps/attach"] != "LonghornAttachBlockedOnUnreadyNode:titan-23" {
+		t.Fatalf("unexpected attach-blocked reasons=%v err=%v", blocked, err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), err: errors.New("api down")},
+	})
+	if _, err := orch.longhornAttachBlockedPodReasons(ctx, podList{Items: []podResource{attachPod}}, time.Second); err == nil {
+		t.Fatalf("expected longhorn node query error")
+	}
+
+	pvcJSON := `{"items":[{"metadata":{"namespace":"apps","name":"data"},"spec":{"accessModes":["ReadWriteOnce"],"volumeName":"pv-data"}}]}`
+	replacement := closeoutPendingControllerPod("apps", "app-new", "titan-24", "ReplicaSet", "ContainerCreating", old)
+	oldOwner := closeoutDeletingRWOPod("apps", "app-old", "titan-23", old, false)
+	blockEvents := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"app-new"},"message":"Multi-Attach error for volume pv-data"}]}`, time.Now().Format(time.RFC3339))
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A"), out: blockEvents},
+		{match: matchContains("kubectl", "get", "pvc", "-A"), out: pvcJSON},
+	})
+	decisions, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{Items: []podResource{oldOwner, replacement}}, time.Second)
+	if err != nil || !decisions["apps/app-old"].ForceDelete || decisions["apps/app-old"].Unsafe {
+		t.Fatalf("unexpected safe stale RWO decision=%v err=%v", decisions, err)
+	}
+	oldUnsafe := closeoutDeletingRWOPod("apps", "app-old", "titan-23", old, true)
+	decisions, err = orch.staleRWOPVCOwnerDecisions(ctx, podList{Items: []podResource{oldUnsafe, replacement}}, time.Second)
+	if err != nil || !decisions["apps/app-old"].Unsafe {
+		t.Fatalf("unexpected unsafe stale RWO decision=%v err=%v", decisions, err)
+	}
+	if pvcSingleWriter(persistentVolumeClaimResource{}) {
+		t.Fatalf("empty pvc should not be single-writer")
+	}
+	if replacementPodPendingForAttach(podResource{}) {
+		t.Fatalf("empty pod should not be pending for attach")
+	}
+	if podKey(podResource{}) != "" {
+		t.Fatalf("empty pod should not have key")
+	}
+
+	runtimeCfg := config.Config{SSHManagedNodes: []string{"titan-23"}, Startup: config.Startup{NodeRuntimeRestartWaitSeconds: 1}}
+	orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
+		{match: matchContains("ssh", "systemctl --no-block restart k3s-agent"), out: ""},
+		{match: matchContains("kubectl", "wait", "node/titan-23"), out: ""},
+		{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/titan-23/proxy/healthz"), out: "ok"},
+		{match: matchContains("ssh", "systemctl is-active k3s-agent"), out: "active"},
+	})
+	if err := orch.recoverManagedNodeRuntime(ctx, "titan-23", true, "already cordoned"); err != nil {
+		t.Fatalf("expected runtime recovery success, got %v", err)
+	}
+	orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
+		{match: matchContains("kubectl", "wait", "node/titan-23"), out: ""},
+		{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/titan-23/proxy/healthz"), out: "ok"},
+		{match: matchContains("ssh", "systemctl is-active k3s-agent"), out: "inactive"},
+	})
+	if err := orch.waitForManagedNodeRuntimeReady(ctx, "titan-23", 0); err == nil || !strings.Contains(err.Error(), "inactive") {
+		t.Fatalf("expected inactive k3s-agent readiness failure, got %v", err)
+	}
+	if got := orch.nodeRuntimeRebootWait(); got != 420*time.Second {
+		t.Fatalf("expected default reboot wait, got %s", got)
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{PostStartProbes: []string{" https://ok ", " "}}}, []commandStub{
+		{match: matchContains("curl", "https://ok"), out: "403"},
+	})
+	if ok, detail := orch.postStartProbesReady(ctx); !ok || detail != "all probes successful" {
+		t.Fatalf("expected post-start probe success, ok=%v detail=%q", ok, detail)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{PostStartProbes: []string{"https://bad"}}}, []commandStub{
+		{match: matchContains("curl", "https://bad"), out: "oops"},
+	})
+	if ok, detail := orch.postStartProbesReady(ctx); ok || !strings.Contains(detail, "parse http status") {
+		t.Fatalf("expected post-start parse failure, ok=%v detail=%q", ok, detail)
+	}
+	if probeStatusAccepted("x", 500) {
+		t.Fatalf("500 should not be an accepted probe status")
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "--raw", "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health"), err: errors.New("transport closed")},
+	})
+	if _, err := orch.vaultSealedViaHTTP(ctx); err == nil || !strings.Contains(err.Error(), "vault HTTP health check failed") {
+		t.Fatalf("expected empty-output vault HTTP error, got %v", err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "--raw", "/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health"), out: "{", err: errors.New("partial")},
+	})
+	if _, err := orch.vaultSealedViaHTTP(ctx); err == nil || !strings.Contains(err.Error(), "parse") {
+		t.Fatalf("expected vault HTTP parse error with transport failure, got %v", err)
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, nil)
+	orch.runner.Kubeconfig = filepath.Join(t.TempDir(), "kubeconfig")
+	if out, err := orch.runSensitiveWithInput(ctx, time.Second, "secret", "sh", "-c", "cat"); err != nil || out != "secret" {
+		t.Fatalf("expected runSensitiveWithInput echo, out=%q err=%v", out, err)
+	}
+	if out, err := orch.runSensitiveWithInput(ctx, time.Second, "", "sh", "-c", "exit 3"); err == nil || out != "" {
+		t.Fatalf("expected empty-output command failure, out=%q err=%v", out, err)
+	}
+
+	binDir := t.TempDir()
+	fakeSSH := filepath.Join(binDir, "ssh")
+	if err := os.WriteFile(fakeSSH, []byte("#!/bin/sh\ncase \" $* \" in *\" -J \"*) echo jump failed; exit 7;; esac\ncat >/dev/null\necho direct-ok\n"), 0o755); err != nil {
+		t.Fatalf("write fake ssh: %v", err)
+	}
+	fakeFlux := filepath.Join(binDir, "flux")
+	if err := os.WriteFile(fakeFlux, []byte("#!/bin/sh\necho flux-ok\n"), 0o755); err != nil {
+		t.Fatalf("write fake flux: %v", err)
+	}
+	t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH"))
+	orch = buildOrchestratorWithStubs(t, config.Config{SSHJumpHost: "bastion", SSHJumpUser: "jump", SSHUser: "atlas", SSHNodeHosts: map[string]string{"titan-23": "10.0.0.23"}}, nil)
+	out, err := orch.sshWithInput(ctx, "titan-23", "sudo -S true", "pw\n", time.Second)
+	if err != nil || out != "direct-ok" {
+		t.Fatalf("expected fake ssh direct fallback success, out=%q err=%v", out, err)
+	}
+	var sawFlux bool
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "patch"), out: ""},
+		{match: matchContains("kubectl", "annotate"), out: ""},
+		{match: func(name string, args []string) bool {
+			if name == "flux" && strings.Contains(strings.Join(args, " "), "reconcile source git flux-system") {
+				sawFlux = true
+				return true
+			}
+			return false
+		}, out: ""},
+	})
+	if err := orch.resumeFluxAndReconcile(ctx); err != nil || !sawFlux {
+		t.Fatalf("expected flux reconcile path, sawFlux=%v err=%v", sawFlux, err)
+	}
+}
+
+// closeoutPendingControllerPod runs one orchestration or CLI step.
+// Signature: closeoutPendingControllerPod(ns, name, node, ownerKind, waitReason string, created time.Time) podResource.
+// Why: coverage closeout tests need compact pod fixtures for storage and runtime
+// classifiers without repeating nested Kubernetes JSON structs.
+func closeoutPendingControllerPod(ns, name, node, ownerKind, waitReason string, created time.Time) podResource {
+	var pod podResource
+	pod.Metadata.Namespace = ns
+	pod.Metadata.Name = name
+	pod.Metadata.CreationTimestamp = created
+	pod.Metadata.OwnerReferences = []ownerReference{{Kind: ownerKind, Name: "owner"}}
+	pod.Spec.NodeName = node
+	pod.Spec.Volumes = []podVolume{{Name: "data", PersistentVolumeClaim: &podPersistentVolumeClaim{ClaimName: "data"}}}
+	pod.Status.Phase = "Pending"
+	pod.Status.ContainerStatuses = []podContainerStatus{{Name: "app", State: podContainerState{Waiting: &podContainerWaitingState{Reason: waitReason}}}}
+	return pod
+}
+
+// closeoutDeletingRWOPod runs one orchestration or CLI step.
+// Signature: closeoutDeletingRWOPod(ns, name, node string, deletedAt time.Time, runningWriter bool) podResource.
+// Why: stale RWO tests need paired safe and unsafe terminating owner pods with
+// the same PVC semantics but different live-writer evidence.
+func closeoutDeletingRWOPod(ns, name, node string, deletedAt time.Time, runningWriter bool) podResource {
+	pod := closeoutPendingControllerPod(ns, name, node, "ReplicaSet", "", deletedAt.Add(-time.Hour))
+	pod.Metadata.DeletionTimestamp = &deletedAt
+	pod.Status.Phase = "Running"
+	pod.Spec.Containers = []podContainer{
+		{Name: "app", VolumeMounts: []podVolumeMount{{Name: "data", MountPath: "/data"}}},
+		{Name: "sidecar"},
+	}
+	statusName := "sidecar"
+	if runningWriter {
+		statusName = "app"
+	}
+	pod.Status.ContainerStatuses = []podContainerStatus{{Name: statusName, State: podContainerState{Running: &podContainerRunningState{StartedAt: deletedAt.Add(-time.Minute)}}}}
+	return pod
+}

internal/cluster/orchestrator_quality_gate_storage_closeout_test.go

@@ -0,0 +1,429 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestLonghornRecoveryClassifierCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestLonghornRecoveryClassifierCloseoutBranches(t *testing.T).
+// Why: Longhorn mount and attach recovery should skip noisy non-matches and
+// only act on current, pod-scoped storage evidence.
+func TestLonghornRecoveryClassifierCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	old := time.Now().Add(-10 * time.Minute)
+	now := time.Now()
+	invalid := closeoutPendingControllerPod("", "invalid", "node-a", "ReplicaSet", "CreateContainerError", old)
+	running := closeoutPendingControllerPod("apps", "running", "node-a", "ReplicaSet", "CreateContainerError", old)
+	running.Status.Phase = "Running"
+	noOwner := closeoutPendingControllerPod("apps", "no-owner", "node-a", "", "CreateContainerError", old)
+	noOwner.Metadata.OwnerReferences = nil
+	newPod := closeoutPendingControllerPod("apps", "new", "node-a", "ReplicaSet", "CreateContainerError", time.Now())
+	unmanaged := closeoutPendingControllerPod("apps", "unmanaged", "node-unmanaged", "ReplicaSet", "CreateContainerError", old)
+	reuseA := closeoutPendingControllerPod("apps", "reuse-a", "node-ok", "ReplicaSet", "CreateContainerError", old)
+	reuseB := closeoutPendingControllerPod("apps", "reuse-b", "node-ok", "ReplicaSet", "CreateContainerError", old)
+	fail := closeoutPendingControllerPod("apps", "fail", "node-fail", "ReplicaSet", "CreateContainerError", old)
+	events := fmt.Sprintf(`{"items":[
+{"type":"Normal","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"},
+{"type":"Warning","reason":"Other","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"},
+{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Service","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"},
+{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"missing"},"message":"cryptsetup: no such file or directory"},
+{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"},
+{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"different mount failure"},
+{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"unmanaged"},"message":"cryptsetup: no such file or directory"},
+{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-a"},"message":"cryptsetup: no such file or directory"},
+{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"reuse-b"},"message":"cryptsetup: no such file or directory"},
+{"type":"Warning","reason":"FailedMount","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"fail"},"message":"cryptsetup: no such file or directory"}
+]}`,
+		now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339),
+		old.Add(-time.Minute).Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339),
+		now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339))
+	orch := buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"node-ok", "node-fail"}}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A"), out: events},
+		{match: matchContains("ssh", "node-ok", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_PRESENT__"},
+		{match: matchContains("ssh", "node-fail", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")},
+		{match: matchContains("kubectl", "annotate", "node", "node-fail"), err: errors.New("annotate denied")},
+	})
+	reasons, err := orch.repairEncryptedVolumeMountPrereqs(ctx, podList{Items: []podResource{invalid, running, noOwner, newPod, unmanaged, reuseA, reuseB, fail}}, time.Second)
+	if err != nil {
+		t.Fatalf("repairEncryptedVolumeMountPrereqs: %v", err)
+	}
+	if reasons["apps/reuse-a"] != "EncryptedVolumeCryptsetupRepaired:node-ok" || reasons["apps/reuse-b"] != "EncryptedVolumeCryptsetupRepaired:node-ok" {
+		t.Fatalf("expected reused cryptsetup repair reasons, got %v", reasons)
+	}
+	if reasons["apps/fail"] != "" {
+		t.Fatalf("cordon failure should not mark failed pod repaired, reasons=%v", reasons)
+	}
+
+	attachEvents := fmt.Sprintf(`{"items":[
+{"type":"Normal","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"},
+{"type":"Warning","reason":"Other","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"},
+{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Service","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"},
+{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"missing"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"},
+{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node node-bad is not ready: longhorn-backend unavailable"},
+{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"some other attach message"},
+{"type":"Warning","reason":"FailedAttachVolume","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"attach"},"message":"AttachVolume.Attach failed for volume pvc: node other is not ready: longhorn-backend unavailable"}
+]}`,
+		now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339),
+		old.Add(-time.Minute).Format(time.RFC3339), now.Format(time.RFC3339), now.Format(time.RFC3339))
+	attach := closeoutPendingControllerPod("apps", "attach", "node-bad", "ReplicaSet", "ContainerCreating", old)
+	notPending := closeoutPendingControllerPod("apps", "not-pending", "node-bad", "ReplicaSet", "ContainerCreating", old)
+	notPending.Status.Phase = "Running"
+	readyNode := closeoutPendingControllerPod("apps", "ready-node", "node-ready", "ReplicaSet", "ContainerCreating", old)
+	noAttachOwner := closeoutPendingControllerPod("apps", "no-owner", "node-bad", "", "ContainerCreating", old)
+	noAttachOwner.Metadata.OwnerReferences = nil
+	newAttach := closeoutPendingControllerPod("apps", "new", "node-bad", "ReplicaSet", "ContainerCreating", time.Now())
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: "node-bad\tFalse\nnode-ready\tTrue\n"},
+		{match: matchContains("kubectl", "get", "events", "-A"), out: attachEvents},
+	})
+	blocked, err := orch.longhornAttachBlockedPodReasons(ctx, podList{Items: []podResource{attach, notPending, readyNode, noAttachOwner, newAttach}}, time.Second)
+	if err != nil {
+		t.Fatalf("longhornAttachBlockedPodReasons: %v", err)
+	}
+	if blocked["apps/attach"] != "" {
+		t.Fatalf("all attach events should have been skipped, got %v", blocked)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), err: errors.New(`Error from server (NotFound): nodes.longhorn.io not found`)},
+	})
+	if unready, err := orch.longhornUnreadyNodes(ctx); err != nil || len(unready) != 0 {
+		t.Fatalf("expected missing longhorn nodes to be empty, unready=%v err=%v", unready, err)
+	}
+}
+
+// TestStaleRWOClassifierCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestStaleRWOClassifierCloseoutBranches(t *testing.T).
+// Why: stale RWO ownership must reject incomplete handoff evidence before it
+// considers force-deleting a terminating pod.
+func TestStaleRWOClassifierCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	old := time.Now().Add(-10 * time.Minute)
+	pvcJSON := `{"items":[{"metadata":{"namespace":"apps","name":"data"},"spec":{"accessModes":["ReadWriteOnce"]}}]}`
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A"), out: "{"},
+	})
+	if _, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{}, time.Second); err == nil {
+		t.Fatalf("expected stale RWO event decode error")
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A"), out: ""},
+		{match: matchContains("kubectl", "get", "pvc", "-A"), err: errors.New("pvc down")},
+	})
+	if _, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{}, time.Second); err == nil {
+		t.Fatalf("expected stale RWO pvc query error")
+	}
+	oldPod := closeoutDeletingRWOPod("apps", "old", "node-a", old, false)
+	sameNode := closeoutPendingControllerPod("apps", "same-node", "node-a", "ReplicaSet", "ContainerCreating", old)
+	noEvent := closeoutPendingControllerPod("apps", "no-event", "node-b", "ReplicaSet", "ContainerCreating", old)
+	noClaim := closeoutPendingControllerPod("apps", "no-claim", "node-b", "ReplicaSet", "ContainerCreating", old)
+	noClaim.Spec.Volumes = nil
+	events := `{"items":[{"type":"Warning","reason":"FailedAttachVolume","involvedObject":{"kind":"Pod","namespace":"apps","name":"no-claim"},"message":"Multi-Attach error"}]}`
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A"), out: events},
+		{match: matchContains("kubectl", "get", "pvc", "-A"), out: pvcJSON},
+	})
+	decisions, err := orch.staleRWOPVCOwnerDecisions(ctx, podList{Items: []podResource{oldPod, sameNode, noEvent, noClaim}}, time.Second)
+	if err != nil {
+		t.Fatalf("staleRWOPVCOwnerDecisions: %v", err)
+	}
+	if len(decisions) != 0 {
+		t.Fatalf("incomplete replacement evidence should not decide, got %v", decisions)
+	}
+	emptyPVCs, err := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "pvc", "-A"), out: ""},
+	}).queryPVCs(ctx)
+	if err != nil || len(emptyPVCs) != 0 {
+		t.Fatalf("expected empty pvc output to return empty map, pvcs=%v err=%v", emptyPVCs, err)
+	}
+	if runningContainersMountAnyVolume(podResource{}, nil) {
+		t.Fatalf("empty volume set should be safe")
+	}
+	if podHasBlockedAttachEvent(podResource{}, eventList{Items: []eventResource{{Type: "Normal"}}}) {
+		t.Fatalf("normal events should not block attach")
+	}
+	if strings.TrimSpace(podControllerKey(podResource{})) != "" {
+		t.Fatalf("empty pod should not have controller key")
+	}
+}
+
+// TestRecoveryCoordinatorMiscCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestRecoveryCoordinatorMiscCloseoutBranches(t *testing.T).
+// Why: a handful of coordinator branches only fire on unusual repair failures;
+// local stubs keep them covered without mutating the cluster.
+func TestRecoveryCoordinatorMiscCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	orch := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "annotate", "node", "ok"), out: ""},
+		{match: matchContains("kubectl", "cordon", "ok"), out: ""},
+	})
+	if err := orch.cordonNodeWithLease(ctx, "ok", "", strings.Repeat("detail ", 80)); err != nil {
+		t.Fatalf("expected cordon lease success: %v", err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{SSHManagedNodes: []string{"crypt"}}, []commandStub{
+		{match: matchContains("ssh", "command -v cryptsetup"), out: "__ANANKE_CRYPTSETUP_NO_APT__", err: errors.New("exit 42")},
+	})
+	if recovered, err := orch.recoverLeasedCordon(ctx, "crypt", map[string]string{anankeCordonReasonAnnotation: cordonReasonMissingCryptsetup}); err == nil || recovered {
+		t.Fatalf("expected cryptsetup recovery to remain failed, recovered=%v err=%v", recovered, err)
+	}
+	old := time.Now().Add(-10 * time.Minute)
+	wedgedPod := closeoutPendingControllerPod("apps", "wedged", "runtime", "ReplicaSet", "CreateContainerError", old)
+	wedgedPod.Spec.Volumes = nil
+	podsJSON := fmt.Sprintf(`{"items":[{"metadata":{"namespace":"apps","name":"wedged","creationTimestamp":%q,"ownerReferences":[{"kind":"ReplicaSet","name":"rs"}]},"spec":{"nodeName":"runtime"},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CreateContainerError"}}}]}}]}`, old.Format(time.RFC3339))
+	eventsJSON := fmt.Sprintf(`{"items":[{"type":"Warning","reason":"Failed","eventTime":%q,"involvedObject":{"kind":"Pod","namespace":"apps","name":"wedged"},"message":"context deadline exceeded while creating container"}]}`, time.Now().Format(time.RFC3339))
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: podsJSON},
+		{match: matchContains("kubectl", "get", "events", "-A"), out: eventsJSON},
+	})
+	if recovered, err := orch.recoverLeasedCordon(ctx, "runtime", map[string]string{anankeCordonReasonAnnotation: cordonReasonRuntimeWedge}); err == nil || recovered {
+		t.Fatalf("expected runtime wedge to keep cordon, recovered=%v err=%v", recovered, err)
+	}
+
+	workloadJSON := `{"spec":{"template":{"spec":{"containers":[{"name":"app","livenessProbe":{"httpGet":{"path":"/healthz"}}}]}}}}`
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
+		CriticalServiceEndpoints:             []string{"apps/api"},
+		CriticalServiceEndpointWaitSec:       1,
+		CriticalServiceEndpointPollSec:       1,
+		CriticalServiceStartupProbeRepair:    true,
+		CriticalServiceStartupProbeThreshold: 2,
+	}}, []commandStub{
+		{match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), out: `{"subsets":[]}`},
+		{match: matchContains("kubectl", "-n", "apps", "scale", "deployment", "api"), err: errors.New("scale denied")},
+		{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: workloadJSON},
+		{match: matchContains("kubectl", "-n", "apps", "patch", "deployment", "api"), out: ""},
+		{match: matchContains("kubectl", "-n", "apps", "get", "statefulset", "api"), err: errors.New(`Error from server (NotFound): statefulsets.apps "api" not found`)},
+	})
+	canceled, cancel := context.WithCancel(ctx)
+	cancel()
+	if err := orch.waitForCriticalServiceEndpoints(canceled); !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected canceled critical wait after repair attempt, got %v", err)
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{
+		CriticalServiceEndpoints:          []string{"apps/api"},
+		CriticalServiceStartupProbeRepair: true,
+		AutoRecycleStuckPods:              true,
+	}}, []commandStub{
+		{match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), out: `{"subsets":[]}`},
+		{match: matchContains("kubectl", "-n", "apps", "scale", "deployment", "api"), err: errors.New("scale denied")},
+		{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: `{"spec":{"template":{"spec":{"containers":[{"name":"app"}]}}}}`},
+		{match: matchContains("kubectl", "-n", "apps", "get", "statefulset", "api"), err: errors.New(`Error from server (NotFound): statefulsets.apps "api" not found`)},
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: `{"items":[]}`},
+		{match: matchContains("kubectl", "get", "events", "-A"), out: `{"items":[]}`},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: ""},
+		{match: matchContains("kubectl", "get", "pvc", "-A"), out: ""},
+	})
+	if _, err := orch.healUnreadyConfiguredServiceBackends(ctx); err == nil {
+		t.Fatalf("expected heal error when probe repair finds no containers")
+	}
+}
+
+// TestRecycleDecisionCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestRecycleDecisionCloseoutBranches(t *testing.T).
+// Why: pod recycling must skip external blockers and force-delete only the
+// stale-owner cases that have enough storage handoff evidence.
+func TestRecycleDecisionCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	old := time.Now().Add(-10 * time.Minute)
+	dnsPod := closeoutPendingControllerPod("apps", "dns", "node-a", "ReplicaSet", "ImagePullBackOff", old)
+	credPod := closeoutPendingControllerPod("apps", "cred", "node-a", "ReplicaSet", "ErrImagePull", old)
+	unsafeOld := closeoutDeletingRWOPod("apps", "unsafe-old", "node-a", old, true)
+	unsafeNew := closeoutPendingControllerPod("apps", "unsafe-new", "node-b", "ReplicaSet", "ContainerCreating", old)
+	forceOld := closeoutDeletingRWOPod("apps", "force-old", "node-a", old, false)
+	forceNew := closeoutPendingControllerPod("apps", "force-new", "node-b", "ReplicaSet", "ContainerCreating", old)
+	staleDeleting := closeoutDeletingRWOPod("apps", "stale-delete", "node-a", old, false)
+	staleDeleting.Spec.Volumes = nil
+	deleteFail := closeoutPendingControllerPod("apps", "delete-fail", "node-a", "ReplicaSet", "CrashLoopBackOff", old)
+	deleteFail.Spec.Volumes = nil
+	podsRaw, err := json.Marshal(podList{Items: []podResource{dnsPod, credPod, unsafeOld, unsafeNew, forceOld, forceNew, staleDeleting, deleteFail}})
+	if err != nil {
+		t.Fatalf("marshal pods: %v", err)
+	}
+	events := eventList{Items: []eventResource{
+		{Type: "Warning", Reason: "Failed", Message: "failed to pull image: lookup registry.local: no such host"},
+		{Type: "Warning", Reason: "FailedPull", Message: "unauthorized: authentication required"},
+		{Type: "Warning", Reason: "FailedAttachVolume", Message: "Multi-Attach error for unsafe"},
+		{Type: "Warning", Reason: "FailedAttachVolume", Message: "Multi-Attach error for force"},
+	}}
+	events.Items[0].InvolvedObject.Kind = "Pod"
+	events.Items[0].InvolvedObject.Namespace = "apps"
+	events.Items[0].InvolvedObject.Name = "dns"
+	events.Items[1].InvolvedObject.Kind = "Pod"
+	events.Items[1].InvolvedObject.Namespace = "apps"
+	events.Items[1].InvolvedObject.Name = "cred"
+	events.Items[2].InvolvedObject.Kind = "Pod"
+	events.Items[2].InvolvedObject.Namespace = "apps"
+	events.Items[2].InvolvedObject.Name = "unsafe-new"
+	events.Items[3].InvolvedObject.Kind = "Pod"
+	events.Items[3].InvolvedObject.Namespace = "apps"
+	events.Items[3].InvolvedObject.Name = "force-new"
+	eventsRaw, err := json.Marshal(events)
+	if err != nil {
+		t.Fatalf("marshal events: %v", err)
+	}
+	pvcJSON := `{"items":[{"metadata":{"namespace":"apps","name":"data"},"spec":{"accessModes":["ReadWriteOnce"]}}]}`
+	orch := buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{StuckPodGraceSeconds: 1}}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A"), out: string(podsRaw)},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: ""},
+		{match: matchContains("kubectl", "get", "events", "-A"), out: string(eventsRaw)},
+		{match: matchContains("kubectl", "get", "pvc", "-A"), out: pvcJSON},
+		{match: matchContains("kubectl", "-n", "apps", "delete", "pod", "delete-fail"), err: errors.New("delete denied")},
+		{match: matchContains("kubectl", "-n", "apps", "delete", "pod"), out: ""},
+	})
+	if err := orch.recycleStuckControllerPods(ctx); err != nil {
+		t.Fatalf("recycleStuckControllerPods: %v", err)
+	}
+}
+
+// TestSmallCoordinatorErrorCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestSmallCoordinatorErrorCloseoutBranches(t *testing.T).
+// Why: small coordinator branches should stay covered even when they are only
+// reached by aggregated repair failures.
+func TestSmallCoordinatorErrorCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	cfg := config.Config{Startup: config.Startup{CriticalServiceEndpoints: []string{"apps/api"}}}
+	orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), err: errors.New("longhorn down")},
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: "{"},
+		{match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"},
+		{match: matchContains("kubectl", "-n", "apps", "get", "endpoints", "api"), err: errors.New("endpoints down")},
+		{match: matchContains("kubectl", "get", "events", "-A"), err: errors.New("events down")},
+	})
+	if err := orch.postStartAutoHeal(ctx); err == nil {
+		t.Fatalf("expected post-start auto-heal aggregation error")
+	}
+	if healthy, err := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "--raw", "/api/v1/nodes/flaky/proxy/healthz"), err: errors.New("i/o timeout")},
+	}).kubeletProxyHealthy(ctx, "flaky"); healthy || err == nil {
+		t.Fatalf("expected transient kubelet proxy retries to fail, healthy=%v err=%v", healthy, err)
+	}
+	if isTransientKubeletProxyCheckErr(nil) {
+		t.Fatalf("nil error is not transient")
+	}
+
+	if repaired, err := buildOrchestratorWithStubs(t, config.Config{}, nil).maybeRepairCriticalBackendStartupProbes(ctx, "apps", "api"); err != nil || repaired != nil {
+		t.Fatalf("disabled startup probe repair should no-op, repaired=%v err=%v", repaired, err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{Startup: config.Startup{CriticalServiceStartupProbeRepair: true}}, []commandStub{
+		{match: matchContains("kubectl", "-n", "apps", "get", "deployment", "api"), out: "server said no", err: errors.New("get denied")},
+		{match: matchContains("kubectl", "-n", "apps", "get", "statefulset", "api"), out: "server said no", err: errors.New("get denied")},
+	})
+	if _, err := orch.maybeRepairCriticalBackendStartupProbes(ctx, "apps", "api"); err == nil {
+		t.Fatalf("expected startup probe repair aggregation error")
+	}
+
+	if err := buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "annotate", "node", "manual"), err: errors.New("annotate denied")},
+	}).clearCordonLease(ctx, "manual"); err == nil {
+		t.Fatalf("expected clear cordon lease error")
+	}
+	buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "annotate", "node", "manual"), err: errors.New("annotate denied")},
+	}).markManualActionRequired(ctx, "manual", "operator please")
+}
+
+// TestFinalSmallCoverageCloseoutBranches runs one orchestration or CLI step.
+// Signature: TestFinalSmallCoverageCloseoutBranches(t *testing.T).
+// Why: final near-threshold files need concise coverage for rare but important
+// recovery error paths.
+func TestFinalSmallCoverageCloseoutBranches(t *testing.T) {
+	ctx := context.Background()
+	oldDelay := etcdRestorePostStartDelay
+	etcdRestorePostStartDelay = 0
+	t.Cleanup(func() { etcdRestorePostStartDelay = oldDelay })
+	hash := strings.Repeat("a", 64)
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		ControlPlanes:   []string{"cp1", "cp2"},
+		SSHManagedNodes: []string{"cp1", "cp2"},
+	}, []commandStub{
+		{match: matchContains("ssh", "sudo systemctl cat k3s"), out: "ExecStart=/usr/local/bin/k3s server"},
+		{match: matchContains("ssh", "test -s '/snap.db'"), out: "2097152"},
+		{match: matchContains("ssh", "etcd-snapshot ls"), out: "/snap.db"},
+		{match: matchContains("ssh", "sha256sum '/snap.db'"), out: hash},
+		{match: matchContains("ssh", "sudo systemctl stop k3s"), out: ""},
+		{match: matchContains("ssh", "server --cluster-reset --cluster-reset-restore-path /snap.db"), out: ""},
+		{match: matchContains("ssh", "sudo systemctl start k3s"), out: ""},
+	})
+	if err := orch.EtcdRestore(ctx, EtcdRestoreOptions{ControlPlane: "cp1", SnapshotPath: "/snap.db"}); err != nil {
+		t.Fatalf("expected etcd restore success: %v", err)
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: ""},
+	})
+	if repaired, err := orch.reconcileLonghornKubernetesReadiness(ctx); err != nil || repaired != 0 {
+		t.Fatalf("empty longhorn nodes should no-op, repaired=%d err=%v", repaired, err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io", "-o", "json"), out: `{"items":[{"metadata":{"name":"n"},"status":{"conditions":[{"type":"Ready","status":"False","reason":"ManagerPodMissing"}]}}]}`},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "daemonset", "longhorn-manager"), out: `{"spec":{"selector":{"matchLabels":{}}}}`},
+	})
+	if _, err := orch.reconcileLonghornKubernetesReadiness(ctx); err == nil {
+		t.Fatalf("expected empty longhorn selector error")
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{
+		SSHManagedNodes: []string{"managed"},
+		Startup: config.Startup{
+			IgnoreUnavailableNodes:        []string{"ignored"},
+			LonghornCryptsetupExemptNodes: []string{"exempt"},
+			RequiredNodeLabels:            map[string]map[string]string{"fallback": {"longhorn-host": "true"}},
+		},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: "ignored\nexempt\nunmanaged\n"},
+		{match: matchContains("kubectl", "annotate", "node", "unmanaged"), out: ""},
+		{match: matchContains("kubectl", "cordon", "unmanaged"), out: ""},
+	})
+	guarded, err := orch.ensureLonghornEncryptedHostPrereqs(ctx, []string{"ignored", "exempt", "unmanaged", "managed"})
+	if err != nil || strings.Join(guarded, ",") != "ignored,exempt,managed" {
+		t.Fatalf("unexpected longhorn prereq guard guarded=%v err=%v", guarded, err)
+	}
+
+	runtimeCfg := config.Config{SSHManagedNodes: []string{"node"}, Startup: config.Startup{NodeRuntimeRestartWaitSeconds: 1}}
+	orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
+		{match: matchContains("ssh", "systemctl --no-block restart k3s-agent"), out: ""},
+		{match: matchContains("kubectl", "wait", "node/node"), err: errors.New("not ready")},
+		{match: matchContains("ssh", "systemctl show k3s-agent"), err: errors.New("show failed")},
+	})
+	if err := orch.recoverManagedNodeRuntime(ctx, "node", true, "test"); err == nil || !strings.Contains(err.Error(), "state check failed") {
+		t.Fatalf("expected runtime show error, got %v", err)
+	}
+	orch = buildOrchestratorWithStubs(t, runtimeCfg, []commandStub{
+		{match: matchContains("ssh", "systemctl --no-block restart k3s-agent"), out: ""},
+		{match: matchContains("kubectl", "wait", "node/node"), err: errors.New("not ready")},
+		{match: matchContains("ssh", "systemctl show k3s-agent"), out: "ActiveState=deactivating"},
+	})
+	if err := orch.recoverManagedNodeRuntime(ctx, "node", true, "test"); err == nil || !strings.Contains(err.Error(), "host_repair_allow_reboot is false") {
+		t.Fatalf("expected reboot disallowed error, got %v", err)
+	}
+
+	orch = buildOrchestratorWithStubs(t, config.Config{}, []commandStub{
+		{match: matchContains("kubectl", "get", "events", "-A"), out: "{"},
+	})
+	if _, err := orch.imagePullCredentialBlockerReasons(ctx); err == nil {
+		t.Fatalf("expected image-pull credential event decode error")
+	}
+	dry := buildOrchestratorWithStubs(t, config.Config{}, nil)
+	dry.runner.DryRun = true
+	if repaired, err := dry.healImagePullCredentialSync(ctx); err != nil || repaired != nil {
+		t.Fatalf("dry-run image credential sync should no-op, repaired=%v err=%v", repaired, err)
+	}
+	orch = buildOrchestratorWithStubs(t, config.Config{}, nil)
+	orch.runSensitiveOverride = nil
+	orch.runner.Kubeconfig = t.TempDir() + "/kubeconfig"
+	if out, err := orch.runSensitive(ctx, time.Second, "sh", "-c", "echo ok"); err != nil || out != "ok" {
+		t.Fatalf("expected runSensitive kubeconfig success, out=%q err=%v", out, err)
+	}
+	pod := closeoutPendingControllerPod("apps", "app", "node", "ReplicaSet", "ContainerCreating", time.Now())
+	pod.Spec.Volumes = []podVolume{{Name: "empty", PersistentVolumeClaim: &podPersistentVolumeClaim{ClaimName: " "}}}
+	if got := podRWOPVCVolumeNames(pod, nil); len(got) != 0 {
+		t.Fatalf("blank claim should not map, got %v", got)
+	}
+}

internal/cluster/orchestrator_scaling.go

@@ -18,13 +18,13 @@ import (
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
 func (o *Orchestrator) effectiveWorkers(ctx context.Context) ([]string, error) {
 	if len(o.cfg.Workers) > 0 {
-		return append([]string{}, o.cfg.Workers...), nil
+		return filterIgnoredNodes(o.cfg.Workers, o.cfg.Startup.IgnoreUnavailableNodes), nil
 	}
 	workers, err := o.discoverWorkers(ctx)
 	if err == nil {
-		return workers, nil
+		return filterIgnoredNodes(workers, o.cfg.Startup.IgnoreUnavailableNodes), nil
 	}
-	fallback := o.fallbackWorkersFromInventory()
+	fallback := filterIgnoredNodes(o.fallbackWorkersFromInventory(), o.cfg.Startup.IgnoreUnavailableNodes)
 	if len(fallback) == 0 {
 		return nil, err
 	}
@@ -32,6 +32,27 @@ func (o *Orchestrator) effectiveWorkers(ctx context.Context) ([]string, error) {
 	return fallback, nil
 }
 
+// filterIgnoredNodes runs one orchestration or CLI step.
+// Signature: filterIgnoredNodes(nodes []string, ignored []string) []string.
+// Why: ignored unavailable nodes should be excluded from active startup and
+// shutdown worker operations, not just readiness gates, so known-absent
+// hardware does not create noisy SSH or uncordon attempts.
+func filterIgnoredNodes(nodes []string, ignored []string) []string {
+	ignoredSet := makeStringSet(ignored)
+	filtered := []string{}
+	for _, node := range nodes {
+		name := strings.TrimSpace(node)
+		if name == "" {
+			continue
+		}
+		if _, skip := ignoredSet[name]; skip {
+			continue
+		}
+		filtered = append(filtered, name)
+	}
+	return filtered
+}
+
 // discoverWorkers runs one orchestration or CLI step.
 // Signature: (o *Orchestrator) discoverWorkers(ctx context.Context) ([]string, error).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

internal/cluster/orchestrator_scheduling_storm.go

@@ -0,0 +1,261 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+// maybeAutoQuarantineSchedulingStorms runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) maybeAutoQuarantineSchedulingStorms(ctx context.Context, lastAttempt *time.Time).
+// Why: a non-core workload that cannot schedule can emit enough warning events to
+// thrash the control plane datastore; quarantine keeps startup moving while
+// preserving core services.
+func (o *Orchestrator) maybeAutoQuarantineSchedulingStorms(ctx context.Context, lastAttempt *time.Time) {
+	if o.runner.DryRun || !o.cfg.Startup.AutoQuarantineSchedulingStorms {
+		return
+	}
+	now := time.Now()
+	if lastAttempt != nil && !lastAttempt.IsZero() && now.Sub(*lastAttempt) < 30*time.Second {
+		return
+	}
+	if lastAttempt != nil {
+		*lastAttempt = now
+	}
+	o.bestEffort("quarantine non-core scheduling storm workloads", func() error {
+		return o.quarantineSchedulingStormWorkloads(ctx)
+	})
+}
+
+// quarantineSchedulingStormWorkloads runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) quarantineSchedulingStormWorkloads(ctx context.Context) error.
+// Why: limits startup-only mitigation to workloads proven to be generating a
+// scheduling event storm, instead of scaling optional apps down blindly.
+func (o *Orchestrator) quarantineSchedulingStormWorkloads(ctx context.Context) error {
+	podsOut, err := o.kubectl(ctx, 30*time.Second, "get", "pods", "-A", "-o", "json")
+	if err != nil {
+		return fmt.Errorf("query pods for scheduling storm scan: %w", err)
+	}
+	var pods podList
+	if err := json.Unmarshal([]byte(podsOut), &pods); err != nil {
+		return fmt.Errorf("decode pods for scheduling storm scan: %w", err)
+	}
+
+	rsOut, err := o.kubectl(ctx, 30*time.Second, "get", "replicasets", "-A", "-o", "json")
+	if err != nil {
+		return fmt.Errorf("query replicasets for scheduling storm scan: %w", err)
+	}
+	var rsList replicaSetList
+	if err := json.Unmarshal([]byte(rsOut), &rsList); err != nil {
+		return fmt.Errorf("decode replicasets for scheduling storm scan: %w", err)
+	}
+
+	eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
+	if err != nil {
+		return fmt.Errorf("query events for scheduling storm scan: %w", err)
+	}
+	var events eventList
+	if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
+		return fmt.Errorf("decode events for scheduling storm scan: %w", err)
+	}
+
+	workloadsOut, err := o.kubectl(ctx, 30*time.Second, "get", "deploy,statefulset", "-A", "-o", "json")
+	if err != nil {
+		return fmt.Errorf("query workloads for scheduling storm scan: %w", err)
+	}
+	var workloads workloadList
+	if err := json.Unmarshal([]byte(workloadsOut), &workloads); err != nil {
+		return fmt.Errorf("decode workloads for scheduling storm scan: %w", err)
+	}
+
+	requiredNamespaces := o.startupRequiredWorkloadNamespaces()
+	ignoredNamespaces := makeStringSet(o.cfg.Startup.IgnoreWorkloadNamespaces)
+	ignoredNodes := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
+	ignoreRules := parseWorkloadIgnoreRules(o.cfg.Startup.IgnoreWorkloads)
+	eventThreshold := o.cfg.Startup.SchedulingStormEventThreshold
+	if eventThreshold <= 0 {
+		eventThreshold = 30
+	}
+	window := time.Duration(o.cfg.Startup.SchedulingStormWindowSeconds) * time.Second
+	if window <= 0 {
+		window = 3 * time.Minute
+	}
+
+	podsByKey := map[string]podResource{}
+	for _, pod := range pods.Items {
+		ns := strings.TrimSpace(pod.Metadata.Namespace)
+		name := strings.TrimSpace(pod.Metadata.Name)
+		if ns == "" || name == "" {
+			continue
+		}
+		podsByKey[ns+"/"+name] = pod
+	}
+
+	rsOwners := map[string]ownerReference{}
+	for _, rs := range rsList.Items {
+		ns := strings.TrimSpace(rs.Metadata.Namespace)
+		name := strings.TrimSpace(rs.Metadata.Name)
+		if ns == "" || name == "" {
+			continue
+		}
+		for _, owner := range rs.Metadata.OwnerReferences {
+			kind := strings.TrimSpace(owner.Kind)
+			ownerName := strings.TrimSpace(owner.Name)
+			if kind == "" || ownerName == "" {
+				continue
+			}
+			rsOwners[ns+"/"+name] = owner
+			break
+		}
+	}
+
+	workloadDesired := map[string]int32{}
+	for _, item := range workloads.Items {
+		kind := strings.ToLower(strings.TrimSpace(item.Kind))
+		ns := strings.TrimSpace(item.Metadata.Namespace)
+		name := strings.TrimSpace(item.Metadata.Name)
+		if kind == "" || ns == "" || name == "" {
+			continue
+		}
+		desired, _, ok := desiredReady(item)
+		if !ok {
+			continue
+		}
+		workloadDesired[ns+"/"+kind+"/"+name] = desired
+	}
+
+	quarantined := []string{}
+	seen := map[string]struct{}{}
+	now := time.Now()
+	for _, event := range events.Items {
+		if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
+			continue
+		}
+		if strings.TrimSpace(event.Reason) != "FailedScheduling" {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
+			continue
+		}
+		lastSeen := eventLastObservedAt(event)
+		if !lastSeen.IsZero() && now.Sub(lastSeen) > window {
+			continue
+		}
+		count := eventObservationCount(event)
+		if count < eventThreshold {
+			continue
+		}
+		podKey := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name)
+		pod, ok := podsByKey[podKey]
+		if !ok {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(pod.Status.Phase), "Pending") {
+			continue
+		}
+		ns := strings.TrimSpace(pod.Metadata.Namespace)
+		if _, ok := requiredNamespaces[ns]; ok {
+			continue
+		}
+		if _, ok := ignoredNamespaces[ns]; ok {
+			continue
+		}
+		if workloadIgnored(ignoreRules, ns, "", strings.TrimSpace(pod.Metadata.Name)) {
+			continue
+		}
+		if podTargetsIgnoredNode(pod, ignoredNodes) {
+			continue
+		}
+		workload, ok := schedulingStormOwnerWorkload(pod, rsOwners)
+		if !ok {
+			continue
+		}
+		if workloadIgnored(ignoreRules, workload.Namespace, workload.Kind, workload.Name) {
+			continue
+		}
+		workloadKey := workload.Namespace + "/" + workload.Kind + "/" + workload.Name
+		if _, done := seen[workloadKey]; done {
+			continue
+		}
+		desired := workloadDesired[workloadKey]
+		if desired <= 0 {
+			continue
+		}
+		if err := o.ensureWorkloadReplicas(ctx, workload, 0); err != nil {
+			return fmt.Errorf("scale scheduling storm workload %s to 0: %w", workloadKey, err)
+		}
+		seen[workloadKey] = struct{}{}
+		quarantined = append(quarantined, fmt.Sprintf("%s events=%d window=%s", workloadKey, count, window))
+	}
+
+	if len(quarantined) == 0 {
+		return nil
+	}
+	sort.Strings(quarantined)
+	detail := fmt.Sprintf("quarantined scheduling storm workload(s): %s", joinLimited(quarantined, 8))
+	o.log.Printf("%s", detail)
+	o.noteStartupAutoHeal(detail)
+	return nil
+}
+
+// schedulingStormOwnerWorkload runs one orchestration or CLI step.
+// Signature: schedulingStormOwnerWorkload(pod podResource, rsOwners map[string]ownerReference) (startupWorkload, bool).
+// Why: scheduling storms happen at the pod layer, but safe mitigation needs to
+// operate on the owning deployment or statefulset.
+func schedulingStormOwnerWorkload(pod podResource, rsOwners map[string]ownerReference) (startupWorkload, bool) {
+	ns := strings.TrimSpace(pod.Metadata.Namespace)
+	for _, owner := range pod.Metadata.OwnerReferences {
+		switch strings.TrimSpace(owner.Kind) {
+		case "StatefulSet":
+			if name := strings.TrimSpace(owner.Name); name != "" {
+				return startupWorkload{Namespace: ns, Kind: "statefulset", Name: name}, true
+			}
+		case "ReplicaSet":
+			rsName := strings.TrimSpace(owner.Name)
+			if rsName == "" {
+				continue
+			}
+			rsOwner, ok := rsOwners[ns+"/"+rsName]
+			if !ok || strings.TrimSpace(rsOwner.Kind) != "Deployment" || strings.TrimSpace(rsOwner.Name) == "" {
+				continue
+			}
+			return startupWorkload{Namespace: ns, Kind: "deployment", Name: strings.TrimSpace(rsOwner.Name)}, true
+		}
+	}
+	return startupWorkload{}, false
+}
+
+// eventObservationCount runs one orchestration or CLI step.
+// Signature: eventObservationCount(event eventResource) int.
+// Why: event count can live either on the root event or in the series payload;
+// using the max keeps detection stable across Kubernetes versions.
+func eventObservationCount(event eventResource) int {
+	count := event.Count
+	if event.Series.Count > count {
+		count = event.Series.Count
+	}
+	if count < 1 {
+		return 1
+	}
+	return count
+}
+
+// eventLastObservedAt runs one orchestration or CLI step.
+// Signature: eventLastObservedAt(event eventResource) time.Time.
+// Why: event recency fields vary by cluster version; prefer the newest explicit
+// observation time and fall back to creation time when needed.
+func eventLastObservedAt(event eventResource) time.Time {
+	switch {
+	case !event.Series.LastObservedTime.IsZero():
+		return event.Series.LastObservedTime
+	case !event.LastTimestamp.IsZero():
+		return event.LastTimestamp
+	case !event.EventTime.IsZero():
+		return event.EventTime
+	default:
+		return event.Metadata.CreationTimestamp
+	}
+}

internal/cluster/orchestrator_service_stability.go

@@ -99,6 +99,7 @@ func (o *Orchestrator) waitForServiceChecklist(ctx context.Context) error {
 	lastRecycleAttempt := time.Time{}
 	lastReplicaHeal := time.Time{}
 	lastIngressHeal := time.Time{}
+	lastTCPHeal := time.Time{}
 	for {
 		o.maybeAutoRecycleStuckPods(ctx, &lastRecycleAttempt)
 		o.maybeAutoHealCriticalWorkloadReplicas(ctx, &lastReplicaHeal)
@@ -110,6 +111,7 @@ func (o *Orchestrator) waitForServiceChecklist(ctx context.Context) error {
 			return nil
 		}
 		o.maybeAutoHealIngressHostBackends(ctx, &lastIngressHeal, lastFailure)
+		o.maybeAutoHealTCPServiceBackends(ctx, &lastTCPHeal)
 		if lastFailure != prevFailure || time.Since(lastLogged) >= 30*time.Second {
 			remaining := time.Until(deadline).Round(time.Second)
 			if remaining < 0 {
@@ -134,7 +136,7 @@ func (o *Orchestrator) waitForServiceChecklist(ctx context.Context) error {
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
 func (o *Orchestrator) serviceChecklistReady(ctx context.Context) (bool, string) {
 	checks := o.cfg.Startup.ServiceChecklist
-	if len(checks) == 0 {
+	if len(checks) == 0 && len(o.cfg.Startup.TCPServiceChecklist) == 0 {
 		return true, "no checklist items configured"
 	}
 	for _, check := range checks {
@@ -147,7 +149,10 @@ func (o *Orchestrator) serviceChecklistReady(ctx context.Context) (bool, string)
 			return false, fmt.Sprintf("%s: %s", name, detail)
 		}
 	}
-	return true, fmt.Sprintf("checks=%d", len(checks))
+	if ok, detail := o.tcpServiceChecklistReady(ctx); !ok {
+		return false, "tcp " + detail
+	}
+	return true, fmt.Sprintf("checks=%d tcp-checks=%d", len(checks), len(o.cfg.Startup.TCPServiceChecklist))
 }
 
 // serviceCheckReady runs one orchestration or CLI step.
@@ -341,8 +346,14 @@ func (o *Orchestrator) waitForStabilityWindow(ctx context.Context) error {
 	if poll <= 0 {
 		poll = 5 * time.Second
 	}
-	deadline := time.Now().Add(window)
+	maxWait := window * 3
+	if maxWait < window+poll {
+		maxWait = window + poll
+	}
+	deadline := time.Now().Add(maxWait)
+	stableSince := time.Time{}
 	lastStatus := time.Time{}
+	lastFailure := ""
 	lastRecycleAttempt := time.Time{}
 	lastReplicaHeal := time.Time{}
 
@@ -350,19 +361,36 @@ func (o *Orchestrator) waitForStabilityWindow(ctx context.Context) error {
 		o.maybeAutoRecycleStuckPods(ctx, &lastRecycleAttempt)
 		o.maybeAutoHealCriticalWorkloadReplicas(ctx, &lastReplicaHeal)
 		if err := o.startupStabilityHealthy(ctx); err != nil {
-			return fmt.Errorf("startup stability window failed: %w", err)
+			lastFailure = err.Error()
-		}
+			stableSince = time.Time{}
-		if time.Now().After(deadline) {
+			if time.Now().After(deadline) {
-			o.log.Printf("startup stability window passed (%s)", window)
+				return fmt.Errorf("startup stability window failed after %s: %s", maxWait, lastFailure)
-			return nil
+			}
-		}
+			if time.Since(lastStatus) >= 30*time.Second {
-		if time.Since(lastStatus) >= 30*time.Second {
+				remaining := time.Until(deadline).Round(time.Second)
-			remaining := time.Until(deadline).Round(time.Second)
+				if remaining < 0 {
-			if remaining < 0 {
+					remaining = 0
-				remaining = 0
+				}
+				o.log.Printf("startup stability soak reset (%s remaining): %s", remaining, lastFailure)
+				lastStatus = time.Now()
+			}
+		} else {
+			if stableSince.IsZero() {
+				stableSince = time.Now()
+			}
+			stableFor := time.Since(stableSince)
+			if stableFor >= window {
+				o.log.Printf("startup stability window passed (%s)", window)
+				return nil
+			}
+			if time.Since(lastStatus) >= 30*time.Second {
+				remaining := (window - stableFor).Round(time.Second)
+				if remaining < 0 {
+					remaining = 0
+				}
+				o.log.Printf("startup stability soak in progress (%s remaining)", remaining)
+				lastStatus = time.Now()
 			}
-			o.log.Printf("startup stability soak in progress (%s remaining)", remaining)
-			lastStatus = time.Now()
 		}
 		select {
 		case <-ctx.Done():

internal/cluster/orchestrator_shutdown.go

@@ -0,0 +1,82 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/state"
+)
+
+// Shutdown runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) Shutdown(ctx context.Context, opts ShutdownOptions) (err error).
+// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
+func (o *Orchestrator) Shutdown(ctx context.Context, opts ShutdownOptions) (err error) {
+	unlock, err := state.AcquireLock(o.cfg.State.LockPath)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+	if invErr := o.validateNodeInventory(); invErr != nil {
+		return invErr
+	}
+
+	record := state.RunRecord{
+		ID:        fmt.Sprintf("shutdown-%d", time.Now().UnixNano()),
+		Action:    "shutdown",
+		Reason:    opts.Reason,
+		DryRun:    o.runner.DryRun,
+		StartedAt: time.Now().UTC(),
+	}
+	defer o.finalizeRecord(&record, &err)
+	if !o.runner.DryRun {
+		if writeErr := state.MustWriteIntent(o.cfg.State.IntentPath, state.IntentShuttingDown, opts.Reason, "shutdown"); writeErr != nil {
+			return fmt.Errorf("set shutdown intent: %w", writeErr)
+		}
+		defer func() {
+			final := state.IntentShuttingDown
+			if err == nil {
+				final = state.IntentShutdownComplete
+			}
+			if writeErr := state.MustWriteIntent(o.cfg.State.IntentPath, final, opts.Reason, "shutdown"); writeErr != nil {
+				o.log.Printf("warning: write shutdown completion intent failed: %v", writeErr)
+			}
+		}()
+	}
+
+	workers, err := o.effectiveWorkers(ctx)
+	if err != nil {
+		return err
+	}
+	o.log.Printf("shutdown control-planes=%s workers=%s", strings.Join(o.cfg.ControlPlanes, ","), strings.Join(workers, ","))
+
+	o.reportFluxSource(ctx, "")
+
+	skipEtcd := opts.SkipEtcdSnapshot || o.cfg.Shutdown.SkipEtcdSnapshot
+	if !skipEtcd {
+		o.bestEffort("etcd snapshot", func() error {
+			return o.takeEtcdSnapshot(ctx, o.cfg.ControlPlanes[0])
+		})
+	}
+
+	o.bestEffort("suspend flux", func() error { return o.patchFluxSuspendAll(ctx, true) })
+	o.bestEffort("scale down apps", func() error { return o.scaleDownApps(ctx) })
+
+	skipDrain := opts.SkipDrain || o.cfg.Shutdown.SkipDrain
+	if !skipDrain {
+		o.bestEffort("drain workers", func() error { return o.drainWorkers(ctx, workers) })
+	}
+
+	shutdownMode := strings.TrimSpace(opts.Mode)
+	effectiveMode, modeErr := normalizeShutdownMode(shutdownMode)
+	if modeErr != nil {
+		return modeErr
+	}
+	o.log.Printf("shutdown execution mode=%s (requested=%q)", effectiveMode, shutdownMode)
+
+	o.stopWorkers(ctx, workers)
+	o.stopControlPlanes(ctx, o.cfg.ControlPlanes)
+	o.log.Printf("shutdown flow complete")
+	return nil
+}

internal/cluster/orchestrator_shutdown_mode.go

@@ -0,0 +1,21 @@
+package cluster
+
+import (
+	"fmt"
+	"strings"
+)
+
+// normalizeShutdownMode runs one orchestration or CLI step.
+// Signature: normalizeShutdownMode(raw string) (string, error).
+// Why: keeps shutdown behavior explicit and safe by allowing only cluster-only
+// semantics while preserving compatibility with legacy "config" callers.
+func normalizeShutdownMode(raw string) (string, error) {
+	switch strings.TrimSpace(raw) {
+	case "", "config", "cluster-only":
+		return "cluster-only", nil
+	case "poweroff":
+		return "", fmt.Errorf("shutdown mode %q has been removed; ananke no longer powers off hosts", raw)
+	default:
+		return "", fmt.Errorf("unsupported shutdown mode %q (expected config|cluster-only)", raw)
+	}
+}

internal/cluster/orchestrator_stale_rwo_owner.go

@@ -0,0 +1,319 @@
+package cluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+type staleRWOOwnerDecision struct {
+	Reason      string
+	ForceDelete bool
+	Unsafe      bool
+}
+
+type persistentVolumeClaimList struct {
+	Items []persistentVolumeClaimResource `json:"items"`
+}
+
+type persistentVolumeClaimResource struct {
+	Metadata struct {
+		Namespace string `json:"namespace"`
+		Name      string `json:"name"`
+	} `json:"metadata"`
+	Spec struct {
+		AccessModes []string `json:"accessModes"`
+		VolumeName  string   `json:"volumeName"`
+	} `json:"spec"`
+}
+
+// staleRWOPVCOwnerDecisions finds stale terminating single-writer pods safely.
+// Signature: (o *Orchestrator) staleRWOPVCOwnerDecisions(ctx context.Context, pods podList, grace time.Duration) (map[string]staleRWOOwnerDecision, error).
+// Why: RWO volume handoff needs container/mount-level safety, not blind repeated
+// pod deletion, so sidecar-only stale owners can clear while live writers block.
+func (o *Orchestrator) staleRWOPVCOwnerDecisions(ctx context.Context, pods podList, grace time.Duration) (map[string]staleRWOOwnerDecision, error) {
+	eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query events for stale RWO owner scan: %w", err)
+	}
+	var events eventList
+	if strings.TrimSpace(eventsOut) != "" {
+		if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
+			return nil, fmt.Errorf("decode events for stale RWO owner scan: %w", err)
+		}
+	}
+	pvcs, err := o.queryPVCs(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	now := time.Now()
+	podsByController := map[string][]podResource{}
+	for _, pod := range pods.Items {
+		controllerKey := podControllerKey(pod)
+		if controllerKey == "" {
+			continue
+		}
+		podsByController[controllerKey] = append(podsByController[controllerKey], pod)
+	}
+
+	decisions := map[string]staleRWOOwnerDecision{}
+	for _, oldPod := range pods.Items {
+		oldKey := podKey(oldPod)
+		if oldKey == "" || oldPod.Metadata.DeletionTimestamp == nil || !podControllerOwned(oldPod) {
+			continue
+		}
+		if now.Sub(*oldPod.Metadata.DeletionTimestamp) < grace {
+			continue
+		}
+		oldNode := strings.TrimSpace(oldPod.Spec.NodeName)
+		if oldNode == "" {
+			continue
+		}
+		blockedClaims := podRWOPVCVolumeNames(oldPod, pvcs)
+		if len(blockedClaims) == 0 {
+			continue
+		}
+		controllerKey := podControllerKey(oldPod)
+		replacements := podsByController[controllerKey]
+		replacementKey := ""
+		for _, replacement := range replacements {
+			if podKey(replacement) == oldKey {
+				continue
+			}
+			if strings.TrimSpace(replacement.Spec.NodeName) == "" || strings.TrimSpace(replacement.Spec.NodeName) == oldNode {
+				continue
+			}
+			if !replacementPodPendingForAttach(replacement) {
+				continue
+			}
+			if !podHasBlockedAttachEvent(replacement, events) {
+				continue
+			}
+			if !podsShareAnyClaim(oldPod, replacement, blockedClaims) {
+				continue
+			}
+			replacementKey = podKey(replacement)
+			break
+		}
+		if replacementKey == "" {
+			continue
+		}
+
+		unsafe := runningContainersMountAnyVolume(oldPod, blockedClaims)
+		claims := make([]string, 0, len(blockedClaims))
+		for _, claim := range blockedClaims {
+			claims = append(claims, claim)
+		}
+		sort.Strings(claims)
+		if unsafe {
+			decisions[oldKey] = staleRWOOwnerDecision{
+				Reason: fmt.Sprintf("UnsafeStaleRWOPVCOwner:%s:%s", oldNode, strings.Join(claims, ",")),
+				Unsafe: true,
+			}
+			continue
+		}
+		decisions[oldKey] = staleRWOOwnerDecision{
+			Reason:      fmt.Sprintf("SidecarOnlyStaleRWOPVCOwner:%s:%s->%s", oldNode, strings.Join(claims, ","), replacementKey),
+			ForceDelete: true,
+		}
+	}
+	return decisions, nil
+}
+
+// queryPVCs runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) queryPVCs(ctx context.Context) (map[string]persistentVolumeClaimResource, error).
+// Why: stale-owner recovery must confirm single-writer PVC semantics before it
+// considers force-deleting a terminating pod object.
+func (o *Orchestrator) queryPVCs(ctx context.Context) (map[string]persistentVolumeClaimResource, error) {
+	out, err := o.kubectl(ctx, 30*time.Second, "get", "pvc", "-A", "-o", "json")
+	if err != nil {
+		return nil, fmt.Errorf("query pvcs for stale RWO owner scan: %w", err)
+	}
+	pvcs := map[string]persistentVolumeClaimResource{}
+	if strings.TrimSpace(out) == "" {
+		return pvcs, nil
+	}
+	var list persistentVolumeClaimList
+	if err := json.Unmarshal([]byte(out), &list); err != nil {
+		return nil, fmt.Errorf("decode pvcs for stale RWO owner scan: %w", err)
+	}
+	for _, pvc := range list.Items {
+		key := strings.TrimSpace(pvc.Metadata.Namespace) + "/" + strings.TrimSpace(pvc.Metadata.Name)
+		if strings.TrimSpace(pvc.Metadata.Namespace) != "" && strings.TrimSpace(pvc.Metadata.Name) != "" {
+			pvcs[key] = pvc
+		}
+	}
+	return pvcs, nil
+}
+
+// podRWOPVCVolumeNames runs one orchestration or CLI step.
+// Signature: podRWOPVCVolumeNames(pod podResource, pvcs map[string]persistentVolumeClaimResource) map[string]string.
+// Why: container mount safety checks need the pod volume names that correspond
+// to ReadWriteOnce PVC claims.
+func podRWOPVCVolumeNames(pod podResource, pvcs map[string]persistentVolumeClaimResource) map[string]string {
+	volumes := map[string]string{}
+	for _, volume := range pod.Spec.Volumes {
+		if volume.PersistentVolumeClaim == nil {
+			continue
+		}
+		claim := strings.TrimSpace(volume.PersistentVolumeClaim.ClaimName)
+		if claim == "" {
+			continue
+		}
+		pvc, ok := pvcs[strings.TrimSpace(pod.Metadata.Namespace)+"/"+claim]
+		if !ok || !pvcSingleWriter(pvc) {
+			continue
+		}
+		volumes[strings.TrimSpace(volume.Name)] = claim
+	}
+	return volumes
+}
+
+// pvcSingleWriter runs one orchestration or CLI step.
+// Signature: pvcSingleWriter(pvc persistentVolumeClaimResource) bool.
+// Why: the stale-owner path is limited to exclusive-writer PVC modes and should
+// leave shared volumes to normal Kubernetes cleanup.
+func pvcSingleWriter(pvc persistentVolumeClaimResource) bool {
+	for _, mode := range pvc.Spec.AccessModes {
+		normalized := strings.ToLower(strings.TrimSpace(mode))
+		if normalized == "readwriteonce" || normalized == "readwriteoncepod" {
+			return true
+		}
+	}
+	return false
+}
+
+// podControllerKey runs one orchestration or CLI step.
+// Signature: podControllerKey(pod podResource) string.
+// Why: stale owner and replacement pods should be grouped by controller without
+// hard-coding ReplicaSet, StatefulSet, or application names.
+func podControllerKey(pod podResource) string {
+	ns := strings.TrimSpace(pod.Metadata.Namespace)
+	for _, owner := range pod.Metadata.OwnerReferences {
+		kind := strings.TrimSpace(owner.Kind)
+		name := strings.TrimSpace(owner.Name)
+		if ns != "" && kind != "" && name != "" {
+			return ns + "/" + kind + "/" + name
+		}
+	}
+	return ""
+}
+
+// podKey runs one orchestration or CLI step.
+// Signature: podKey(pod podResource) string.
+// Why: recovery incidents need stable namespace/name keys for pod maps and
+// operator summaries.
+func podKey(pod podResource) string {
+	ns := strings.TrimSpace(pod.Metadata.Namespace)
+	name := strings.TrimSpace(pod.Metadata.Name)
+	if ns == "" || name == "" {
+		return ""
+	}
+	return ns + "/" + name
+}
+
+// replacementPodPendingForAttach runs one orchestration or CLI step.
+// Signature: replacementPodPendingForAttach(pod podResource) bool.
+// Why: stale RWO recovery should only act when a replacement is actually blocked
+// during scheduling or initialization.
+func replacementPodPendingForAttach(pod podResource) bool {
+	phase := strings.TrimSpace(pod.Status.Phase)
+	if strings.EqualFold(phase, "Pending") {
+		return true
+	}
+	for _, st := range append(append([]podContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) {
+		if st.State.Waiting == nil {
+			continue
+		}
+		reason := strings.TrimSpace(st.State.Waiting.Reason)
+		if strings.EqualFold(reason, "ContainerCreating") || strings.EqualFold(reason, "PodInitializing") {
+			return true
+		}
+	}
+	return false
+}
+
+// podHasBlockedAttachEvent runs one orchestration or CLI step.
+// Signature: podHasBlockedAttachEvent(pod podResource, events eventList) bool.
+// Why: force deletion should be tied to concrete Multi-Attach or exclusive-use
+// evidence rather than any old terminating PVC pod.
+func podHasBlockedAttachEvent(pod podResource, events eventList) bool {
+	key := podKey(pod)
+	for _, event := range events.Items {
+		if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
+			continue
+		}
+		eventKey := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name)
+		if eventKey != key {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
+			continue
+		}
+		message := strings.ToLower(strings.TrimSpace(event.Message))
+		reason := strings.ToLower(strings.TrimSpace(event.Reason))
+		if reason == "failedattachvolume" ||
+			strings.Contains(message, "multi-attach") ||
+			strings.Contains(message, "volume is already used by pod") ||
+			strings.Contains(message, "already exclusively attached") {
+			return true
+		}
+	}
+	return false
+}
+
+// podsShareAnyClaim runs one orchestration or CLI step.
+// Signature: podsShareAnyClaim(oldPod podResource, replacement podResource, oldClaims map[string]string) bool.
+// Why: the old stale pod and replacement must be competing for the same PVC
+// before Ananke treats them as one storage handoff incident.
+func podsShareAnyClaim(oldPod podResource, replacement podResource, oldClaims map[string]string) bool {
+	replacementClaims := map[string]struct{}{}
+	for _, volume := range replacement.Spec.Volumes {
+		if volume.PersistentVolumeClaim == nil {
+			continue
+		}
+		claim := strings.TrimSpace(volume.PersistentVolumeClaim.ClaimName)
+		if claim != "" {
+			replacementClaims[claim] = struct{}{}
+		}
+	}
+	for _, claim := range oldClaims {
+		if _, ok := replacementClaims[claim]; ok {
+			return true
+		}
+	}
+	return false
+}
+
+// runningContainersMountAnyVolume runs one orchestration or CLI step.
+// Signature: runningContainersMountAnyVolume(pod podResource, volumeNames map[string]string) bool.
+// Why: a live app container mounting the blocked PVC is unsafe to clear, while
+// sidecars without that mount can be handled as stale API ownership.
+func runningContainersMountAnyVolume(pod podResource, volumeNames map[string]string) bool {
+	if len(volumeNames) == 0 {
+		return false
+	}
+	specByName := map[string]podContainer{}
+	for _, c := range pod.Spec.Containers {
+		specByName[strings.TrimSpace(c.Name)] = c
+	}
+	for _, status := range pod.Status.ContainerStatuses {
+		if status.State.Running == nil {
+			continue
+		}
+		spec, ok := specByName[strings.TrimSpace(status.Name)]
+		if !ok {
+			return true
+		}
+		for _, mount := range spec.VolumeMounts {
+			if _, ok := volumeNames[strings.TrimSpace(mount.Name)]; ok {
+				return true
+			}
+		}
+	}
+	return false
+}

internal/cluster/orchestrator_startup_scope.go

@@ -0,0 +1,88 @@
+package cluster
+
+import "strings"
+
+const fluxSystemSelfKustomization = "flux-system/flux-system"
+
+// startupRequiredNodes runs one orchestration or CLI step.
+// Signature: startupRequiredNodes(nodes []string, required []string) []string.
+// Why: lets startup enforce a smaller core node set during outage recovery
+// without losing the stricter all-nodes behavior when no override is configured.
+func startupRequiredNodes(nodes []string, required []string) []string {
+	requiredSet := makeStringSet(required)
+	if len(requiredSet) == 0 {
+		return nodes
+	}
+	filtered := make([]string, 0, len(nodes))
+	for _, node := range nodes {
+		node = strings.TrimSpace(node)
+		if node == "" {
+			continue
+		}
+		if _, ok := requiredSet[node]; ok {
+			filtered = append(filtered, node)
+		}
+	}
+	return filtered
+}
+
+// startupNodeStrictlyRequired runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) startupNodeStrictlyRequired(node string) bool.
+// Why: absent or broken non-core nodes should not block recovery-only actions
+// like label reconciliation once the operator has narrowed startup to core nodes.
+func (o *Orchestrator) startupNodeStrictlyRequired(node string) bool {
+	node = strings.TrimSpace(node)
+	if node == "" {
+		return false
+	}
+	if len(o.cfg.Startup.NodeInventoryReachRequiredNodes) == 0 && len(o.cfg.Startup.NodeSSHAuthRequiredNodes) == 0 {
+		return true
+	}
+	for _, controlPlane := range o.cfg.ControlPlanes {
+		if strings.TrimSpace(controlPlane) == node {
+			return true
+		}
+	}
+	if containsNode(o.cfg.Startup.NodeInventoryReachRequiredNodes, node) {
+		return true
+	}
+	return containsNode(o.cfg.Startup.NodeSSHAuthRequiredNodes, node)
+}
+
+// startupRequiredFluxKustomizations runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) startupRequiredFluxKustomizations() map[string]struct{}.
+// Why: lets outage recovery wait on a declared core GitOps slice while leaving
+// optional stacks free to converge after bootstrap succeeds.
+func (o *Orchestrator) startupRequiredFluxKustomizations() map[string]struct{} {
+	required := makeStringSet(o.cfg.Startup.FluxHealthRequiredKustomizations)
+	ignored := makeStringSet(o.cfg.Startup.IgnoreFluxKustomizations)
+	if _, ok := ignored[fluxSystemSelfKustomization]; !ok {
+		required[fluxSystemSelfKustomization] = struct{}{}
+	}
+	return required
+}
+
+// startupRequiredWorkloadNamespaces runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) startupRequiredWorkloadNamespaces() map[string]struct{}.
+// Why: keeps workload readiness scoped to core namespaces during recovery while
+// preserving broad convergence checks when no explicit core list is configured.
+func (o *Orchestrator) startupRequiredWorkloadNamespaces() map[string]struct{} {
+	return makeStringSet(o.cfg.Startup.WorkloadConvergenceRequiredNamespaces)
+}
+
+// containsNode runs one orchestration or CLI step.
+// Signature: containsNode(entries []string, needle string) bool.
+// Why: keeps node-scope checks small and explicit anywhere startup narrows its
+// recovery gates to a declared core set.
+func containsNode(entries []string, needle string) bool {
+	needle = strings.TrimSpace(needle)
+	if needle == "" {
+		return false
+	}
+	for _, entry := range entries {
+		if strings.TrimSpace(entry) == needle {
+			return true
+		}
+	}
+	return false
+}

internal/cluster/orchestrator_startup_vault.go

@@ -0,0 +1,52 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"time"
+)
+
+// maybeRunEarlyVaultUnseal runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) maybeRunEarlyVaultUnseal(ctx context.Context).
+// Why: gives startup a best-effort Vault recovery path when the API is already
+// live, without consuming the hard startup failure path before workloads recover.
+func (o *Orchestrator) maybeRunEarlyVaultUnseal(ctx context.Context) {
+	if err := o.waitForAPI(ctx, 1, time.Second); err != nil {
+		return
+	}
+
+	o.noteStartupCheckState("vault-unseal-early", "running", "best-effort early vault unseal while kubernetes api is already available")
+	deferred, detail, err := o.ensureVaultUnsealedWhenRunnable(ctx)
+	if err != nil {
+		o.log.Printf("warning: early vault unseal deferred: %v", err)
+		o.noteStartupAutoHeal(fmt.Sprintf("deferred early vault unseal: %v", err))
+		return
+	}
+	if deferred {
+		o.log.Printf("vault early unseal deferred: %s", detail)
+		o.noteStartupAutoHeal(detail)
+		return
+	}
+	o.noteStartupCheck("vault-unseal-early", true, "vault is already unsealed")
+}
+
+// runStartupVaultUnsealGate runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) runStartupVaultUnsealGate(ctx context.Context) error.
+// Why: keeps the top-level startup flow readable while allowing Vault unseal to
+// defer cleanly until critical workload recovery when the pod is not runnable yet.
+func (o *Orchestrator) runStartupVaultUnsealGate(ctx context.Context) error {
+	o.noteStartupCheckState("vault-unseal", "running", "ensuring vault is unsealed before startup gates")
+	deferred, detail, err := o.ensureVaultUnsealedWhenRunnable(ctx)
+	if err != nil {
+		o.noteStartupCheck("vault-unseal", false, err.Error())
+		return err
+	}
+	if deferred {
+		o.log.Printf("vault unseal deferred until workload recovery: %s", detail)
+		o.noteStartupAutoHeal(detail)
+		o.noteStartupCheck("vault-unseal", true, detail)
+		return nil
+	}
+	o.noteStartupCheck("vault-unseal", true, "vault is unsealed")
+	return nil
+}

internal/cluster/orchestrator_storage_types.go

@@ -177,6 +177,47 @@ type jobConditionRef struct {
 	Status string `json:"status"`
 }
 
+type eventList struct {
+	Items []eventResource `json:"items"`
+}
+
+type eventResource struct {
+	Metadata struct {
+		Namespace         string    `json:"namespace"`
+		CreationTimestamp time.Time `json:"creationTimestamp"`
+	} `json:"metadata"`
+	InvolvedObject struct {
+		UID       string `json:"uid"`
+		Kind      string `json:"kind"`
+		Namespace string `json:"namespace"`
+		Name      string `json:"name"`
+	} `json:"involvedObject"`
+	Type          string      `json:"type"`
+	Reason        string      `json:"reason"`
+	Message       string      `json:"message"`
+	Count         int         `json:"count"`
+	EventTime     time.Time   `json:"eventTime"`
+	LastTimestamp time.Time   `json:"lastTimestamp"`
+	Series        eventSeries `json:"series"`
+}
+
+type eventSeries struct {
+	Count            int       `json:"count"`
+	LastObservedTime time.Time `json:"lastObservedTime"`
+}
+
+type replicaSetList struct {
+	Items []replicaSetResource `json:"items"`
+}
+
+type replicaSetResource struct {
+	Metadata struct {
+		Namespace       string           `json:"namespace"`
+		Name            string           `json:"name"`
+		OwnerReferences []ownerReference `json:"ownerReferences"`
+	} `json:"metadata"`
+}
+
 type workloadResource struct {
 	Kind     string `json:"kind"`
 	Metadata struct {
@@ -202,10 +243,13 @@ type podList struct {
 
 type podResource struct {
 	Metadata struct {
+		UID               string            `json:"uid"`
 		Namespace         string            `json:"namespace"`
 		Name              string            `json:"name"`
+		Labels            map[string]string `json:"labels"`
 		Annotations       map[string]string `json:"annotations"`
 		CreationTimestamp time.Time         `json:"creationTimestamp"`
+		DeletionTimestamp *time.Time        `json:"deletionTimestamp"`
 		OwnerReferences   []ownerReference  `json:"ownerReferences"`
 	} `json:"metadata"`
 	Spec struct {
@@ -221,6 +265,7 @@ type podResource struct {
 
 type ownerReference struct {
 	Kind string `json:"kind"`
+	Name string `json:"name"`
 }
 
 type podContainerStatus struct {
@@ -242,8 +287,31 @@ type podContainerRunningState struct {
 }
 
 type podSpec struct {
-	NodeSelector map[string]string `json:"nodeSelector"`
+	NodeSelector   map[string]string `json:"nodeSelector"`
-	Affinity     *podAffinity      `json:"affinity"`
+	Affinity       *podAffinity      `json:"affinity"`
+	Volumes        []podVolume       `json:"volumes"`
+	Containers     []podContainer    `json:"containers"`
+	InitContainers []podContainer    `json:"initContainers"`
+}
+
+type podVolume struct {
+	Name                  string                    `json:"name"`
+	PersistentVolumeClaim *podPersistentVolumeClaim `json:"persistentVolumeClaim"`
+}
+
+type podContainer struct {
+	Name         string           `json:"name"`
+	Image        string           `json:"image"`
+	VolumeMounts []podVolumeMount `json:"volumeMounts"`
+}
+
+type podVolumeMount struct {
+	Name      string `json:"name"`
+	MountPath string `json:"mountPath"`
+}
+
+type podPersistentVolumeClaim struct {
+	ClaimName string `json:"claimName"`
 }
 
 type podAffinity struct {

internal/cluster/orchestrator_tcp_service_check.go

@@ -0,0 +1,164 @@
+package cluster
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"sort"
+	"strings"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// tcpServiceChecklistReady runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) tcpServiceChecklistReady(ctx context.Context) (bool, string).
+// Why: SMTP, IMAP, and other non-HTTP services need protocol checks alongside
+// HTTP ingress checks before startup can call the cluster healthy.
+func (o *Orchestrator) tcpServiceChecklistReady(ctx context.Context) (bool, string) {
+	checks := o.cfg.Startup.TCPServiceChecklist
+	if len(checks) == 0 {
+		return true, "no tcp checklist items configured"
+	}
+	for _, check := range checks {
+		ok, detail := o.tcpServiceCheckReady(ctx, check)
+		if !ok {
+			name := strings.TrimSpace(check.Name)
+			if name == "" {
+				name = fmt.Sprintf("%s:%d", strings.TrimSpace(check.Host), check.Port)
+			}
+			return false, fmt.Sprintf("%s: %s", name, detail)
+		}
+	}
+	return true, fmt.Sprintf("tcp-checks=%d", len(checks))
+}
+
+// tcpServiceCheckReady runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) tcpServiceCheckReady(ctx context.Context, check config.TCPServiceChecklistCheck) (bool, string).
+// Why: a single mail protocol endpoint should report whether connect, TLS, and
+// banner expectations passed without exposing any configured secret material.
+func (o *Orchestrator) tcpServiceCheckReady(ctx context.Context, check config.TCPServiceChecklistCheck) (bool, string) {
+	banner, err := tcpServiceProbe(ctx, check)
+	if err != nil {
+		return false, err.Error()
+	}
+	expected := strings.TrimSpace(check.ExpectContains)
+	if expected != "" && !checklistContains(banner, expected) {
+		return false, fmt.Sprintf("banner missing expected marker %q", expected)
+	}
+	return true, "connected"
+}
+
+// tcpServiceProbe runs one orchestration or CLI step.
+// Signature: tcpServiceProbe(ctx context.Context, check config.TCPServiceChecklistCheck) (string, error).
+// Why: protocol probes need a small direct TCP/TLS implementation so Ananke can
+// validate Mailu without shelling out to nc or openssl.
+func tcpServiceProbe(ctx context.Context, check config.TCPServiceChecklistCheck) (string, error) {
+	timeout := time.Duration(check.TimeoutSeconds) * time.Second
+	if timeout <= 0 {
+		timeout = 8 * time.Second
+	}
+	address := net.JoinHostPort(strings.TrimSpace(check.Host), fmt.Sprintf("%d", check.Port))
+	dialer := net.Dialer{Timeout: timeout}
+	conn, err := dialer.DialContext(ctx, "tcp", address)
+	if err != nil {
+		return "", fmt.Errorf("connect %s failed: %w", address, err)
+	}
+	defer conn.Close()
+	deadline := time.Now().Add(timeout)
+	_ = conn.SetDeadline(deadline)
+
+	if check.TLS {
+		tlsConn := tls.Client(conn, &tls.Config{
+			ServerName:         strings.TrimSpace(check.Host),
+			InsecureSkipVerify: check.InsecureSkipTLS,
+		})
+		if err := tlsConn.HandshakeContext(ctx); err != nil {
+			return "", fmt.Errorf("tls handshake %s failed: %w", address, err)
+		}
+		conn = tlsConn
+		_ = conn.SetDeadline(deadline)
+	}
+
+	reader := bufio.NewReader(conn)
+	bannerParts := []string{}
+	line, _ := reader.ReadString('\n')
+	if strings.TrimSpace(line) != "" {
+		bannerParts = append(bannerParts, strings.TrimSpace(line))
+	}
+	if send := check.Send; send != "" {
+		if _, err := conn.Write([]byte(send)); err != nil {
+			return strings.Join(bannerParts, "\n"), fmt.Errorf("write probe command failed: %w", err)
+		}
+		line, _ = reader.ReadString('\n')
+		if strings.TrimSpace(line) != "" {
+			bannerParts = append(bannerParts, strings.TrimSpace(line))
+		}
+	}
+	return strings.Join(bannerParts, "\n"), nil
+}
+
+// maybeAutoHealTCPServiceBackends runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) maybeAutoHealTCPServiceBackends(ctx context.Context, lastAttempt *time.Time).
+// Why: failed Mailu protocol probes should drive the same backend repair loop as
+// HTTP services when a Kubernetes service hint is configured.
+func (o *Orchestrator) maybeAutoHealTCPServiceBackends(ctx context.Context, lastAttempt *time.Time) {
+	if o.runner.DryRun || len(o.cfg.Startup.TCPServiceChecklist) == 0 {
+		return
+	}
+	now := time.Now()
+	if lastAttempt != nil && !lastAttempt.IsZero() && now.Sub(*lastAttempt) < 45*time.Second {
+		return
+	}
+	if lastAttempt != nil {
+		*lastAttempt = now
+	}
+	healed, err := o.healFailedTCPServiceBackends(ctx)
+	if err != nil {
+		o.log.Printf("warning: tcp service backend auto-heal failed: %v", err)
+		return
+	}
+	if len(healed) == 0 {
+		return
+	}
+	sort.Strings(healed)
+	detail := fmt.Sprintf("restored tcp service backends: %s", joinLimited(healed, 8))
+	o.log.Printf("%s", detail)
+	o.noteStartupAutoHeal(detail)
+}
+
+// healFailedTCPServiceBackends runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) healFailedTCPServiceBackends(ctx context.Context) ([]string, error).
+// Why: failed TCP protocol checks need a direct backend-heal primitive that can
+// be reused by startup waits and post-start daemon repair.
+func (o *Orchestrator) healFailedTCPServiceBackends(ctx context.Context) ([]string, error) {
+	if o.runner.DryRun || len(o.cfg.Startup.TCPServiceChecklist) == 0 {
+		return nil, nil
+	}
+	healed := []string{}
+	attempted := map[string]struct{}{}
+	for _, check := range o.cfg.Startup.TCPServiceChecklist {
+		namespace := strings.TrimSpace(check.Namespace)
+		service := strings.TrimSpace(check.Service)
+		if namespace == "" || service == "" {
+			continue
+		}
+		key := namespace + "/" + service
+		if _, ok := attempted[key]; ok {
+			continue
+		}
+		ok, _ := o.tcpServiceCheckReady(ctx, check)
+		if ok {
+			continue
+		}
+		attempted[key] = struct{}{}
+		items, err := o.maybeHealCriticalEndpointBackends(ctx, namespace, service)
+		if err != nil {
+			return healed, fmt.Errorf("%s/%s: %w", namespace, service, err)
+		}
+		healed = append(healed, items...)
+	}
+	return healed, nil
+}

internal/cluster/orchestrator_timesync_inventory.go

@@ -135,6 +135,10 @@ func isTimeSynced(raw string) bool {
 // Signature: (o *Orchestrator) preflightExternalDatastore(ctx context.Context) error.
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
 func (o *Orchestrator) preflightExternalDatastore(ctx context.Context) error {
+	// Cancellation must win even when a local datastore is already reachable.
+	if err := ctx.Err(); err != nil {
+		return err
+	}
 	if len(o.cfg.ControlPlanes) == 0 {
 		return nil
 	}

internal/cluster/orchestrator_unit_additional_test.go

@@ -8,6 +8,7 @@ import (
 	"net"
 	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -73,6 +74,73 @@ func matchContains(cmd string, parts ...string) func(string, []string) bool {
 	}
 }
 
+// TestStartupEarlyFailureLeavesFluxSuspensionUnchanged runs one orchestration or CLI step.
+// Signature: TestStartupEarlyFailureLeavesFluxSuspensionUnchanged(t *testing.T).
+// Why: recovery must not release Flux when bootstrap fails before storage and
+// critical workloads are ready, or Flux can re-create the same dependency loop.
+func TestStartupEarlyFailureLeavesFluxSuspensionUnchanged(t *testing.T) {
+	tmpDir := t.TempDir()
+	cfg := config.Config{
+		SSHPort: 2277,
+		Startup: config.Startup{
+			APIWaitSeconds:              1,
+			APIPollSeconds:              1,
+			RequireNodeInventoryReach:   false,
+			RequireTimeSync:             false,
+			RequireNodeSSHAuth:          false,
+			ReconcileAccessOnBoot:       false,
+			AutoEtcdRestoreOnAPIFailure: false,
+			RequiredNodeLabels: map[string]map[string]string{
+				"titan-missing": {
+					"node-role.kubernetes.io/worker": "true",
+				},
+			},
+		},
+		State: config.State{
+			Dir:            tmpDir,
+			ReportsDir:     filepath.Join(tmpDir, "reports"),
+			RunHistoryPath: filepath.Join(tmpDir, "runs.json"),
+			LockPath:       filepath.Join(tmpDir, "ananke.lock"),
+			IntentPath:     filepath.Join(tmpDir, "intent.json"),
+		},
+	}
+
+	var mu sync.Mutex
+	calls := []string{}
+	orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
+		{
+			match: func(name string, args []string) bool {
+				mu.Lock()
+				calls = append(calls, name+" "+strings.Join(args, " "))
+				mu.Unlock()
+				return false
+			},
+		},
+		{match: matchContains("kubectl", "version", "--request-timeout=5s"), out: "ok"},
+		{match: matchContains("kubectl", "-n", "vault", "get", "pod", "vault-0"), out: "Pending"},
+		{
+			match: matchContains("kubectl", "label", "node", "titan-missing"),
+			err:   errors.New(`nodes "titan-missing" not found`),
+		},
+	})
+
+	err := orch.Startup(context.Background(), StartupOptions{Reason: "test early failure"})
+	if err == nil {
+		t.Fatalf("expected startup to fail before flux resume")
+	}
+	if !strings.Contains(err.Error(), "ensure required node labels on titan-missing") {
+		t.Fatalf("expected required-label failure, got: %v", err)
+	}
+
+	mu.Lock()
+	defer mu.Unlock()
+	for _, call := range calls {
+		if strings.Contains(call, `"suspend":false`) {
+			t.Fatalf("early failed startup unexpectedly resumed flux via call: %s", call)
+		}
+	}
+}
+
 // TestNewConstructsOrchestrator runs one orchestration or CLI step.
 // Signature: TestNewConstructsOrchestrator(t *testing.T).
 // Why: covers constructor path in orchestrator core module.
@@ -398,86 +466,3 @@ func TestRunSudoK3SFailsWhenAllCandidatesFail(t *testing.T) {
 		t.Fatalf("expected runSudoK3S failure when all candidates fail")
 	}
 }
-
-// TestCriticalEndpointHelpers runs one orchestration or CLI step.
-// Signature: TestCriticalEndpointHelpers(t *testing.T).
-// Why: covers critical endpoint parsing and readiness checks that gate startup completion.
-func TestCriticalEndpointHelpers(t *testing.T) {
-	cfg := config.Config{
-		Startup: config.Startup{
-			CriticalServiceEndpoints: []string{"monitoring/victoria-metrics-single-server"},
-		},
-	}
-	orch := buildOrchestratorWithStubs(t, cfg, []commandStub{
-		{match: matchContains("kubectl", "get endpoints victoria-metrics-single-server"), out: "10.42.0.10\n10.42.0.11\n"},
-	})
-	ok, detail, ns, svc, err := orch.criticalServiceEndpointsReady(context.Background())
-	if err != nil || !ok {
-		t.Fatalf("expected criticalServiceEndpointsReady success, got ok=%v detail=%q ns=%q svc=%q err=%v", ok, detail, ns, svc, err)
-	}
-	if detail != "services=1" {
-		t.Fatalf("unexpected readiness detail: %q", detail)
-	}
-	gotNS, gotSvc, err := parseCriticalServiceEndpoint("monitoring/victoria-metrics-single-server")
-	if err != nil || gotNS != "monitoring" || gotSvc != "victoria-metrics-single-server" {
-		t.Fatalf("unexpected parse result ns=%q svc=%q err=%v", gotNS, gotSvc, err)
-	}
-	if _, _, err := parseCriticalServiceEndpoint("invalid"); err == nil {
-		t.Fatalf("expected parseCriticalServiceEndpoint error")
-	}
-}
-
-// TestCriticalEndpointAutoHealWorkflow runs one orchestration or CLI step.
-// Signature: TestCriticalEndpointAutoHealWorkflow(t *testing.T).
-// Why: covers endpoint-zero recovery where startup heals workload replicas before succeeding.
-func TestCriticalEndpointAutoHealWorkflow(t *testing.T) {
-	cfg := config.Config{
-		Startup: config.Startup{
-			CriticalServiceEndpointWaitSec: 2,
-			CriticalServiceEndpointPollSec: 1,
-			CriticalServiceEndpoints:       []string{"monitoring/victoria-metrics-single-server"},
-		},
-		State: config.State{
-			Dir:            t.TempDir(),
-			ReportsDir:     filepath.Join(t.TempDir(), "reports"),
-			RunHistoryPath: filepath.Join(t.TempDir(), "runs.json"),
-		},
-	}
-	orch := &Orchestrator{
-		cfg:    cfg,
-		runner: &execx.Runner{},
-		store:  state.New(cfg.State.RunHistoryPath),
-		log:    log.New(io.Discard, "", 0),
-	}
-
-	endpointChecks := 0
-	dispatch := func(_ context.Context, _ time.Duration, name string, args ...string) (string, error) {
-		joined := name + " " + strings.Join(args, " ")
-		if strings.Contains(joined, "get endpoints victoria-metrics-single-server") {
-			endpointChecks++
-			if endpointChecks == 1 {
-				return "", nil
-			}
-			return "10.42.0.10\n", nil
-		}
-		if strings.Contains(joined, "scale deployment victoria-metrics-single-server") {
-			return "", errors.New(`Error from server (NotFound): deployments.apps "victoria-metrics-single-server" not found`)
-		}
-		if strings.Contains(joined, "scale statefulset victoria-metrics-single-server") {
-			return "", nil
-		}
-		if strings.Contains(joined, "rollout status statefulset/victoria-metrics-single-server") {
-			return "statefulset rolled out", nil
-		}
-		return "", nil
-	}
-	orch.runOverride = dispatch
-	orch.runSensitiveOverride = dispatch
-
-	if err := orch.waitForCriticalServiceEndpoints(context.Background()); err != nil {
-		t.Fatalf("waitForCriticalServiceEndpoints failed: %v", err)
-	}
-	if endpointChecks < 2 {
-		t.Fatalf("expected repeated endpoint checks, got %d", endpointChecks)
-	}
-}

internal/cluster/orchestrator_vault_health.go

@@ -0,0 +1,74 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// vaultSealed runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error).
+// Why: treats exec and HTTP health as independent Vault signals so one flaky
+// kubectl exec does not block startup when HTTP health proves Vault is usable.
+func (o *Orchestrator) vaultSealed(ctx context.Context) (bool, error) {
+	sealed, err := o.vaultSealedViaExec(ctx)
+	if err == nil {
+		return sealed, nil
+	}
+	httpSealed, httpErr := o.vaultSealedViaHTTP(ctx)
+	if httpErr == nil {
+		o.log.Printf("warning: vault exec status probe failed but HTTP health succeeded; continuing with HTTP health result: %v", err)
+		return httpSealed, nil
+	}
+	return false, fmt.Errorf("vault status check failed: exec=%v http=%v", err, httpErr)
+}
+
+// vaultSealedViaExec runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) vaultSealedViaExec(ctx context.Context) (bool, error).
+// Why: preserves the existing direct Vault CLI probe as the primary status
+// signal when kubectl exec is healthy.
+func (o *Orchestrator) vaultSealedViaExec(ctx context.Context) (bool, error) {
+	out, err := o.kubectl(
+		ctx,
+		25*time.Second,
+		"-n", "vault",
+		"exec", "vault-0", "--",
+		"sh", "-lc",
+		"VAULT_ADDR=http://127.0.0.1:8200 vault status -format=json 2>/dev/null || true",
+	)
+	if err != nil {
+		return false, fmt.Errorf("vault status check failed: %w", err)
+	}
+	sealed, err := parseVaultSealed(out)
+	if err != nil {
+		return false, fmt.Errorf("parse vault status: %w", err)
+	}
+	return sealed, nil
+}
+
+// vaultSealedViaHTTP runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) vaultSealedViaHTTP(ctx context.Context) (bool, error).
+// Why: gives startup a second authoritative Vault health path when exec is
+// transiently killed or the apiserver exec tunnel is degraded.
+func (o *Orchestrator) vaultSealedViaHTTP(ctx context.Context) (bool, error) {
+	out, err := o.kubectl(
+		ctx,
+		15*time.Second,
+		"-n", "vault",
+		"get",
+		"--raw",
+		"/api/v1/namespaces/vault/pods/vault-0:8200/proxy/v1/sys/health",
+	)
+	if err != nil && strings.TrimSpace(out) == "" {
+		return false, fmt.Errorf("vault HTTP health check failed: %w", err)
+	}
+	sealed, parseErr := parseVaultSealed(out)
+	if parseErr != nil {
+		if err != nil {
+			return false, fmt.Errorf("vault HTTP health check failed: %w; parse: %v", err, parseErr)
+		}
+		return false, fmt.Errorf("parse vault HTTP health: %w", parseErr)
+	}
+	return sealed, nil
+}

internal/cluster/orchestrator_workload_convergence.go

@@ -26,10 +26,12 @@ func (o *Orchestrator) waitForWorkloadConvergence(ctx context.Context) error {
 	lastLogged := time.Time{}
 	lastRecycleAttempt := time.Time{}
 	lastReplicaHeal := time.Time{}
+	lastSchedulingStormHeal := time.Time{}
 	for {
 		prevFailure := lastFailure
 		o.maybeAutoRecycleStuckPods(ctx, &lastRecycleAttempt)
 		o.maybeAutoHealCriticalWorkloadReplicas(ctx, &lastReplicaHeal)
+		o.maybeAutoQuarantineSchedulingStorms(ctx, &lastSchedulingStormHeal)
 		ready, detail, err := o.workloadConvergenceReady(ctx)
 		if err != nil {
 			lastFailure = err.Error()
@@ -71,6 +73,7 @@ func (o *Orchestrator) workloadConvergenceReady(ctx context.Context) (bool, stri
 	if err := json.Unmarshal([]byte(out), &list); err != nil {
 		return false, "", fmt.Errorf("decode controllers: %w", err)
 	}
+	requiredNamespaces := o.startupRequiredWorkloadNamespaces()
 	ignoredNamespaces := makeStringSet(o.cfg.Startup.IgnoreWorkloadNamespaces)
 	ignoredNodes := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
 	ignoreRules := parseWorkloadIgnoreRules(o.cfg.Startup.IgnoreWorkloads)
@@ -84,6 +87,11 @@ func (o *Orchestrator) workloadConvergenceReady(ctx context.Context) (bool, stri
 		if kind == "" || ns == "" || name == "" {
 			continue
 		}
+		if len(requiredNamespaces) > 0 {
+			if _, ok := requiredNamespaces[ns]; !ok {
+				continue
+			}
+		}
 		if _, ok := ignoredNamespaces[ns]; ok {
 			continue
 		}
@@ -162,6 +170,49 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error {
 		"CreateContainerConfigError": {},
 		"CreateContainerError":       {},
 	}
+	longhornAttachReasons := map[string]string{}
+	if reasons, scanErr := o.longhornAttachBlockedPodReasons(ctx, list, grace); scanErr != nil {
+		o.log.Printf("warning: longhorn attach-blocked pod scan failed: %v", scanErr)
+	} else {
+		longhornAttachReasons = reasons
+	}
+	encryptedMountReasons := map[string]string{}
+	if reasons, scanErr := o.repairEncryptedVolumeMountPrereqs(ctx, list, grace); scanErr != nil {
+		o.log.Printf("warning: encrypted volume mount prerequisite scan failed: %v", scanErr)
+	} else {
+		encryptedMountReasons = reasons
+	}
+	stalePhaseReasons := map[string]string{}
+	if reasons, scanErr := o.staleControllerPodReasons(ctx, list, grace); scanErr != nil {
+		o.log.Printf("warning: stale controller pod scan failed: %v", scanErr)
+	} else {
+		stalePhaseReasons = reasons
+	}
+	staleRWODecisions := map[string]staleRWOOwnerDecision{}
+	if decisions, scanErr := o.staleRWOPVCOwnerDecisions(ctx, list, grace); scanErr != nil {
+		o.log.Printf("warning: stale RWO PVC owner scan failed: %v", scanErr)
+	} else {
+		staleRWODecisions = decisions
+	}
+	imagePullDNSReasons := map[string]string{}
+	if reasons, scanErr := o.imagePullDNSBlockerReasons(ctx); scanErr != nil {
+		o.log.Printf("warning: image-pull DNS blocker scan failed: %v", scanErr)
+	} else {
+		imagePullDNSReasons = reasons
+	}
+	imagePullCredentialReasons := map[string]string{}
+	if reasons, scanErr := o.imagePullCredentialBlockerReasons(ctx); scanErr != nil {
+		o.log.Printf("warning: image-pull credential blocker scan failed: %v", scanErr)
+	} else {
+		imagePullCredentialReasons = reasons
+	}
+	containerRuntimeWedgeReasons := map[string]string{}
+	if reasons, scanErr := o.containerRuntimeWedgePodReasons(ctx, list, grace); scanErr != nil {
+		o.log.Printf("warning: container runtime wedge scan failed: %v", scanErr)
+	} else {
+		containerRuntimeWedgeReasons = reasons
+		o.quarantineContainerRuntimeWedgeNodes(ctx, list, reasons, grace, ignoredNamespaces, ignoredNodes, ignoreRules)
+	}
 	recycled := []string{}
 	for _, pod := range list.Items {
 		ns := strings.TrimSpace(pod.Metadata.Namespace)
@@ -186,14 +237,58 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error {
 			continue
 		}
 		reason := stuckContainerReason(pod, stuckReasons)
+		if (reason == "ImagePullBackOff" || reason == "ErrImagePull") && imagePullDNSReasons[ns+"/"+name] != "" {
+			o.log.Printf("warning: not recycling pod %s/%s because image pull is blocked by DNS/registry lookup: %s", ns, name, imagePullDNSReasons[ns+"/"+name])
+			continue
+		}
+		if (reason == "ImagePullBackOff" || reason == "ErrImagePull") && imagePullCredentialReasons[ns+"/"+name] != "" {
+			o.log.Printf("warning: not recycling pod %s/%s because image pull is blocked by registry credentials: %s", ns, name, imagePullCredentialReasons[ns+"/"+name])
+			continue
+		}
 		if reason == "" {
 			reason = stuckVaultInitReason(pod, grace)
 		}
 		if reason == "" {
+			reason = longhornAttachReasons[ns+"/"+name]
+		}
+		if reason == "" {
+			reason = encryptedMountReasons[ns+"/"+name]
+		}
+		if reason == "" {
+			reason = stalePhaseReasons[ns+"/"+name]
+		}
+		if decision, ok := staleRWODecisions[ns+"/"+name]; ok && decision.Unsafe {
+			o.log.Printf("warning: stale RWO PVC owner is unsafe to force delete pod=%s/%s reason=%s", ns, name, decision.Reason)
 			continue
 		}
-		o.log.Printf("warning: recycling stuck pod %s/%s reason=%s age=%s", ns, name, reason, age.Round(time.Second))
+		if reason == "" {
-		if _, err := o.kubectl(ctx, 30*time.Second, "-n", ns, "delete", "pod", name, "--wait=false"); err != nil && !isNotFoundErr(err) {
+			if decision, ok := staleRWODecisions[ns+"/"+name]; ok && decision.ForceDelete {
+				reason = decision.Reason
+			}
+		}
+		if runtimeReason := containerRuntimeWedgeReasons[ns+"/"+name]; runtimeReason != "" {
+			reason = runtimeReason
+		}
+		if reason == "" && staleControllerPodForceDeleteSafe(pod, grace) {
+			reason = "StaleDeletingControllerPod"
+		}
+		if reason == "" {
+			continue
+		}
+		deleteArgs := []string{"-n", ns, "delete", "pod", name, "--wait=false"}
+		forceDelete := staleControllerPodForceDeleteSafe(pod, grace)
+		if decision, ok := staleRWODecisions[ns+"/"+name]; ok && decision.ForceDelete {
+			forceDelete = true
+		}
+		if forceDelete {
+			deleteArgs = append(deleteArgs, "--grace-period=0", "--force")
+		}
+		if forceDelete {
+			o.log.Printf("warning: force recycling stuck pod %s/%s reason=%s age=%s", ns, name, reason, age.Round(time.Second))
+		} else {
+			o.log.Printf("warning: recycling stuck pod %s/%s reason=%s age=%s", ns, name, reason, age.Round(time.Second))
+		}
+		if _, err := o.kubectl(ctx, 30*time.Second, deleteArgs...); err != nil && !isNotFoundErr(err) {
 			o.log.Printf("warning: recycle pod failed for %s/%s: %v", ns, name, err)
 			continue
 		}
@@ -207,66 +302,178 @@ func (o *Orchestrator) recycleStuckControllerPods(ctx context.Context) error {
 	return nil
 }
 
-// podControllerOwned runs one orchestration or CLI step.
+// quarantineContainerRuntimeWedgeNodesFromCluster runs one orchestration or CLI step.
-// Signature: podControllerOwned(p podResource) bool.
+// Signature: (o *Orchestrator) quarantineContainerRuntimeWedgeNodesFromCluster(ctx context.Context) ([]string, error).
-// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
+// Why: worker startup needs this scan before SSH-heavy steps so a Ready but
-func podControllerOwned(p podResource) bool {
+// container-runtime-wedged node cannot stall the whole recovery run.
-	for _, owner := range p.Metadata.OwnerReferences {
+func (o *Orchestrator) quarantineContainerRuntimeWedgeNodesFromCluster(ctx context.Context) ([]string, error) {
-		switch strings.TrimSpace(owner.Kind) {
+	out, err := o.kubectl(ctx, 30*time.Second, "get", "pods", "-A", "-o", "json")
-		case "ReplicaSet", "StatefulSet", "DaemonSet":
+	if err != nil {
-			return true
+		return nil, fmt.Errorf("query pods for container runtime wedge scan: %w", err)
-		}
 	}
-	return false
+	var list podList
+	if err := json.Unmarshal([]byte(out), &list); err != nil {
+		return nil, fmt.Errorf("decode pods for container runtime wedge scan: %w", err)
+	}
+	grace := time.Duration(o.cfg.Startup.StuckPodGraceSeconds) * time.Second
+	if grace <= 0 {
+		grace = 180 * time.Second
+	}
+	reasons, err := o.containerRuntimeWedgePodReasons(ctx, list, grace)
+	if err != nil {
+		return nil, err
+	}
+	ignoredNamespaces := makeStringSet(o.cfg.Startup.IgnoreWorkloadNamespaces)
+	ignoredNodes := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
+	ignoreRules := parseWorkloadIgnoreRules(o.cfg.Startup.IgnoreWorkloads)
+	return o.quarantineContainerRuntimeWedgeNodes(ctx, list, reasons, grace, ignoredNamespaces, ignoredNodes, ignoreRules), nil
 }
 
-// stuckContainerReason runs one orchestration or CLI step.
+// containerRuntimeWedgePodReasons runs one orchestration or CLI step.
-// Signature: stuckContainerReason(p podResource, reasons map[string]struct{}) string.
+// Signature: (o *Orchestrator) containerRuntimeWedgePodReasons(ctx context.Context, pods podList, grace time.Duration) (map[string]string, error).
-// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
+// Why: after a power event, a node-local container runtime can reserve names and
-func stuckContainerReason(p podResource, reasons map[string]struct{}) string {
+// fail every new container start while Kubernetes still reports the node Ready.
-	check := func(statuses []podContainerStatus) string {
+// Detecting the runtime symptom lets startup move work elsewhere without
-		for _, st := range statuses {
+// restarting the node or touching storage objects.
-			if st.State.Waiting == nil {
+func (o *Orchestrator) containerRuntimeWedgePodReasons(ctx context.Context, pods podList, grace time.Duration) (map[string]string, error) {
-				continue
+	eventsOut, err := o.kubectl(ctx, 30*time.Second, "get", "events", "-A", "-o", "json")
-			}
+	if err != nil {
-			reason := strings.TrimSpace(st.State.Waiting.Reason)
+		return nil, fmt.Errorf("query events for container runtime wedge scan: %w", err)
-			if reason == "" {
+	}
-				continue
+	var events eventList
-			}
+	if err := json.Unmarshal([]byte(eventsOut), &events); err != nil {
-			if _, ok := reasons[reason]; ok {
+		return nil, fmt.Errorf("decode events for container runtime wedge scan: %w", err)
-				return reason
+	}
-			}
+
+	runtimeReasons := map[string]struct{}{
+		"CreateContainerError": {},
+		"RunContainerError":    {},
+	}
+	podsByKey := map[string]podResource{}
+	for _, pod := range pods.Items {
+		ns := strings.TrimSpace(pod.Metadata.Namespace)
+		name := strings.TrimSpace(pod.Metadata.Name)
+		node := strings.TrimSpace(pod.Spec.NodeName)
+		if ns == "" || name == "" || node == "" {
+			continue
 		}
-		return ""
+		if !strings.EqualFold(strings.TrimSpace(pod.Status.Phase), "Pending") {
+			continue
+		}
+		if !podControllerOwned(pod) {
+			continue
+		}
+		if !pod.Metadata.CreationTimestamp.IsZero() && time.Since(pod.Metadata.CreationTimestamp) < grace {
+			continue
+		}
+		if stuckContainerReason(pod, runtimeReasons) == "" {
+			continue
+		}
+		podsByKey[ns+"/"+name] = pod
 	}
-	if reason := check(p.Status.InitContainerStatuses); reason != "" {
+	if len(podsByKey) == 0 {
-		return reason
+		return map[string]string{}, nil
 	}
-	return check(p.Status.ContainerStatuses)
+
+	reasons := map[string]string{}
+	for _, event := range events.Items {
+		if !strings.EqualFold(strings.TrimSpace(event.Type), "Warning") {
+			continue
+		}
+		if strings.TrimSpace(event.Reason) != "Failed" {
+			continue
+		}
+		if !strings.EqualFold(strings.TrimSpace(event.InvolvedObject.Kind), "Pod") {
+			continue
+		}
+		key := strings.TrimSpace(event.InvolvedObject.Namespace) + "/" + strings.TrimSpace(event.InvolvedObject.Name)
+		pod, ok := podsByKey[key]
+		if !ok {
+			continue
+		}
+		lastSeen := eventLastObservedAt(event)
+		if !lastSeen.IsZero() && !pod.Metadata.CreationTimestamp.IsZero() && lastSeen.Before(pod.Metadata.CreationTimestamp) {
+			continue
+		}
+		message := strings.ToLower(strings.TrimSpace(event.Message))
+		if !strings.Contains(message, "failed to reserve container name") &&
+			!strings.Contains(message, " is reserved for ") &&
+			!strings.Contains(message, "context deadline exceeded") {
+			continue
+		}
+		reasons[key] = "ContainerRuntimeWedge:" + strings.TrimSpace(pod.Spec.NodeName)
+	}
+	return reasons, nil
 }
 
-// stuckVaultInitReason runs one orchestration or CLI step.
+// quarantineContainerRuntimeWedgeNodes runs one orchestration or CLI step.
-// Signature: stuckVaultInitReason(p podResource, grace time.Duration) string.
+// Signature: (o *Orchestrator) quarantineContainerRuntimeWedgeNodes(ctx context.Context, pods podList, reasons map[string]string, grace time.Duration, ignoredNamespaces map[string]struct{}, ignoredNodes map[string]struct{}, ignoreRules []workloadIgnoreRule) []string.
-// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
+// Why: cordoning a proven-bad start node is scheduler-only; it prevents fresh
-func stuckVaultInitReason(p podResource, grace time.Duration) string {
+// non-storage pods from being trapped while leaving running workloads and
-	if !strings.EqualFold(strings.TrimSpace(p.Status.Phase), "Pending") {
+// Longhorn data-plane state alone.
-		return ""
+func (o *Orchestrator) quarantineContainerRuntimeWedgeNodes(ctx context.Context, pods podList, reasons map[string]string, grace time.Duration, ignoredNamespaces map[string]struct{}, ignoredNodes map[string]struct{}, ignoreRules []workloadIgnoreRule) []string {
+	if len(reasons) == 0 {
+		return nil
 	}
-	if !strings.EqualFold(strings.TrimSpace(p.Metadata.Annotations["vault.hashicorp.com/agent-inject"]), "true") {
+	const minRuntimeWedgePodsPerNode = 2
-		return ""
+	byNode := map[string][]string{}
-	}
+	for _, pod := range pods.Items {
-	for _, st := range p.Status.InitContainerStatuses {
+		ns := strings.TrimSpace(pod.Metadata.Namespace)
-		if strings.TrimSpace(st.Name) != "vault-agent-init" || st.State.Running == nil {
+		name := strings.TrimSpace(pod.Metadata.Name)
+		node := strings.TrimSpace(pod.Spec.NodeName)
+		if ns == "" || name == "" || node == "" {
 			continue
 		}
-		startedAt := st.State.Running.StartedAt
+		key := ns + "/" + name
-		if startedAt.IsZero() {
+		if reasons[key] == "" {
 			continue
 		}
-		if time.Since(startedAt) < grace {
+		if _, ok := ignoredNamespaces[ns]; ok {
-			return ""
+			continue
 		}
-		return "VaultInitStuck"
+		if workloadIgnored(ignoreRules, ns, "", name) {
+			continue
+		}
+		if podTargetsIgnoredNode(pod, ignoredNodes) {
+			continue
+		}
+		if !podControllerOwned(pod) {
+			continue
+		}
+		if !pod.Metadata.CreationTimestamp.IsZero() && time.Since(pod.Metadata.CreationTimestamp) < grace {
+			continue
+		}
+		if podUsesPersistentVolumeClaim(pod) {
+			continue
+		}
+		byNode[node] = append(byNode[node], key)
 	}
-	return ""
+
+	quarantined := []string{}
+	for node, keys := range byNode {
+		if len(keys) < minRuntimeWedgePodsPerNode {
+			continue
+		}
+		sort.Strings(keys)
+		detail := fmt.Sprintf("pods=%d %s", len(keys), joinLimited(keys, 8))
+		if err := o.cordonNodeWithLease(ctx, node, cordonReasonRuntimeWedge, detail); err != nil {
+			o.log.Printf("warning: cordon container-runtime-wedged node %s failed: %v", node, err)
+			continue
+		}
+		o.log.Printf("warning: cordoned node %s after repeated container runtime start failures: %s", node, joinLimited(keys, 8))
+		quarantined = append(quarantined, fmt.Sprintf("%s pods=%d", node, len(keys)))
+	}
+	if len(quarantined) == 0 {
+		return nil
+	}
+	sort.Strings(quarantined)
+	o.noteStartupAutoHeal(fmt.Sprintf("cordoned container-runtime-wedged node(s): %s", joinLimited(quarantined, 8)))
+	nodes := make([]string, 0, len(quarantined))
+	for _, item := range quarantined {
+		fields := strings.Fields(item)
+		if len(fields) > 0 {
+			nodes = append(nodes, fields[0])
+		}
+	}
+	return nodes
 }

internal/cluster/orchestrator_workload_ignore.go

@@ -116,6 +116,7 @@ func (o *Orchestrator) startupFailurePods(ctx context.Context) ([]string, error)
 		return nil, fmt.Errorf("decode pods: %w", err)
 	}
 
+	requiredNamespaces := o.startupRequiredWorkloadNamespaces()
 	ignoredNamespaces := makeStringSet(o.cfg.Startup.IgnoreWorkloadNamespaces)
 	ignoredNodes := makeStringSet(o.cfg.Startup.IgnoreUnavailableNodes)
 	stuckReasons := map[string]struct{}{
@@ -138,6 +139,11 @@ func (o *Orchestrator) startupFailurePods(ctx context.Context) ([]string, error)
 		if ns == "" || name == "" {
 			continue
 		}
+		if len(requiredNamespaces) > 0 {
+			if _, ok := requiredNamespaces[ns]; !ok {
+				continue
+			}
+		}
 		if _, ok := ignoredNamespaces[ns]; ok {
 			continue
 		}

internal/cluster/orchestrator_workload_recovery_test.go

@@ -0,0 +1,458 @@
+package cluster
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestRecycleStuckControllerPodsHandlesLonghornAttachBlockedPods runs one orchestration or CLI step.
+// Signature: TestRecycleStuckControllerPodsHandlesLonghornAttachBlockedPods(t *testing.T).
+// Why: Pending Longhorn-backed pods on Longhorn-unready nodes should be
+// rescheduled without mutating Longhorn volume, replica, or disk objects.
+func TestRecycleStuckControllerPodsHandlesLonghornAttachBlockedPods(t *testing.T) {
+	created := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
+	lastSeen := time.Now().UTC().Format(time.RFC3339)
+	pods := `{"items":[{"metadata":{"namespace":"monitoring","name":"victoria-metrics-single-server-0","creationTimestamp":"` + created + `","ownerReferences":[{"kind":"StatefulSet","name":"victoria-metrics-single-server"}]},"spec":{"nodeName":"titan-0b"},"status":{"phase":"Pending"}}]}`
+	events := `{"items":[{"metadata":{"namespace":"monitoring","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"monitoring","name":"victoria-metrics-single-server-0"},"type":"Warning","reason":"FailedAttachVolume","message":"AttachVolume.Attach failed for volume \"pvc-1\" : rpc error from [http://longhorn-backend:9500/v1/volumes/pvc-1?action=attach]: unable to attach volume pvc-1 to titan-0b: node titan-0b is not ready","lastTimestamp":"` + lastSeen + `"}]}`
+
+	deleted := false
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{StuckPodGraceSeconds: 180},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: pods},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: "titan-0b\tFalse\n"},
+		{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-0b"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "monitoring", "delete", "pod", "victoria-metrics-single-server-0", "--wait=false")(name, args) {
+					return false
+				}
+				deleted = true
+				return true
+			},
+		},
+	})
+
+	if err := orch.recycleStuckControllerPods(context.Background()); err != nil {
+		t.Fatalf("recycleStuckControllerPods failed: %v", err)
+	}
+	if !deleted {
+		t.Fatalf("expected longhorn attach-blocked pending pod to be recycled")
+	}
+}
+
+// TestRecycleStuckControllerPodsRepairsEncryptedVolumeCryptsetup runs one orchestration or CLI step.
+// Signature: TestRecycleStuckControllerPodsRepairsEncryptedVolumeCryptsetup(t *testing.T).
+// Why: encrypted Longhorn PVC recovery should repair missing host cryptsetup and
+// then recycle the blocked pod without touching Longhorn data-plane objects.
+func TestRecycleStuckControllerPodsRepairsEncryptedVolumeCryptsetup(t *testing.T) {
+	created := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
+	lastSeen := time.Now().UTC().Format(time.RFC3339)
+	pods := `{"items":[{"metadata":{"namespace":"finance","name":"actual-budget-abc","creationTimestamp":"` + created + `","ownerReferences":[{"kind":"ReplicaSet","name":"actual-budget"}]},"spec":{"nodeName":"titan-19"},"status":{"phase":"Pending"}}]}`
+	events := `{"items":[{"metadata":{"namespace":"finance","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"finance","name":"actual-budget-abc"},"type":"Warning","reason":"FailedMount","message":"MountVolume.MountDevice failed for volume \"pvc-1\" : nsenter: failed to execute cryptsetup: No such file or directory","lastTimestamp":"` + lastSeen + `"}]}`
+
+	installed := false
+	deleted := false
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{StuckPodGraceSeconds: 180},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: pods},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: "titan-19\tTrue\n"},
+		{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-19"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`},
+		{
+			match: func(name string, args []string) bool {
+				if name != "ssh" || !strings.Contains(strings.Join(args, " "), "apt-get install -y --no-install-recommends cryptsetup-bin") {
+					return false
+				}
+				installed = true
+				return true
+			},
+			out: "__ANANKE_CRYPTSETUP_INSTALLED__",
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "finance", "delete", "pod", "actual-budget-abc", "--wait=false")(name, args) {
+					return false
+				}
+				deleted = true
+				return true
+			},
+		},
+	})
+
+	if err := orch.recycleStuckControllerPods(context.Background()); err != nil {
+		t.Fatalf("recycleStuckControllerPods failed: %v", err)
+	}
+	if !installed {
+		t.Fatalf("expected missing host cryptsetup to be installed")
+	}
+	if !deleted {
+		t.Fatalf("expected encrypted-volume blocked pod to be recycled")
+	}
+}
+
+// TestRecycleStuckControllerPodsCordonsEncryptedVolumeNodeWhenRepairFails runs one orchestration or CLI step.
+// Signature: TestRecycleStuckControllerPodsCordonsEncryptedVolumeNodeWhenRepairFails(t *testing.T).
+// Why: when host package repair is blocked by sudo policy, Ananke should avoid
+// the bad node and retry the controller-owned pod elsewhere.
+func TestRecycleStuckControllerPodsCordonsEncryptedVolumeNodeWhenRepairFails(t *testing.T) {
+	created := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
+	lastSeen := time.Now().UTC().Format(time.RFC3339)
+	pods := `{"items":[{"metadata":{"namespace":"finance","name":"actual-budget-abc","creationTimestamp":"` + created + `","ownerReferences":[{"kind":"ReplicaSet","name":"actual-budget"}]},"spec":{"nodeName":"titan-19"},"status":{"phase":"Pending"}}]}`
+	events := `{"items":[{"metadata":{"namespace":"finance","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"finance","name":"actual-budget-abc"},"type":"Warning","reason":"FailedMount","message":"MountVolume.MountDevice failed for volume \"pvc-1\" : nsenter: failed to execute cryptsetup: No such file or directory","lastTimestamp":"` + lastSeen + `"}]}`
+
+	cordoned := false
+	deleted := false
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{StuckPodGraceSeconds: 180},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: pods},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: "titan-19\tTrue\n"},
+		{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-19"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`},
+		{
+			match: matchContains("ssh", "apt-get install -y --no-install-recommends cryptsetup-bin"),
+			err:   errors.New("sudo: a password is required"),
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "cordon", "titan-19")(name, args) {
+					return false
+				}
+				cordoned = true
+				return true
+			},
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "finance", "delete", "pod", "actual-budget-abc", "--wait=false")(name, args) {
+					return false
+				}
+				deleted = true
+				return true
+			},
+		},
+	})
+
+	if err := orch.recycleStuckControllerPods(context.Background()); err != nil {
+		t.Fatalf("recycleStuckControllerPods failed: %v", err)
+	}
+	if !cordoned {
+		t.Fatalf("expected cryptsetup-missing node to be cordoned")
+	}
+	if !deleted {
+		t.Fatalf("expected encrypted-volume blocked pod to be recycled")
+	}
+}
+
+// TestRecycleStuckControllerPodsHandlesStalePodsOnReadyNodes runs one orchestration or CLI step.
+// Signature: TestRecycleStuckControllerPodsHandlesStalePodsOnReadyNodes(t *testing.T).
+// Why: post-outage controller pods can remain Unknown or Failed after their
+// node recovers; deletion clears stale status while force deletion stays away
+// from PVC-backed storage.
+func TestRecycleStuckControllerPodsHandlesStalePodsOnReadyNodes(t *testing.T) {
+	old := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
+	recent := time.Now().Add(-30 * time.Second).UTC().Format(time.RFC3339)
+	pods := `{"items":[` +
+		`{"metadata":{"namespace":"longhorn-system","name":"longhorn-vault-sync-old","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"longhorn-vault-sync"}]},"spec":{"nodeName":"titan-12"},"status":{"phase":"Unknown"}},` +
+		`{"metadata":{"namespace":"longhorn-system","name":"longhorn-vault-sync-failed","creationTimestamp":"` + old + `","deletionTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"longhorn-vault-sync"}]},"spec":{"nodeName":"titan-12","volumes":[{"name":"secret"}]},"status":{"phase":"Failed"}},` +
+		`{"metadata":{"namespace":"logging","name":"oauth2-proxy-terminating","creationTimestamp":"` + old + `","deletionTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"oauth2-proxy-logs"}]},"spec":{"nodeName":"titan-18","volumes":[{"name":"secret"}]},"status":{"phase":"Running"}},` +
+		`{"metadata":{"namespace":"longhorn-system","name":"pvc-backed-failed","creationTimestamp":"` + old + `","deletionTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"pvc-backed"}]},"spec":{"nodeName":"titan-12","volumes":[{"name":"data","persistentVolumeClaim":{"claimName":"data"}}]},"status":{"phase":"Failed"}},` +
+		`{"metadata":{"namespace":"longhorn-system","name":"longhorn-vault-sync-fresh","creationTimestamp":"` + recent + `","ownerReferences":[{"kind":"ReplicaSet","name":"longhorn-vault-sync"}]},"spec":{"nodeName":"titan-12"},"status":{"phase":"Unknown"}},` +
+		`{"metadata":{"namespace":"maintenance","name":"stale-on-bad-node","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"maintenance"}]},"spec":{"nodeName":"titan-22"},"status":{"phase":"Unknown"}},` +
+		`{"metadata":{"namespace":"default","name":"bare-pod","creationTimestamp":"` + old + `"},"spec":{"nodeName":"titan-12"},"status":{"phase":"Unknown"}}]}`
+
+	deleted := []string{}
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{StuckPodGraceSeconds: 180},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: pods},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: "titan-12\tTrue\ntitan-22\tTrue\n"},
+		{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: `{"items":[]}`},
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-12"},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"titan-22"},"status":{"conditions":[{"type":"Ready","status":"False"}]}}]}`},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "longhorn-system", "delete", "pod", "longhorn-vault-sync-old", "--wait=false")(name, args) {
+					return false
+				}
+				deleted = append(deleted, "longhorn-vault-sync-old")
+				return true
+			},
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "longhorn-system", "delete", "pod", "longhorn-vault-sync-failed", "--wait=false", "--grace-period=0", "--force")(name, args) {
+					return false
+				}
+				deleted = append(deleted, "longhorn-vault-sync-failed")
+				return true
+			},
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "logging", "delete", "pod", "oauth2-proxy-terminating", "--wait=false", "--grace-period=0", "--force")(name, args) {
+					return false
+				}
+				deleted = append(deleted, "oauth2-proxy-terminating")
+				return true
+			},
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "-n", "longhorn-system", "delete", "pod", "pvc-backed-failed", "--wait=false")(name, args) {
+					return false
+				}
+				if strings.Contains(strings.Join(args, " "), "--force") {
+					t.Fatalf("pvc-backed stale pod must not be force deleted")
+				}
+				deleted = append(deleted, "pvc-backed-failed")
+				return true
+			},
+		},
+	})
+
+	if err := orch.recycleStuckControllerPods(context.Background()); err != nil {
+		t.Fatalf("recycleStuckControllerPods failed: %v", err)
+	}
+	if strings.Join(deleted, ",") != "longhorn-vault-sync-old,longhorn-vault-sync-failed,oauth2-proxy-terminating,pvc-backed-failed" {
+		t.Fatalf("expected only stale controller pods on Ready node to be recycled, got %#v", deleted)
+	}
+}
+
+// TestRecycleStuckControllerPodsCordonsContainerRuntimeWedgeNode runs one orchestration or CLI step.
+// Signature: TestRecycleStuckControllerPodsCordonsContainerRuntimeWedgeNode(t *testing.T).
+// Why: a Ready node with a wedged container runtime can trap replacement pods
+// indefinitely; startup should cordon that scheduler target without draining it
+// or touching Longhorn data-plane objects.
+func TestRecycleStuckControllerPodsCordonsContainerRuntimeWedgeNode(t *testing.T) {
+	old := time.Now().Add(-10 * time.Minute).UTC().Format(time.RFC3339)
+	lastSeen := time.Now().UTC().Format(time.RFC3339)
+	pods := `{"items":[` +
+		`{"metadata":{"namespace":"logging","name":"oauth2-proxy-bad","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"oauth2-proxy"}]},"spec":{"nodeName":"titan-18","volumes":[{"name":"scratch"}]},"status":{"phase":"Pending","containerStatuses":[{"name":"oauth2-proxy","state":{"waiting":{"reason":"CreateContainerError"}}}]}},` +
+		`{"metadata":{"namespace":"monitoring","name":"suite-probe-bad","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"Job","name":"suite-probe"}]},"spec":{"nodeName":"titan-18","volumes":[{"name":"scratch"}]},"status":{"phase":"Pending","containerStatuses":[{"name":"probe","state":{"waiting":{"reason":"CreateContainerError"}}}]}},` +
+		`{"metadata":{"namespace":"sso","name":"secret-ensure-bad","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"Job","name":"secret-ensure"}]},"spec":{"nodeName":"titan-18","volumes":[{"name":"scratch"}]},"status":{"phase":"Pending","initContainerStatuses":[{"name":"init","state":{"waiting":{"reason":"CreateContainerError"}}}]}},` +
+		`{"metadata":{"namespace":"finance","name":"single-node-bad","creationTimestamp":"` + old + `","ownerReferences":[{"kind":"ReplicaSet","name":"single"}]},"spec":{"nodeName":"titan-19","volumes":[{"name":"scratch"}]},"status":{"phase":"Pending","containerStatuses":[{"name":"app","state":{"waiting":{"reason":"CreateContainerError"}}}]}}]}`
+	events := `{"items":[` +
+		`{"metadata":{"namespace":"logging","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"logging","name":"oauth2-proxy-bad"},"type":"Warning","reason":"Failed","message":"spec.containers{oauth2-proxy}: Error: failed to reserve container name oauth2-proxy_logging","lastTimestamp":"` + lastSeen + `"},` +
+		`{"metadata":{"namespace":"monitoring","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"monitoring","name":"suite-probe-bad"},"type":"Warning","reason":"Failed","message":"spec.containers{probe}: Error: context deadline exceeded","lastTimestamp":"` + lastSeen + `"},` +
+		`{"metadata":{"namespace":"sso","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"sso","name":"secret-ensure-bad"},"type":"Warning","reason":"Failed","message":"spec.initContainers{init}: Error: failed to reserve container name init_sso","lastTimestamp":"` + lastSeen + `"},` +
+		`{"metadata":{"namespace":"finance","creationTimestamp":"` + lastSeen + `"},"involvedObject":{"kind":"Pod","namespace":"finance","name":"single-node-bad"},"type":"Warning","reason":"Failed","message":"spec.containers{app}: Error: failed to reserve container name app_finance","lastTimestamp":"` + lastSeen + `"}]}`
+
+	cordoned := []string{}
+	deleted := []string{}
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{StuckPodGraceSeconds: 180},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "pods", "-A", "-o", "json"), out: pods},
+		{match: matchContains("kubectl", "-n", "longhorn-system", "get", "nodes.longhorn.io"), out: ""},
+		{match: matchContains("kubectl", "get", "events", "-A", "-o", "json"), out: events},
+		{match: matchContains("kubectl", "get", "nodes", "-o", "json"), out: `{"items":[{"metadata":{"name":"titan-18"},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"titan-19"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}`},
+		{
+			match: func(name string, args []string) bool {
+				if name != "kubectl" || len(args) == 0 || args[0] != "cordon" {
+					return false
+				}
+				cordoned = append(cordoned, args[len(args)-1])
+				return true
+			},
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "delete", "pod", "--wait=false")(name, args) {
+					return false
+				}
+				joined := strings.Join(args, " ")
+				if strings.Contains(joined, "--force") {
+					t.Fatalf("container-runtime wedge recycle must not force-delete fresh pods")
+				}
+				if len(args) >= 5 {
+					deleted = append(deleted, args[4])
+				}
+				return true
+			},
+		},
+	})
+
+	if err := orch.recycleStuckControllerPods(context.Background()); err != nil {
+		t.Fatalf("recycleStuckControllerPods failed: %v", err)
+	}
+	if strings.Join(cordoned, ",") != "titan-18" {
+		t.Fatalf("expected only titan-18 to be cordoned, got %#v", cordoned)
+	}
+	if strings.Join(deleted, ",") != "oauth2-proxy-bad,suite-probe-bad,secret-ensure-bad,single-node-bad" {
+		t.Fatalf("expected runtime-wedged pods to be recycled, got %#v", deleted)
+	}
+}
+
+// TestEffectiveWorkersFiltersIgnoredUnavailableNodes runs one orchestration or CLI step.
+// Signature: TestEffectiveWorkersFiltersIgnoredUnavailableNodes(t *testing.T).
+// Why: ignored unavailable nodes should be excluded before startup tries SSH,
+// k3s-agent start, or uncordon operations against intentionally absent hosts.
+func TestEffectiveWorkersFiltersIgnoredUnavailableNodes(t *testing.T) {
+	cfg := config.Config{
+		Workers: []string{" titan-08 ", "titan-09", "titan-10", "titan-11"},
+		Startup: config.Startup{
+			IgnoreUnavailableNodes: []string{"titan-09", "titan-10"},
+		},
+	}
+	orch := buildOrchestratorWithStubs(t, cfg, nil)
+	got, err := orch.effectiveWorkers(context.Background())
+	if err != nil {
+		t.Fatalf("effectiveWorkers failed: %v", err)
+	}
+	want := []string{"titan-08", "titan-11"}
+	if strings.Join(got, ",") != strings.Join(want, ",") {
+		t.Fatalf("effectiveWorkers mismatch got=%v want=%v", got, want)
+	}
+}
+
+// TestEnsureLonghornEncryptedHostPrereqsFiltersUnsafeWorkers runs one orchestration or CLI step.
+// Signature: TestEnsureLonghornEncryptedHostPrereqsFiltersUnsafeWorkers(t *testing.T).
+// Why: startup must not uncordon Longhorn workers that cannot mount encrypted
+// PVCs; cordoning those nodes is safe and avoids repeating the post-outage
+// mount deadlock.
+func TestEnsureLonghornEncryptedHostPrereqsFiltersUnsafeWorkers(t *testing.T) {
+	cordoned := []string{}
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		SSHManagedNodes: []string{"titan-04", "titan-19"},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: "titan-04\ntitan-19\ntitan-23\n"},
+		{
+			match: matchContains("ssh", "titan-04", "command -v cryptsetup"),
+			out:   "__ANANKE_CRYPTSETUP_PRESENT__",
+		},
+		{
+			match: matchContains("ssh", "titan-19", "apt-get install -y --no-install-recommends cryptsetup-bin"),
+			err:   errors.New("sudo: a password is required"),
+		},
+		{
+			match: func(name string, args []string) bool {
+				if name != "kubectl" || len(args) == 0 || args[0] != "cordon" {
+					return false
+				}
+				if len(args) > 1 {
+					cordoned = append(cordoned, args[len(args)-1])
+				}
+				return true
+			},
+		},
+	})
+
+	got, err := orch.ensureLonghornEncryptedHostPrereqs(context.Background(), []string{"titan-04", "titan-19", "titan-20"})
+	if err != nil {
+		t.Fatalf("ensureLonghornEncryptedHostPrereqs failed: %v", err)
+	}
+	want := []string{"titan-04", "titan-20"}
+	if strings.Join(got, ",") != strings.Join(want, ",") {
+		t.Fatalf("guarded workers mismatch got=%v want=%v", got, want)
+	}
+	if strings.Join(cordoned, ",") != "titan-19,titan-23" {
+		t.Fatalf("expected unsafe longhorn hosts to be cordoned, got %v", cordoned)
+	}
+}
+
+// TestLonghornCryptsetupExemptNodesAreNotQuarantined runs one orchestration or CLI step.
+// Signature: TestLonghornCryptsetupExemptNodesAreNotQuarantined(t *testing.T).
+// Why: Veles/Oceanus uses titan-23 as a Longhorn host for unencrypted local
+// volumes; startup should uncordon that policy-exempt node without requiring
+// host SSH or weakening encrypted-volume safety on other workers.
+func TestLonghornCryptsetupExemptNodesAreNotQuarantined(t *testing.T) {
+	cordoned := []string{}
+	uncordoned := []string{}
+	sshTitan23 := false
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		SSHManagedNodes: []string{"titan-04"},
+		Startup: config.Startup{
+			LonghornCryptsetupExemptNodes: []string{"titan-23"},
+		},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: "titan-04\ntitan-23\n"},
+		{
+			match: matchContains("ssh", "titan-04", "command -v cryptsetup"),
+			out:   "__ANANKE_CRYPTSETUP_PRESENT__",
+		},
+		{
+			match: func(name string, args []string) bool {
+				if name == "ssh" && strings.Contains(strings.Join(args, " "), "titan-23") {
+					sshTitan23 = true
+					return true
+				}
+				return false
+			},
+		},
+		{
+			match: func(name string, args []string) bool {
+				if name != "kubectl" || len(args) == 0 || args[0] != "cordon" {
+					return false
+				}
+				if len(args) > 1 {
+					cordoned = append(cordoned, args[len(args)-1])
+				}
+				return true
+			},
+		},
+		{
+			match: func(name string, args []string) bool {
+				if !matchContains("kubectl", "uncordon")(name, args) {
+					return false
+				}
+				if len(args) > 1 {
+					uncordoned = append(uncordoned, args[len(args)-1])
+				}
+				return true
+			},
+		},
+	})
+
+	got, err := orch.ensureLonghornEncryptedHostPrereqs(context.Background(), []string{"titan-04"})
+	if err != nil {
+		t.Fatalf("ensureLonghornEncryptedHostPrereqs failed: %v", err)
+	}
+	if strings.Join(got, ",") != "titan-04" {
+		t.Fatalf("guarded workers mismatch got=%v", got)
+	}
+	if err := orch.uncordonLonghornCryptsetupExemptNodes(context.Background()); err != nil {
+		t.Fatalf("uncordonLonghornCryptsetupExemptNodes failed: %v", err)
+	}
+	if sshTitan23 {
+		t.Fatalf("did not expect cryptsetup SSH check for exempt titan-23")
+	}
+	if len(cordoned) != 0 {
+		t.Fatalf("did not expect exempt node to be cordoned, got %v", cordoned)
+	}
+	if strings.Join(uncordoned, ",") != "titan-23" {
+		t.Fatalf("expected exempt titan-23 to be uncordoned, got %v", uncordoned)
+	}
+}
+
+// TestLonghornHostNodesFallsBackToConfiguredLabels runs one orchestration or CLI step.
+// Signature: TestLonghornHostNodesFallsBackToConfiguredLabels(t *testing.T).
+// Why: bootstrap caches or minimal test clusters can lack live labels; the
+// static startup inventory should still protect configured storage workers.
+func TestLonghornHostNodesFallsBackToConfiguredLabels(t *testing.T) {
+	orch := buildOrchestratorWithStubs(t, config.Config{
+		Startup: config.Startup{
+			RequiredNodeLabels: map[string]map[string]string{
+				"titan-04": {"longhorn-host": "true"},
+				"titan-20": {"node-role.kubernetes.io/worker": "true"},
+			},
+		},
+	}, []commandStub{
+		{match: matchContains("kubectl", "get", "nodes", "-l", "longhorn-host=true"), out: ""},
+	})
+
+	got, err := orch.longhornHostNodes(context.Background())
+	if err != nil {
+		t.Fatalf("longhornHostNodes failed: %v", err)
+	}
+	if _, ok := got["titan-04"]; !ok || len(got) != 1 {
+		t.Fatalf("expected configured longhorn host fallback, got %v", got)
+	}
+}

internal/cluster/orchestrator_workload_status.go

@@ -0,0 +1,49 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// workloadReady runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) workloadReady(ctx context.Context, w startupWorkload) (bool, error).
+// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
+func (o *Orchestrator) workloadReady(ctx context.Context, w startupWorkload) (bool, error) {
+	out, err := o.kubectl(
+		ctx,
+		20*time.Second,
+		"-n",
+		w.Namespace,
+		"get",
+		w.Kind,
+		w.Name,
+		"-o",
+		"jsonpath={.status.readyReplicas}",
+	)
+	if err != nil {
+		return false, err
+	}
+	raw := strings.TrimSpace(out)
+	if raw == "" || raw == "<no value>" {
+		return false, nil
+	}
+	n, err := strconv.Atoi(raw)
+	if err != nil {
+		return false, fmt.Errorf("parse readyReplicas %q: %w", raw, err)
+	}
+	return n >= 1, nil
+}
+
+// isNotFoundErr runs one orchestration or CLI step.
+// Signature: isNotFoundErr(err error) bool.
+// Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
+func isNotFoundErr(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := strings.ToLower(err.Error())
+	return strings.Contains(msg, "not found") || strings.Contains(msg, "(notfound)")
+}

internal/cluster/testing_hooks_scheduling_storm.go

@@ -0,0 +1,88 @@
+package cluster
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// TestHookMaybeAutoQuarantineSchedulingStorms runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) TestHookMaybeAutoQuarantineSchedulingStorms(ctx context.Context, lastAttempt *time.Time).
+// Why: exposes the scheduling-storm trigger guard to the split top-level test module.
+func (o *Orchestrator) TestHookMaybeAutoQuarantineSchedulingStorms(ctx context.Context, lastAttempt *time.Time) {
+	o.maybeAutoQuarantineSchedulingStorms(ctx, lastAttempt)
+}
+
+// TestHookQuarantineSchedulingStormWorkloads runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) TestHookQuarantineSchedulingStormWorkloads(ctx context.Context) error.
+// Why: exposes the scheduling-storm auto-heal body to the split top-level test module.
+func (o *Orchestrator) TestHookQuarantineSchedulingStormWorkloads(ctx context.Context) error {
+	return o.quarantineSchedulingStormWorkloads(ctx)
+}
+
+// TestHookSchedulingStormOwnerWorkload runs one orchestration or CLI step.
+// Signature: TestHookSchedulingStormOwnerWorkload(namespace string, ownerKind string, ownerName string, rsOwnerKind string, rsOwnerName string) (string, bool).
+// Why: exposes owner-resolution behavior without leaking internal workload types.
+func TestHookSchedulingStormOwnerWorkload(
+	namespace string,
+	ownerKind string,
+	ownerName string,
+	rsOwnerKind string,
+	rsOwnerName string,
+) (string, bool) {
+	var pod podResource
+	pod.Metadata.Namespace = strings.TrimSpace(namespace)
+	pod.Metadata.OwnerReferences = []ownerReference{{
+		Kind: strings.TrimSpace(ownerKind),
+		Name: strings.TrimSpace(ownerName),
+	}}
+	rsOwners := map[string]ownerReference{}
+	if rsName := strings.TrimSpace(ownerName); rsName != "" && strings.TrimSpace(ownerKind) == "ReplicaSet" {
+		rsOwners[pod.Metadata.Namespace+"/"+rsName] = ownerReference{
+			Kind: strings.TrimSpace(rsOwnerKind),
+			Name: strings.TrimSpace(rsOwnerName),
+		}
+	}
+	workload, ok := schedulingStormOwnerWorkload(pod, rsOwners)
+	if !ok {
+		return "", false
+	}
+	return fmt.Sprintf("%s/%s/%s", workload.Namespace, workload.Kind, workload.Name), true
+}
+
+// TestHookEventObservationCount runs one orchestration or CLI step.
+// Signature: TestHookEventObservationCount(count int, seriesCount int) int.
+// Why: exposes event-count normalization used by scheduling-storm detection.
+func TestHookEventObservationCount(count int, seriesCount int) int {
+	return eventObservationCount(eventResource{
+		Count: count,
+		Series: eventSeries{
+			Count: seriesCount,
+		},
+	})
+}
+
+// TestHookEventLastObservedAt runs one orchestration or CLI step.
+// Signature: TestHookEventLastObservedAt(seriesLastObserved time.Time, lastTimestamp time.Time, eventTime time.Time, creationTimestamp time.Time) time.Time.
+// Why: exposes event-time fallback behavior used by scheduling-storm detection.
+func TestHookEventLastObservedAt(
+	seriesLastObserved time.Time,
+	lastTimestamp time.Time,
+	eventTime time.Time,
+	creationTimestamp time.Time,
+) time.Time {
+	return eventLastObservedAt(eventResource{
+		LastTimestamp: lastTimestamp,
+		EventTime:     eventTime,
+		Series: eventSeries{
+			LastObservedTime: seriesLastObserved,
+		},
+		Metadata: struct {
+			Namespace         string    `json:"namespace"`
+			CreationTimestamp time.Time `json:"creationTimestamp"`
+		}{
+			CreationTimestamp: creationTimestamp,
+		},
+	})
+}

internal/cluster/testing_hooks_startup.go

@@ -0,0 +1,55 @@
+package cluster
+
+import "context"
+
+// TestHookStartupRequiredNodes runs one orchestration or CLI step.
+// Signature: TestHookStartupRequiredNodes(nodes []string, required []string) []string.
+// Why: exposes recovery-scope node filtering so top-level tests can cover core-only startup narrowing.
+func TestHookStartupRequiredNodes(nodes []string, required []string) []string {
+	return startupRequiredNodes(nodes, required)
+}
+
+// TestHookContainsNode runs one orchestration or CLI step.
+// Signature: TestHookContainsNode(entries []string, needle string) bool.
+// Why: exposes the small startup-scope membership helper to top-level tests.
+func TestHookContainsNode(entries []string, needle string) bool {
+	return containsNode(entries, needle)
+}
+
+// TestHookStartupNodeStrictlyRequired runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) TestHookStartupNodeStrictlyRequired(node string) bool.
+// Why: exposes strict-node startup scoping so outage-recovery tests can confirm
+// non-core nodes stop blocking bootstrap.
+func (o *Orchestrator) TestHookStartupNodeStrictlyRequired(node string) bool {
+	return o.startupNodeStrictlyRequired(node)
+}
+
+// TestHookStartupRequiredFluxKustomizations runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) TestHookStartupRequiredFluxKustomizations() map[string]struct{}.
+// Why: exposes flux startup scoping so top-level tests can confirm only core
+// kustomizations block emergency bootstrap.
+func (o *Orchestrator) TestHookStartupRequiredFluxKustomizations() map[string]struct{} {
+	return o.startupRequiredFluxKustomizations()
+}
+
+// TestHookStartupRequiredWorkloadNamespaces runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) TestHookStartupRequiredWorkloadNamespaces() map[string]struct{}.
+// Why: exposes workload namespace startup scoping so top-level tests can
+// confirm only core workloads block emergency bootstrap.
+func (o *Orchestrator) TestHookStartupRequiredWorkloadNamespaces() map[string]struct{} {
+	return o.startupRequiredWorkloadNamespaces()
+}
+
+// TestHookMaybeRunEarlyVaultUnseal runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) TestHookMaybeRunEarlyVaultUnseal(ctx context.Context).
+// Why: exposes the early startup Vault deferral helper to top-level tests.
+func (o *Orchestrator) TestHookMaybeRunEarlyVaultUnseal(ctx context.Context) {
+	o.maybeRunEarlyVaultUnseal(ctx)
+}
+
+// TestHookRunStartupVaultUnsealGate runs one orchestration or CLI step.
+// Signature: (o *Orchestrator) TestHookRunStartupVaultUnsealGate(ctx context.Context) error.
+// Why: exposes the startup Vault gate helper to top-level tests.
+func (o *Orchestrator) TestHookRunStartupVaultUnsealGate(ctx context.Context) error {
+	return o.runStartupVaultUnsealGate(ctx)
+}

internal/config/apply_defaults.go

@@ -33,6 +33,9 @@ func (c *Config) applyDefaults() {
 	if c.Startup.NodeInventoryReachPollSeconds <= 0 {
 		c.Startup.NodeInventoryReachPollSeconds = 5
 	}
+	if c.Startup.NodeInventoryReachRequiredNodes == nil {
+		c.Startup.NodeInventoryReachRequiredNodes = []string{}
+	}
 	if c.Startup.RequiredNodeLabels == nil {
 		c.Startup.RequiredNodeLabels = map[string]map[string]string{
 			"titan-09": {
@@ -40,6 +43,10 @@ func (c *Config) applyDefaults() {
 			},
 		}
 	}
+	c.Startup.RequiredNodeLabelsMode = strings.ToLower(strings.TrimSpace(c.Startup.RequiredNodeLabelsMode))
+	if c.Startup.RequiredNodeLabelsMode == "" {
+		c.Startup.RequiredNodeLabelsMode = "enforce"
+	}
 	if c.Startup.TimeSyncWaitSeconds <= 0 {
 		c.Startup.TimeSyncWaitSeconds = 240
 	}
@@ -121,12 +128,26 @@ func (c *Config) applyDefaults() {
 	if strings.TrimSpace(c.Startup.ServiceChecklistAuth.AdminSecretPasswordKey) == "" {
 		c.Startup.ServiceChecklistAuth.AdminSecretPasswordKey = "password"
 	}
-	c.Startup.ServiceChecklist = mergeServiceChecklistDefaults(c.Startup.ServiceChecklist, defaultServiceChecklist())
+	if c.Startup.ServiceChecklistExplicitOnly {
+		c.Startup.ServiceChecklist = mergeServiceChecklistDefaults(c.Startup.ServiceChecklist, []ServiceChecklistCheck{})
+	} else {
+		c.Startup.ServiceChecklist = mergeServiceChecklistDefaults(c.Startup.ServiceChecklist, defaultServiceChecklist())
+	}
+	if c.Startup.TCPServiceChecklistExplicitOnly {
+		c.Startup.TCPServiceChecklist = mergeTCPServiceChecklistDefaults(c.Startup.TCPServiceChecklist, []TCPServiceChecklistCheck{})
+	} else {
+		c.Startup.TCPServiceChecklist = mergeTCPServiceChecklistDefaults(c.Startup.TCPServiceChecklist, defaultTCPServiceChecklist())
+	}
 	for i := range c.Startup.ServiceChecklist {
 		if c.Startup.ServiceChecklist[i].TimeoutSeconds <= 0 {
 			c.Startup.ServiceChecklist[i].TimeoutSeconds = 12
 		}
 	}
+	for i := range c.Startup.TCPServiceChecklist {
+		if c.Startup.TCPServiceChecklist[i].TimeoutSeconds <= 0 {
+			c.Startup.TCPServiceChecklist[i].TimeoutSeconds = 8
+		}
+	}
 	if c.Startup.CriticalServiceEndpointWaitSec <= 0 {
 		c.Startup.CriticalServiceEndpointWaitSec = 420
 	}
@@ -134,6 +155,9 @@ func (c *Config) applyDefaults() {
 		c.Startup.CriticalServiceEndpointPollSec = 5
 	}
 	c.Startup.CriticalServiceEndpoints = mergeStringDefaults(c.Startup.CriticalServiceEndpoints, defaultCriticalServiceEndpoints())
+	if c.Startup.CriticalServiceStartupProbeThreshold <= 0 {
+		c.Startup.CriticalServiceStartupProbeThreshold = 30
+	}
 	if c.Startup.IngressChecklistWaitSeconds <= 0 {
 		c.Startup.IngressChecklistWaitSeconds = 420
 	}
@@ -152,12 +176,18 @@ func (c *Config) applyDefaults() {
 	if c.Startup.NodeSSHAuthPollSeconds <= 0 {
 		c.Startup.NodeSSHAuthPollSeconds = 5
 	}
+	if c.Startup.NodeSSHAuthRequiredNodes == nil {
+		c.Startup.NodeSSHAuthRequiredNodes = []string{}
+	}
 	if c.Startup.FluxHealthWaitSeconds <= 0 {
 		c.Startup.FluxHealthWaitSeconds = 900
 	}
 	if c.Startup.FluxHealthPollSeconds <= 0 {
 		c.Startup.FluxHealthPollSeconds = 5
 	}
+	if c.Startup.FluxHealthRequiredKustomizations == nil {
+		c.Startup.FluxHealthRequiredKustomizations = []string{}
+	}
 	if c.Startup.IgnoreFluxKustomizations == nil {
 		c.Startup.IgnoreFluxKustomizations = []string{}
 	}
@@ -167,6 +197,9 @@ func (c *Config) applyDefaults() {
 	if c.Startup.WorkloadConvergencePollSeconds <= 0 {
 		c.Startup.WorkloadConvergencePollSeconds = 5
 	}
+	if c.Startup.WorkloadConvergenceRequiredNamespaces == nil {
+		c.Startup.WorkloadConvergenceRequiredNamespaces = []string{}
+	}
 	if c.Startup.IgnoreWorkloadNamespaces == nil {
 		c.Startup.IgnoreWorkloadNamespaces = []string{}
 	}
@@ -176,9 +209,33 @@ func (c *Config) applyDefaults() {
 	if c.Startup.IgnoreUnavailableNodes == nil {
 		c.Startup.IgnoreUnavailableNodes = []string{}
 	}
+	if c.Startup.LonghornCryptsetupExemptNodes == nil {
+		c.Startup.LonghornCryptsetupExemptNodes = []string{}
+	}
 	if c.Startup.StuckPodGraceSeconds <= 0 {
 		c.Startup.StuckPodGraceSeconds = 180
 	}
+	if c.Startup.PostStartAutoHealSeconds <= 0 {
+		c.Startup.PostStartAutoHealSeconds = 60
+	}
+	if c.Startup.RecoveryCordonMaxSeconds <= 0 {
+		c.Startup.RecoveryCordonMaxSeconds = 3600
+	}
+	if c.Startup.DeadNodeCleanupGraceSeconds <= 0 {
+		c.Startup.DeadNodeCleanupGraceSeconds = 300
+	}
+	if strings.TrimSpace(c.Startup.HostSudoSecretPasswordKey) == "" {
+		c.Startup.HostSudoSecretPasswordKey = "password"
+	}
+	if c.Startup.HostPrivilegedCommandTimeoutSec <= 0 {
+		c.Startup.HostPrivilegedCommandTimeoutSec = 90
+	}
+	if c.Startup.NodeRuntimeRestartWaitSeconds <= 0 {
+		c.Startup.NodeRuntimeRestartWaitSeconds = 140
+	}
+	if c.Startup.NodeRuntimeRebootWaitSeconds <= 0 {
+		c.Startup.NodeRuntimeRebootWaitSeconds = 420
+	}
 	if strings.TrimSpace(c.Startup.VaultUnsealKeyFile) == "" {
 		c.Startup.VaultUnsealKeyFile = "/var/lib/ananke/vault-unseal.key"
 	}
@@ -221,6 +278,12 @@ func (c *Config) applyDefaults() {
 	if c.UPS.TelemetryTimeoutSeconds <= 0 {
 		c.UPS.TelemetryTimeoutSeconds = 90
 	}
+	if c.Startup.SchedulingStormEventThreshold <= 0 {
+		c.Startup.SchedulingStormEventThreshold = 30
+	}
+	if c.Startup.SchedulingStormWindowSeconds <= 0 {
+		c.Startup.SchedulingStormWindowSeconds = 180
+	}
 	if c.Coordination.ForwardShutdownConfig == "" {
 		c.Coordination.ForwardShutdownConfig = "/etc/ananke/ananke.yaml"
 	}
@@ -242,6 +305,9 @@ func (c *Config) applyDefaults() {
 	if c.Metrics.Path == "" {
 		c.Metrics.Path = "/metrics"
 	}
+	if c.GitOps.PollSeconds <= 0 {
+		c.GitOps.PollSeconds = 60
+	}
 	if c.State.Dir == "" {
 		c.State.Dir = "/var/lib/ananke"
 	}

internal/config/config_test.go

@@ -132,6 +132,71 @@ state:
 	}
 }
 
+// TestLoadDefaultsCriticalStartupProbeRepairWhenOmitted runs one orchestration or CLI step.
+// Signature: TestLoadDefaultsCriticalStartupProbeRepairWhenOmitted(t *testing.T).
+// Why: existing configs with a startup block should inherit the Mailu hardening
+// repair instead of silently decoding the new boolean as false.
+func TestLoadDefaultsCriticalStartupProbeRepairWhenOmitted(t *testing.T) {
+	tmp := t.TempDir()
+	cfgPath := filepath.Join(tmp, "ananke.yaml")
+	raw := `
+control_planes: [titan-0a, titan-0b, titan-0c]
+expected_flux_branch: main
+iac_repo_path: /opt/titan-iac
+startup:
+  api_wait_seconds: 900
+ups:
+  enabled: false
+state:
+  run_history_path: /tmp/runs.json
+  lock_path: /tmp/ananke.lock
+`
+	if err := os.WriteFile(cfgPath, []byte(strings.TrimSpace(raw)), 0o644); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+	cfg, err := Load(cfgPath)
+	if err != nil {
+		t.Fatalf("load config: %v", err)
+	}
+	if !cfg.Startup.CriticalServiceStartupProbeRepair {
+		t.Fatalf("expected omitted startup-probe repair setting to default true")
+	}
+	if cfg.Startup.CriticalServiceStartupProbeThreshold != 30 {
+		t.Fatalf("expected startup-probe threshold default 30, got %d", cfg.Startup.CriticalServiceStartupProbeThreshold)
+	}
+}
+
+// TestLoadHonorsExplicitCriticalStartupProbeRepairFalse runs one orchestration or CLI step.
+// Signature: TestLoadHonorsExplicitCriticalStartupProbeRepairFalse(t *testing.T).
+// Why: operators should still be able to disable the repair explicitly even
+// though omitted values inherit the hardening default.
+func TestLoadHonorsExplicitCriticalStartupProbeRepairFalse(t *testing.T) {
+	tmp := t.TempDir()
+	cfgPath := filepath.Join(tmp, "ananke.yaml")
+	raw := `
+control_planes: [titan-0a, titan-0b, titan-0c]
+expected_flux_branch: main
+iac_repo_path: /opt/titan-iac
+startup:
+  critical_service_startup_probe_repair: false
+ups:
+  enabled: false
+state:
+  run_history_path: /tmp/runs.json
+  lock_path: /tmp/ananke.lock
+`
+	if err := os.WriteFile(cfgPath, []byte(strings.TrimSpace(raw)), 0o644); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+	cfg, err := Load(cfgPath)
+	if err != nil {
+		t.Fatalf("load config: %v", err)
+	}
+	if cfg.Startup.CriticalServiceStartupProbeRepair {
+		t.Fatalf("expected explicit startup-probe repair false to be honored")
+	}
+}
+
 // TestValidateRejectsInvalidStartupShutdownCooldown runs one orchestration or CLI step.
 // Signature: TestValidateRejectsInvalidStartupShutdownCooldown(t *testing.T).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

internal/config/defaults.go

@@ -39,24 +39,26 @@ func defaults() Config {
 			"maintenance",
 		},
 		Startup: Startup{
-			APIWaitSeconds:                1200,
+			APIWaitSeconds:                  1200,
-			APIPollSeconds:                2,
+			APIPollSeconds:                  2,
-			ShutdownCooldownSeconds:       45,
+			ShutdownCooldownSeconds:         45,
-			RequireNodeInventoryReach:     true,
+			RequireNodeInventoryReach:       true,
-			NodeInventoryReachWaitSeconds: 300,
+			NodeInventoryReachWaitSeconds:   300,
-			NodeInventoryReachPollSeconds: 5,
+			NodeInventoryReachPollSeconds:   5,
-			RequireTimeSync:               true,
+			NodeInventoryReachRequiredNodes: []string{},
-			TimeSyncWaitSeconds:           240,
+			LonghornCryptsetupExemptNodes:   []string{},
-			TimeSyncPollSeconds:           5,
+			RequireTimeSync:                 true,
-			TimeSyncMode:                  "quorum",
+			TimeSyncWaitSeconds:             240,
-			TimeSyncQuorum:                2,
+			TimeSyncPollSeconds:             5,
-			ReconcileAccessOnBoot:         true,
+			TimeSyncMode:                    "quorum",
-			AutoEtcdRestoreOnAPIFailure:   true,
+			TimeSyncQuorum:                  2,
-			EtcdRestoreControlPlane:       "titan-0a",
+			ReconcileAccessOnBoot:           true,
-			RequireStorageReady:           true,
+			AutoEtcdRestoreOnAPIFailure:     true,
-			StorageReadyWaitSeconds:       420,
+			EtcdRestoreControlPlane:         "titan-0a",
-			StorageReadyPollSeconds:       5,
+			RequireStorageReady:             true,
-			StorageMinReadyNodes:          2,
+			StorageReadyWaitSeconds:         420,
+			StorageReadyPollSeconds:         5,
+			StorageMinReadyNodes:            2,
 			StorageCriticalPVCs: []string{
 				"vault/data-vault-0",
 				"postgres/postgres-data-postgres-0",
@@ -69,6 +71,7 @@ func defaults() Config {
 					"ananke.bstein.dev/harbor-bootstrap": "true",
 				},
 			},
+			RequiredNodeLabelsMode:    "enforce",
 			RequirePostStartProbes:    true,
 			PostStartProbeWaitSeconds: 240,
 			PostStartProbePollSeconds: 5,
@@ -91,33 +94,44 @@ func defaults() Config {
 				AdminSecretUsernameKey: "username",
 				AdminSecretPasswordKey: "password",
 			},
-			ServiceChecklist:                defaultServiceChecklist(),
+			ServiceChecklist:                      defaultServiceChecklist(),
-			RequireCriticalServiceEndpoints: true,
+			TCPServiceChecklist:                   defaultTCPServiceChecklist(),
-			CriticalServiceEndpointWaitSec:  420,
+			RequireCriticalServiceEndpoints:       true,
-			CriticalServiceEndpointPollSec:  5,
+			CriticalServiceEndpointWaitSec:        420,
-			CriticalServiceEndpoints:        defaultCriticalServiceEndpoints(),
+			CriticalServiceEndpointPollSec:        5,
-			RequireIngressChecklist:         true,
+			CriticalServiceEndpoints:              defaultCriticalServiceEndpoints(),
-			IngressChecklistWaitSeconds:     420,
+			CriticalServiceStartupProbeRepair:     true,
-			IngressChecklistPollSeconds:     5,
+			CriticalServiceStartupProbeThreshold:  30,
-			IngressChecklistAccepted:        []int{200, 301, 302, 307, 308, 401, 403, 404},
+			RequireIngressChecklist:               true,
-			IngressChecklistIgnoreHosts:     []string{},
+			IngressChecklistWaitSeconds:           420,
-			RequireNodeSSHAuth:              true,
+			IngressChecklistPollSeconds:           5,
-			NodeSSHAuthWaitSeconds:          240,
+			IngressChecklistAccepted:              []int{200, 301, 302, 307, 308, 401, 403, 404},
-			NodeSSHAuthPollSeconds:          5,
+			IngressChecklistIgnoreHosts:           []string{},
-			RequireFluxHealth:               true,
+			RequireNodeSSHAuth:                    true,
-			FluxHealthWaitSeconds:           900,
+			NodeSSHAuthWaitSeconds:                240,
-			FluxHealthPollSeconds:           5,
+			NodeSSHAuthPollSeconds:                5,
-			IgnoreFluxKustomizations:        []string{},
+			NodeSSHAuthRequiredNodes:              []string{},
-			RequireWorkloadConvergence:      true,
+			RequireFluxHealth:                     true,
-			WorkloadConvergenceWaitSeconds:  900,
+			FluxHealthWaitSeconds:                 900,
-			WorkloadConvergencePollSeconds:  5,
+			FluxHealthPollSeconds:                 5,
-			IgnoreWorkloadNamespaces:        []string{},
+			FluxHealthRequiredKustomizations:      []string{},
-			IgnoreWorkloads:                 []string{},
+			IgnoreFluxKustomizations:              []string{},
-			IgnoreUnavailableNodes:          []string{},
+			RequireWorkloadConvergence:            true,
-			AutoRecycleStuckPods:            true,
+			WorkloadConvergenceWaitSeconds:        900,
-			StuckPodGraceSeconds:            180,
+			WorkloadConvergencePollSeconds:        5,
-			VaultUnsealKeyFile:              "/var/lib/ananke/vault-unseal.key",
+			WorkloadConvergenceRequiredNamespaces: []string{},
-			VaultUnsealBreakglassTimeout:    15,
+			IgnoreWorkloadNamespaces:              []string{},
+			IgnoreWorkloads:                       []string{},
+			IgnoreUnavailableNodes:                []string{},
+			AutoRecycleStuckPods:                  true,
+			StuckPodGraceSeconds:                  180,
+			RecoveryCordonMaxSeconds:              3600,
+			HostSudoSecretPasswordKey:             "password",
+			HostPrivilegedCommandTimeoutSec:       90,
+			NodeRuntimeRestartWaitSeconds:         140,
+			NodeRuntimeRebootWaitSeconds:          420,
+			VaultUnsealKeyFile:                    "/var/lib/ananke/vault-unseal.key",
+			VaultUnsealBreakglassTimeout:          15,
 		},
 		Shutdown: Shutdown{
 			DefaultBudgetSeconds: 1380,
@@ -152,6 +166,10 @@ func defaults() Config {
 			BindAddr: "0.0.0.0:9560",
 			Path:     "/metrics",
 		},
+		GitOps: GitOps{
+			Enabled:     true,
+			PollSeconds: 60,
+		},
 		State: State{
 			Dir:            "/var/lib/ananke",
 			ReportsDir:     "/var/lib/ananke/reports",

internal/config/load.go

@@ -7,23 +7,147 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+type inventoryFragment struct {
+	SSHUser         *string            `yaml:"ssh_user"`
+	SSHPort         *int               `yaml:"ssh_port"`
+	SSHConfigFile   *string            `yaml:"ssh_config_file"`
+	SSHIdentityFile *string            `yaml:"ssh_identity_file"`
+	SSHNodeHosts    *map[string]string `yaml:"ssh_node_hosts"`
+	SSHNodeUsers    *map[string]string `yaml:"ssh_node_users"`
+	SSHManagedNodes *[]string          `yaml:"ssh_managed_nodes"`
+	ControlPlanes   *[]string          `yaml:"control_planes"`
+	Workers         *[]string          `yaml:"workers"`
+	Startup         *struct {
+		RequiredNodeLabels     *map[string]map[string]string `yaml:"required_node_labels"`
+		RequiredNodeLabelsMode *string                       `yaml:"required_node_labels_mode"`
+		IgnoreUnavailableNodes *[]string                     `yaml:"ignore_unavailable_nodes"`
+	} `yaml:"startup"`
+}
+
 // Load runs one orchestration or CLI step.
 // Signature: Load(path string) (Config, error).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
 func Load(path string) (Config, error) {
+	return LoadWithFragments(path)
+}
+
+// LoadWithFragments runs one orchestration or CLI step.
+// Signature: LoadWithFragments(path string, fragments ...string) (Config, error).
+// Why: Terraform can generate a narrow inventory fragment while Ananke keeps
+// loading the ordinary host runtime config for imperative recovery behavior.
+func LoadWithFragments(path string, fragments ...string) (Config, error) {
 	cfg := defaults()
 
 	b, err := os.ReadFile(path)
 	if err != nil {
 		return Config{}, fmt.Errorf("read config %s: %w", path, err)
 	}
+	startupProbeRepairConfigured, err := yamlPathExists(b, "startup", "critical_service_startup_probe_repair")
+	if err != nil {
+		return Config{}, fmt.Errorf("decode config %s: %w", path, err)
+	}
 	if err := yaml.Unmarshal(b, &cfg); err != nil {
 		return Config{}, fmt.Errorf("decode config %s: %w", path, err)
 	}
+	for _, fragment := range fragments {
+		fb, err := os.ReadFile(fragment)
+		if err != nil {
+			return Config{}, fmt.Errorf("read config fragment %s: %w", fragment, err)
+		}
+		if err := applyInventoryFragment(fb, &cfg); err != nil {
+			return Config{}, fmt.Errorf("decode config fragment %s: %w", fragment, err)
+		}
+	}
 
 	cfg.applyDefaults()
+	if !startupProbeRepairConfigured {
+		cfg.Startup.CriticalServiceStartupProbeRepair = defaults().Startup.CriticalServiceStartupProbeRepair
+	}
 	if err := cfg.Validate(); err != nil {
 		return Config{}, err
 	}
 	return cfg, nil
 }
+
+// applyInventoryFragment runs one orchestration or CLI step.
+// Signature: applyInventoryFragment(raw []byte, cfg *Config) error.
+// Why: restricts generated Terraform fragments to declarative inventory fields
+// so they cannot accidentally take over Ananke's recovery behavior.
+func applyInventoryFragment(raw []byte, cfg *Config) error {
+	var fragment inventoryFragment
+	if err := yaml.Unmarshal(raw, &fragment); err != nil {
+		return err
+	}
+	if fragment.SSHUser != nil {
+		cfg.SSHUser = *fragment.SSHUser
+	}
+	if fragment.SSHPort != nil {
+		cfg.SSHPort = *fragment.SSHPort
+	}
+	if fragment.SSHConfigFile != nil {
+		cfg.SSHConfigFile = *fragment.SSHConfigFile
+	}
+	if fragment.SSHIdentityFile != nil {
+		cfg.SSHIdentityFile = *fragment.SSHIdentityFile
+	}
+	if fragment.SSHNodeHosts != nil {
+		cfg.SSHNodeHosts = *fragment.SSHNodeHosts
+	}
+	if fragment.SSHNodeUsers != nil {
+		cfg.SSHNodeUsers = *fragment.SSHNodeUsers
+	}
+	if fragment.SSHManagedNodes != nil {
+		cfg.SSHManagedNodes = *fragment.SSHManagedNodes
+	}
+	if fragment.ControlPlanes != nil {
+		cfg.ControlPlanes = *fragment.ControlPlanes
+	}
+	if fragment.Workers != nil {
+		cfg.Workers = *fragment.Workers
+	}
+	if fragment.Startup == nil {
+		return nil
+	}
+	if fragment.Startup.RequiredNodeLabels != nil {
+		cfg.Startup.RequiredNodeLabels = *fragment.Startup.RequiredNodeLabels
+	}
+	if fragment.Startup.RequiredNodeLabelsMode != nil {
+		cfg.Startup.RequiredNodeLabelsMode = *fragment.Startup.RequiredNodeLabelsMode
+	}
+	if fragment.Startup.IgnoreUnavailableNodes != nil {
+		cfg.Startup.IgnoreUnavailableNodes = *fragment.Startup.IgnoreUnavailableNodes
+	}
+	return nil
+}
+
+// yamlPathExists runs one orchestration or CLI step.
+// Signature: yamlPathExists(raw []byte, path ...string) (bool, error).
+// Why: selected boolean defaults need to distinguish an omitted YAML key from an
+// explicit false value after nested startup mappings are decoded.
+func yamlPathExists(raw []byte, path ...string) (bool, error) {
+	var root yaml.Node
+	if err := yaml.Unmarshal(raw, &root); err != nil {
+		return false, err
+	}
+	node := &root
+	if node.Kind == yaml.DocumentNode && len(node.Content) > 0 {
+		node = node.Content[0]
+	}
+	for _, key := range path {
+		if node.Kind != yaml.MappingNode {
+			return false, nil
+		}
+		found := false
+		for i := 0; i+1 < len(node.Content); i += 2 {
+			if node.Content[i].Value == key {
+				node = node.Content[i+1]
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false, nil
+		}
+	}
+	return true, nil
+}

internal/config/load_additional_test.go

@@ -51,3 +51,224 @@ startup:
 		t.Fatalf("expected validation failure")
 	}
 }
+
+// TestLoadKeepsExplicitServiceChecklist runs one orchestration or CLI step.
+// Signature: TestLoadKeepsExplicitServiceChecklist(t *testing.T).
+// Why: host recovery configs must be able to keep a narrow, explicit checklist
+// without silently inheriting the full default service catalog.
+func TestLoadKeepsExplicitServiceChecklist(t *testing.T) {
+	cfgPath := filepath.Join(t.TempDir(), "ananke.yaml")
+	raw := `
+control_planes: [titan-0a]
+expected_flux_branch: main
+expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
+iac_repo_path: /opt/titan-iac
+startup:
+  service_checklist_explicit_only: true
+  service_checklist:
+    - name: gitea-api
+      url: https://scm.bstein.dev/api/healthz
+      accepted_statuses: [200]
+      body_contains: pass
+      timeout_seconds: 12
+ups:
+  enabled: false
+`
+	if err := os.WriteFile(cfgPath, []byte(raw), 0o644); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+
+	cfg, err := Load(cfgPath)
+	if err != nil {
+		t.Fatalf("load config: %v", err)
+	}
+	if len(cfg.Startup.ServiceChecklist) != 1 {
+		t.Fatalf("expected 1 explicit service check, got %d", len(cfg.Startup.ServiceChecklist))
+	}
+	if cfg.Startup.ServiceChecklist[0].Name != "gitea-api" {
+		t.Fatalf("expected explicit gitea-api check, got %q", cfg.Startup.ServiceChecklist[0].Name)
+	}
+}
+
+// TestLoadWithFragmentsReplacesGeneratedInventory runs one orchestration or CLI step.
+// Signature: TestLoadWithFragmentsReplacesGeneratedInventory(t *testing.T).
+// Why: Terraform-generated inventory fragments must be authoritative for the
+// declarative bootstrap fields they own, without merging stale host config.
+func TestLoadWithFragmentsReplacesGeneratedInventory(t *testing.T) {
+	tmp := t.TempDir()
+	basePath := filepath.Join(tmp, "ananke.yaml")
+	fragmentPath := filepath.Join(tmp, "ananke.inventory.yaml")
+	base := `
+control_planes: [old-cp]
+workers: [old-worker]
+ssh_user: atlas
+ssh_port: 2277
+ssh_node_hosts:
+  old-cp: 192.0.2.10
+ssh_managed_nodes: [old-cp, old-worker]
+expected_flux_branch: main
+expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
+iac_repo_path: /opt/titan-iac
+startup:
+  required_node_labels:
+    old-worker:
+      old-label: "true"
+  ignore_unavailable_nodes: [old-worker]
+ups:
+  enabled: false
+`
+	fragment := `
+ssh_user: atlas
+ssh_port: 2277
+ssh_config_file: /home/atlas/.ssh/config
+ssh_identity_file: /home/atlas/.ssh/id_ed25519
+ssh_node_hosts:
+  titan-0a: 192.168.22.11
+  titan-04: 192.168.22.30
+ssh_node_users:
+  titan-04: workeradmin
+ssh_managed_nodes: [titan-0a, titan-04]
+control_planes: [titan-0a]
+workers: [titan-04]
+startup:
+  required_node_labels_mode: validate
+  required_node_labels:
+    titan-04:
+      node-role.kubernetes.io/worker: "true"
+  ignore_unavailable_nodes: []
+`
+	if err := os.WriteFile(basePath, []byte(base), 0o644); err != nil {
+		t.Fatalf("write base config: %v", err)
+	}
+	if err := os.WriteFile(fragmentPath, []byte(fragment), 0o644); err != nil {
+		t.Fatalf("write fragment config: %v", err)
+	}
+
+	cfg, err := LoadWithFragments(basePath, fragmentPath)
+	if err != nil {
+		t.Fatalf("load config with fragment: %v", err)
+	}
+	if len(cfg.ControlPlanes) != 1 || cfg.ControlPlanes[0] != "titan-0a" {
+		t.Fatalf("expected fragment control planes, got %v", cfg.ControlPlanes)
+	}
+	if _, ok := cfg.SSHNodeHosts["old-cp"]; ok {
+		t.Fatalf("expected fragment ssh_node_hosts to replace base map, got %#v", cfg.SSHNodeHosts)
+	}
+	if cfg.SSHNodeUsers["titan-04"] != "workeradmin" {
+		t.Fatalf("expected fragment ssh_node_users override, got %#v", cfg.SSHNodeUsers)
+	}
+	if _, ok := cfg.Startup.RequiredNodeLabels["old-worker"]; ok {
+		t.Fatalf("expected fragment required_node_labels to replace base map, got %#v", cfg.Startup.RequiredNodeLabels)
+	}
+	if cfg.Startup.RequiredNodeLabelsMode != "validate" {
+		t.Fatalf("expected fragment label mode validate, got %q", cfg.Startup.RequiredNodeLabelsMode)
+	}
+	if len(cfg.Startup.IgnoreUnavailableNodes) != 0 {
+		t.Fatalf("expected fragment ignored unavailable nodes to replace base slice, got %v", cfg.Startup.IgnoreUnavailableNodes)
+	}
+}
+
+// TestLoadWithFragmentsAllowsTopLevelInventoryOnly runs one orchestration or CLI step.
+// Signature: TestLoadWithFragmentsAllowsTopLevelInventoryOnly(t *testing.T).
+// Why: generated inventory fragments may omit startup labels during staged
+// rollout while still replacing host/node inventory.
+func TestLoadWithFragmentsAllowsTopLevelInventoryOnly(t *testing.T) {
+	tmp := t.TempDir()
+	basePath := filepath.Join(tmp, "ananke.yaml")
+	fragmentPath := filepath.Join(tmp, "ananke.inventory.yaml")
+	base := `
+control_planes: [old-cp]
+workers: [old-worker]
+ssh_user: atlas
+ssh_port: 2277
+ssh_node_hosts:
+  old-cp: 192.0.2.10
+ssh_managed_nodes: [old-cp, old-worker]
+expected_flux_branch: main
+expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
+iac_repo_path: /opt/titan-iac
+startup:
+  required_node_labels_mode: enforce
+  required_node_labels:
+    old-worker:
+      old-label: "true"
+ups:
+  enabled: false
+`
+	fragment := `
+ssh_node_hosts:
+  titan-0a: 192.168.22.11
+ssh_managed_nodes: [titan-0a]
+control_planes: [titan-0a]
+workers: []
+`
+	if err := os.WriteFile(basePath, []byte(base), 0o644); err != nil {
+		t.Fatalf("write base config: %v", err)
+	}
+	if err := os.WriteFile(fragmentPath, []byte(fragment), 0o644); err != nil {
+		t.Fatalf("write fragment config: %v", err)
+	}
+
+	cfg, err := LoadWithFragments(basePath, fragmentPath)
+	if err != nil {
+		t.Fatalf("load config with top-level fragment: %v", err)
+	}
+	if len(cfg.ControlPlanes) != 1 || cfg.ControlPlanes[0] != "titan-0a" {
+		t.Fatalf("expected top-level fragment control plane, got %v", cfg.ControlPlanes)
+	}
+	if _, ok := cfg.Startup.RequiredNodeLabels["old-worker"]; !ok {
+		t.Fatalf("expected startup labels to remain when fragment omits startup, got %#v", cfg.Startup.RequiredNodeLabels)
+	}
+	if cfg.Startup.RequiredNodeLabelsMode != "enforce" {
+		t.Fatalf("expected base label mode to remain enforce, got %q", cfg.Startup.RequiredNodeLabelsMode)
+	}
+}
+
+// TestLoadWithFragmentsFailsOnMissingFragment runs one orchestration or CLI step.
+// Signature: TestLoadWithFragmentsFailsOnMissingFragment(t *testing.T).
+// Why: installer and systemd mistakes should fail closed instead of silently
+// running with stale embedded inventory.
+func TestLoadWithFragmentsFailsOnMissingFragment(t *testing.T) {
+	cfgPath := filepath.Join(t.TempDir(), "ananke.yaml")
+	raw := `
+control_planes: [titan-0a]
+expected_flux_branch: main
+expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
+iac_repo_path: /opt/titan-iac
+ups:
+  enabled: false
+`
+	if err := os.WriteFile(cfgPath, []byte(raw), 0o644); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+	if _, err := LoadWithFragments(cfgPath, filepath.Join(t.TempDir(), "missing.yaml")); err == nil {
+		t.Fatalf("expected missing fragment error")
+	}
+}
+
+// TestLoadWithFragmentsFailsOnBadFragmentYAML runs one orchestration or CLI step.
+// Signature: TestLoadWithFragmentsFailsOnBadFragmentYAML(t *testing.T).
+// Why: malformed generated inventory must fail closed instead of running with
+// partially decoded declarative state.
+func TestLoadWithFragmentsFailsOnBadFragmentYAML(t *testing.T) {
+	tmp := t.TempDir()
+	cfgPath := filepath.Join(tmp, "ananke.yaml")
+	fragmentPath := filepath.Join(tmp, "ananke.inventory.yaml")
+	raw := `
+control_planes: [titan-0a]
+expected_flux_branch: main
+expected_flux_source_url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git
+iac_repo_path: /opt/titan-iac
+ups:
+  enabled: false
+`
+	if err := os.WriteFile(cfgPath, []byte(raw), 0o644); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+	if err := os.WriteFile(fragmentPath, []byte(":\n- invalid"), 0o644); err != nil {
+		t.Fatalf("write bad fragment: %v", err)
+	}
+	if _, err := LoadWithFragments(cfgPath, fragmentPath); err == nil {
+		t.Fatalf("expected bad fragment decode error")
+	}
+}

internal/config/startup_service_catalog.go

@@ -251,6 +251,57 @@ func defaultServiceChecklist() []ServiceChecklistCheck {
 	}
 }
 
+// defaultTCPServiceChecklist runs one orchestration or CLI step.
+// Signature: defaultTCPServiceChecklist() []TCPServiceChecklistCheck.
+// Why: mail and other non-HTTP edge services need protocol-level startup checks
+// so Ananke does not declare success while SMTP/IMAP ports are broken.
+func defaultTCPServiceChecklist() []TCPServiceChecklistCheck {
+	return []TCPServiceChecklistCheck{
+		{
+			Name:           "mail-smtp",
+			Host:           "mail.bstein.dev",
+			Port:           25,
+			Send:           "QUIT\r\n",
+			ExpectContains: "ESMTP",
+			TimeoutSeconds: 8,
+			Namespace:      "mailu-mailserver",
+			Service:        "mailu-front",
+		},
+		{
+			Name:           "mail-smtps",
+			Host:           "mail.bstein.dev",
+			Port:           465,
+			TLS:            true,
+			Send:           "QUIT\r\n",
+			ExpectContains: "ready",
+			TimeoutSeconds: 8,
+			Namespace:      "mailu-mailserver",
+			Service:        "mailu-front",
+		},
+		{
+			Name:           "mail-submission",
+			Host:           "mail.bstein.dev",
+			Port:           587,
+			Send:           "QUIT\r\n",
+			ExpectContains: "ready",
+			TimeoutSeconds: 8,
+			Namespace:      "mailu-mailserver",
+			Service:        "mailu-front",
+		},
+		{
+			Name:           "mail-imaps",
+			Host:           "mail.bstein.dev",
+			Port:           993,
+			TLS:            true,
+			Send:           "a001 LOGOUT\r\n",
+			ExpectContains: "OK",
+			TimeoutSeconds: 8,
+			Namespace:      "mailu-mailserver",
+			Service:        "mailu-front",
+		},
+	}
+}
+
 // defaultCriticalServiceEndpoints runs one orchestration or CLI step.
 // Signature: defaultCriticalServiceEndpoints() []string.
 // Why: service edge checks are insufficient for protected stacks; endpoint
@@ -263,6 +314,11 @@ func defaultCriticalServiceEndpoints() []string {
 		"logging/oauth2-proxy-logs",
 		"logging/opensearch-dashboards",
 		"logging/opensearch-master",
+		"mailu-mailserver/mailu-admin",
+		"mailu-mailserver/mailu-dovecot",
+		"mailu-mailserver/mailu-front",
+		"mailu-mailserver/mailu-postfix",
+		"mailu-mailserver/mailu-rspamd",
 	}
 }
 
@@ -301,6 +357,41 @@ func mergeServiceChecklistDefaults(existing, defaults []ServiceChecklistCheck) [
 	return out
 }
 
+// mergeTCPServiceChecklistDefaults runs one orchestration or CLI step.
+// Signature: mergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck.
+// Why: host configs should inherit new mail/TCP health checks while preserving
+// site-specific protocol probes.
+func mergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck {
+	if len(existing) == 0 {
+		out := make([]TCPServiceChecklistCheck, 0, len(defaults))
+		out = append(out, defaults...)
+		return out
+	}
+
+	defaultByName := map[string]struct{}{}
+	for _, check := range defaults {
+		name := strings.TrimSpace(check.Name)
+		if name == "" {
+			continue
+		}
+		defaultByName[name] = struct{}{}
+	}
+
+	out := make([]TCPServiceChecklistCheck, 0, len(defaults)+len(existing))
+	out = append(out, defaults...)
+	for _, check := range existing {
+		name := strings.TrimSpace(check.Name)
+		if name == "" {
+			continue
+		}
+		if _, exists := defaultByName[name]; exists {
+			continue
+		}
+		out = append(out, check)
+	}
+	return out
+}
+
 // mergeStringDefaults runs one orchestration or CLI step.
 // Signature: mergeStringDefaults(existing, defaults []string) []string.
 // Why: keeps baseline startup guards applied while preserving site-specific

internal/config/startup_service_catalog_closeout_test.go

@@ -0,0 +1,25 @@
+package config
+
+import "testing"
+
+// TestMergeStartupCatalogDefaultsSkipsBlankNames runs one orchestration or CLI step.
+// Signature: TestMergeStartupCatalogDefaultsSkipsBlankNames(t *testing.T).
+// Why: blank generated or hand-written checklist names should not poison
+// default-catalog merging or create duplicate unnamed runtime checks.
+func TestMergeStartupCatalogDefaultsSkipsBlankNames(t *testing.T) {
+	httpMerged := mergeServiceChecklistDefaults(
+		[]ServiceChecklistCheck{{Name: " "}, {Name: "custom"}},
+		[]ServiceChecklistCheck{{Name: " "}, {Name: "baseline"}},
+	)
+	if len(httpMerged) != 3 || httpMerged[0].Name != " " || httpMerged[1].Name != "baseline" || httpMerged[2].Name != "custom" {
+		t.Fatalf("unexpected HTTP checklist merge: %#v", httpMerged)
+	}
+
+	tcpMerged := mergeTCPServiceChecklistDefaults(
+		[]TCPServiceChecklistCheck{{Name: " "}, {Name: "smtp-custom"}},
+		[]TCPServiceChecklistCheck{{Name: " "}, {Name: "smtp-baseline"}},
+	)
+	if len(tcpMerged) != 3 || tcpMerged[0].Name != " " || tcpMerged[1].Name != "smtp-baseline" || tcpMerged[2].Name != "smtp-custom" {
+		t.Fatalf("unexpected TCP checklist merge: %#v", tcpMerged)
+	}
+}

internal/config/testing_hooks.go

@@ -18,6 +18,15 @@ func TestHookDefaultCriticalServiceEndpoints() []string {
 	return out
 }
 
+// TestHookDefaultTCPServiceChecklist runs one orchestration or CLI step.
+// Signature: TestHookDefaultTCPServiceChecklist() []TCPServiceChecklistCheck.
+// Why: exposes default TCP/mail service checks to top-level tests.
+func TestHookDefaultTCPServiceChecklist() []TCPServiceChecklistCheck {
+	out := make([]TCPServiceChecklistCheck, 0, len(defaultTCPServiceChecklist()))
+	out = append(out, defaultTCPServiceChecklist()...)
+	return out
+}
+
 // TestHookMergeServiceChecklistDefaults runs one orchestration or CLI step.
 // Signature: TestHookMergeServiceChecklistDefaults(existing, defaults []ServiceChecklistCheck) []ServiceChecklistCheck.
 // Why: exposes checklist merge helper to top-level tests.
@@ -31,3 +40,10 @@ func TestHookMergeServiceChecklistDefaults(existing, defaults []ServiceChecklist
 func TestHookMergeStringDefaults(existing, defaults []string) []string {
 	return mergeStringDefaults(existing, defaults)
 }
+
+// TestHookMergeTCPServiceChecklistDefaults runs one orchestration or CLI step.
+// Signature: TestHookMergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck.
+// Why: exposes TCP checklist merge helper to top-level tests.
+func TestHookMergeTCPServiceChecklistDefaults(existing, defaults []TCPServiceChecklistCheck) []TCPServiceChecklistCheck {
+	return mergeTCPServiceChecklistDefaults(existing, defaults)
+}

internal/config/types.go

@@ -23,69 +23,94 @@ type Config struct {
 	UPS                 UPS               `yaml:"ups"`
 	Coordination        Coordination      `yaml:"coordination"`
 	Metrics             Metrics           `yaml:"metrics"`
+	GitOps              GitOps            `yaml:"gitops"`
 	State               State             `yaml:"state"`
 }
 
 type Startup struct {
-	APIWaitSeconds                  int                          `yaml:"api_wait_seconds"`
+	APIWaitSeconds                        int                          `yaml:"api_wait_seconds"`
-	APIPollSeconds                  int                          `yaml:"api_poll_seconds"`
+	APIPollSeconds                        int                          `yaml:"api_poll_seconds"`
-	ShutdownCooldownSeconds         int                          `yaml:"shutdown_cooldown_seconds"`
+	ShutdownCooldownSeconds               int                          `yaml:"shutdown_cooldown_seconds"`
-	MinimumBatteryPercent           float64                      `yaml:"minimum_battery_percent"`
+	MinimumBatteryPercent                 float64                      `yaml:"minimum_battery_percent"`
-	RequireNodeInventoryReach       bool                         `yaml:"require_node_inventory_reachability"`
+	RequireNodeInventoryReach             bool                         `yaml:"require_node_inventory_reachability"`
-	NodeInventoryReachWaitSeconds   int                          `yaml:"node_inventory_reachability_wait_seconds"`
+	NodeInventoryReachWaitSeconds         int                          `yaml:"node_inventory_reachability_wait_seconds"`
-	NodeInventoryReachPollSeconds   int                          `yaml:"node_inventory_reachability_poll_seconds"`
+	NodeInventoryReachPollSeconds         int                          `yaml:"node_inventory_reachability_poll_seconds"`
-	RequiredNodeLabels              map[string]map[string]string `yaml:"required_node_labels"`
+	NodeInventoryReachRequiredNodes       []string                     `yaml:"node_inventory_reachability_required_nodes"`
-	RequireTimeSync                 bool                         `yaml:"require_time_sync"`
+	LonghornCryptsetupExemptNodes         []string                     `yaml:"longhorn_cryptsetup_exempt_nodes"`
-	TimeSyncWaitSeconds             int                          `yaml:"time_sync_wait_seconds"`
+	RequiredNodeLabels                    map[string]map[string]string `yaml:"required_node_labels"`
-	TimeSyncPollSeconds             int                          `yaml:"time_sync_poll_seconds"`
+	RequiredNodeLabelsMode                string                       `yaml:"required_node_labels_mode"`
-	TimeSyncMode                    string                       `yaml:"time_sync_mode"`
+	RequireTimeSync                       bool                         `yaml:"require_time_sync"`
-	TimeSyncQuorum                  int                          `yaml:"time_sync_quorum"`
+	TimeSyncWaitSeconds                   int                          `yaml:"time_sync_wait_seconds"`
-	ReconcileAccessOnBoot           bool                         `yaml:"reconcile_access_on_boot"`
+	TimeSyncPollSeconds                   int                          `yaml:"time_sync_poll_seconds"`
-	AutoEtcdRestoreOnAPIFailure     bool                         `yaml:"auto_etcd_restore_on_api_failure"`
+	TimeSyncMode                          string                       `yaml:"time_sync_mode"`
-	EtcdRestoreControlPlane         string                       `yaml:"etcd_restore_control_plane"`
+	TimeSyncQuorum                        int                          `yaml:"time_sync_quorum"`
-	RequireStorageReady             bool                         `yaml:"require_storage_ready"`
+	ReconcileAccessOnBoot                 bool                         `yaml:"reconcile_access_on_boot"`
-	StorageReadyWaitSeconds         int                          `yaml:"storage_ready_wait_seconds"`
+	AutoEtcdRestoreOnAPIFailure           bool                         `yaml:"auto_etcd_restore_on_api_failure"`
-	StorageReadyPollSeconds         int                          `yaml:"storage_ready_poll_seconds"`
+	EtcdRestoreControlPlane               string                       `yaml:"etcd_restore_control_plane"`
-	StorageMinReadyNodes            int                          `yaml:"storage_min_ready_nodes"`
+	RequireStorageReady                   bool                         `yaml:"require_storage_ready"`
-	StorageCriticalPVCs             []string                     `yaml:"storage_critical_pvcs"`
+	StorageReadyWaitSeconds               int                          `yaml:"storage_ready_wait_seconds"`
-	RequirePostStartProbes          bool                         `yaml:"require_post_start_probes"`
+	StorageReadyPollSeconds               int                          `yaml:"storage_ready_poll_seconds"`
-	PostStartProbeWaitSeconds       int                          `yaml:"post_start_probe_wait_seconds"`
+	StorageMinReadyNodes                  int                          `yaml:"storage_min_ready_nodes"`
-	PostStartProbePollSeconds       int                          `yaml:"post_start_probe_poll_seconds"`
+	StorageCriticalPVCs                   []string                     `yaml:"storage_critical_pvcs"`
-	PostStartProbes                 []string                     `yaml:"post_start_probes"`
+	RequirePostStartProbes                bool                         `yaml:"require_post_start_probes"`
-	RequireServiceChecklist         bool                         `yaml:"require_service_checklist"`
+	PostStartProbeWaitSeconds             int                          `yaml:"post_start_probe_wait_seconds"`
-	ServiceChecklistWaitSeconds     int                          `yaml:"service_checklist_wait_seconds"`
+	PostStartProbePollSeconds             int                          `yaml:"post_start_probe_poll_seconds"`
-	ServiceChecklistPollSeconds     int                          `yaml:"service_checklist_poll_seconds"`
+	PostStartProbes                       []string                     `yaml:"post_start_probes"`
-	ServiceChecklistStabilitySec    int                          `yaml:"service_checklist_stability_seconds"`
+	RequireServiceChecklist               bool                         `yaml:"require_service_checklist"`
-	ServiceChecklistAuth            ServiceChecklistAuthSettings `yaml:"service_checklist_auth"`
+	ServiceChecklistWaitSeconds           int                          `yaml:"service_checklist_wait_seconds"`
-	ServiceChecklist                []ServiceChecklistCheck      `yaml:"service_checklist"`
+	ServiceChecklistPollSeconds           int                          `yaml:"service_checklist_poll_seconds"`
-	RequireCriticalServiceEndpoints bool                         `yaml:"require_critical_service_endpoints"`
+	ServiceChecklistStabilitySec          int                          `yaml:"service_checklist_stability_seconds"`
-	CriticalServiceEndpointWaitSec  int                          `yaml:"critical_service_endpoint_wait_seconds"`
+	ServiceChecklistAuth                  ServiceChecklistAuthSettings `yaml:"service_checklist_auth"`
-	CriticalServiceEndpointPollSec  int                          `yaml:"critical_service_endpoint_poll_seconds"`
+	ServiceChecklistExplicitOnly          bool                         `yaml:"service_checklist_explicit_only"`
-	CriticalServiceEndpoints        []string                     `yaml:"critical_service_endpoints"`
+	ServiceChecklist                      []ServiceChecklistCheck      `yaml:"service_checklist"`
-	RequireIngressChecklist         bool                         `yaml:"require_ingress_checklist"`
+	TCPServiceChecklistExplicitOnly       bool                         `yaml:"tcp_service_checklist_explicit_only"`
-	IngressChecklistWaitSeconds     int                          `yaml:"ingress_checklist_wait_seconds"`
+	TCPServiceChecklist                   []TCPServiceChecklistCheck   `yaml:"tcp_service_checklist"`
-	IngressChecklistPollSeconds     int                          `yaml:"ingress_checklist_poll_seconds"`
+	RequireCriticalServiceEndpoints       bool                         `yaml:"require_critical_service_endpoints"`
-	IngressChecklistAccepted        []int                        `yaml:"ingress_checklist_accepted_statuses"`
+	CriticalServiceEndpointWaitSec        int                          `yaml:"critical_service_endpoint_wait_seconds"`
-	IngressChecklistIgnoreHosts     []string                     `yaml:"ingress_checklist_ignore_hosts"`
+	CriticalServiceEndpointPollSec        int                          `yaml:"critical_service_endpoint_poll_seconds"`
-	IngressChecklistInsecureSkip    bool                         `yaml:"ingress_checklist_insecure_skip_tls"`
+	CriticalServiceEndpoints              []string                     `yaml:"critical_service_endpoints"`
-	RequireNodeSSHAuth              bool                         `yaml:"require_node_ssh_auth"`
+	CriticalServiceStartupProbeRepair     bool                         `yaml:"critical_service_startup_probe_repair"`
-	NodeSSHAuthWaitSeconds          int                          `yaml:"node_ssh_auth_wait_seconds"`
+	CriticalServiceStartupProbeThreshold  int                          `yaml:"critical_service_startup_probe_failure_threshold"`
-	NodeSSHAuthPollSeconds          int                          `yaml:"node_ssh_auth_poll_seconds"`
+	RequireIngressChecklist               bool                         `yaml:"require_ingress_checklist"`
-	RequireFluxHealth               bool                         `yaml:"require_flux_health"`
+	IngressChecklistWaitSeconds           int                          `yaml:"ingress_checklist_wait_seconds"`
-	FluxHealthWaitSeconds           int                          `yaml:"flux_health_wait_seconds"`
+	IngressChecklistPollSeconds           int                          `yaml:"ingress_checklist_poll_seconds"`
-	FluxHealthPollSeconds           int                          `yaml:"flux_health_poll_seconds"`
+	IngressChecklistAccepted              []int                        `yaml:"ingress_checklist_accepted_statuses"`
-	IgnoreFluxKustomizations        []string                     `yaml:"ignore_flux_kustomizations"`
+	IngressChecklistIgnoreHosts           []string                     `yaml:"ingress_checklist_ignore_hosts"`
-	RequireWorkloadConvergence      bool                         `yaml:"require_workload_convergence"`
+	IngressChecklistInsecureSkip          bool                         `yaml:"ingress_checklist_insecure_skip_tls"`
-	WorkloadConvergenceWaitSeconds  int                          `yaml:"workload_convergence_wait_seconds"`
+	RequireNodeSSHAuth                    bool                         `yaml:"require_node_ssh_auth"`
-	WorkloadConvergencePollSeconds  int                          `yaml:"workload_convergence_poll_seconds"`
+	NodeSSHAuthWaitSeconds                int                          `yaml:"node_ssh_auth_wait_seconds"`
-	IgnoreWorkloadNamespaces        []string                     `yaml:"ignore_workload_namespaces"`
+	NodeSSHAuthPollSeconds                int                          `yaml:"node_ssh_auth_poll_seconds"`
-	IgnoreWorkloads                 []string                     `yaml:"ignore_workloads"`
+	NodeSSHAuthRequiredNodes              []string                     `yaml:"node_ssh_auth_required_nodes"`
-	IgnoreUnavailableNodes          []string                     `yaml:"ignore_unavailable_nodes"`
+	RequireFluxHealth                     bool                         `yaml:"require_flux_health"`
-	AutoRecycleStuckPods            bool                         `yaml:"auto_recycle_stuck_pods"`
+	FluxHealthWaitSeconds                 int                          `yaml:"flux_health_wait_seconds"`
-	StuckPodGraceSeconds            int                          `yaml:"stuck_pod_grace_seconds"`
+	FluxHealthPollSeconds                 int                          `yaml:"flux_health_poll_seconds"`
-	VaultUnsealKeyFile              string                       `yaml:"vault_unseal_key_file"`
+	FluxHealthRequiredKustomizations      []string                     `yaml:"flux_health_required_kustomizations"`
-	VaultUnsealBreakglassCommand    string                       `yaml:"vault_unseal_breakglass_command"`
+	IgnoreFluxKustomizations              []string                     `yaml:"ignore_flux_kustomizations"`
-	VaultUnsealBreakglassTimeout    int                          `yaml:"vault_unseal_breakglass_timeout_seconds"`
+	RequireWorkloadConvergence            bool                         `yaml:"require_workload_convergence"`
+	WorkloadConvergenceWaitSeconds        int                          `yaml:"workload_convergence_wait_seconds"`
+	WorkloadConvergencePollSeconds        int                          `yaml:"workload_convergence_poll_seconds"`
+	WorkloadConvergenceRequiredNamespaces []string                     `yaml:"workload_convergence_required_namespaces"`
+	IgnoreWorkloadNamespaces              []string                     `yaml:"ignore_workload_namespaces"`
+	IgnoreWorkloads                       []string                     `yaml:"ignore_workloads"`
+	IgnoreUnavailableNodes                []string                     `yaml:"ignore_unavailable_nodes"`
+	AutoRecycleStuckPods                  bool                         `yaml:"auto_recycle_stuck_pods"`
+	AutoQuarantineSchedulingStorms        bool                         `yaml:"auto_quarantine_scheduling_storms"`
+	SchedulingStormEventThreshold         int                          `yaml:"scheduling_storm_event_threshold"`
+	SchedulingStormWindowSeconds          int                          `yaml:"scheduling_storm_window_seconds"`
+	StuckPodGraceSeconds                  int                          `yaml:"stuck_pod_grace_seconds"`
+	PostStartAutoHealSeconds              int                          `yaml:"post_start_auto_heal_seconds"`
+	RecoveryCordonMaxSeconds              int                          `yaml:"recovery_cordon_max_seconds"`
+	DeadNodeCleanupGraceSeconds           int                          `yaml:"dead_node_cleanup_grace_seconds"`
+	HostSudoSecretNamespace               string                       `yaml:"host_sudo_secret_namespace"`
+	HostSudoSecretNameTemplate            string                       `yaml:"host_sudo_secret_name_template"`
+	HostSudoSecretPasswordKey             string                       `yaml:"host_sudo_secret_password_key"`
+	HostRepairAllowReboot                 bool                         `yaml:"host_repair_allow_reboot"`
+	HostPrivilegedCommandTimeoutSec       int                          `yaml:"host_privileged_command_timeout_seconds"`
+	NodeRuntimeRestartWaitSeconds         int                          `yaml:"node_runtime_restart_wait_seconds"`
+	NodeRuntimeRebootWaitSeconds          int                          `yaml:"node_runtime_reboot_wait_seconds"`
+	VaultUnsealKeyFile                    string                       `yaml:"vault_unseal_key_file"`
+	VaultUnsealBreakglassCommand          string                       `yaml:"vault_unseal_breakglass_command"`
+	VaultUnsealBreakglassTimeout          int                          `yaml:"vault_unseal_breakglass_timeout_seconds"`
 }
 
 type ServiceChecklistCheck struct {
@@ -104,6 +129,19 @@ type ServiceChecklistCheck struct {
 	InsecureSkipTLS     bool   `yaml:"insecure_skip_tls"`
 }
 
+type TCPServiceChecklistCheck struct {
+	Name            string `yaml:"name"`
+	Host            string `yaml:"host"`
+	Port            int    `yaml:"port"`
+	TLS             bool   `yaml:"tls"`
+	Send            string `yaml:"send"`
+	ExpectContains  string `yaml:"expect_contains"`
+	TimeoutSeconds  int    `yaml:"timeout_seconds"`
+	InsecureSkipTLS bool   `yaml:"insecure_skip_tls"`
+	Namespace       string `yaml:"namespace"`
+	Service         string `yaml:"service"`
+}
+
 type ServiceChecklistAuthSettings struct {
 	Mode                   string `yaml:"mode"`
 	KeycloakBaseURL        string `yaml:"keycloak_base_url"`
@@ -136,6 +174,7 @@ type UPS struct {
 	Targets                 []UPSTarget `yaml:"targets"`
 	PollSeconds             int         `yaml:"poll_seconds"`
 	RuntimeSafetyFactor     float64     `yaml:"runtime_safety_factor"`
+	OnBatteryGraceSeconds   int         `yaml:"on_battery_grace_seconds"`
 	DebounceCount           int         `yaml:"debounce_count"`
 	TelemetryTimeoutSeconds int         `yaml:"telemetry_timeout_seconds"`
 }
@@ -163,6 +202,11 @@ type Metrics struct {
 	Path     string `yaml:"path"`
 }
 
+type GitOps struct {
+	Enabled     bool `yaml:"enabled"`
+	PollSeconds int  `yaml:"poll_seconds"`
+}
+
 type State struct {
 	Dir            string `yaml:"dir"`
 	ReportsDir     string `yaml:"reports_dir"`

internal/config/validate.go

@@ -61,6 +61,16 @@ func (c Config) Validate() error {
 	if c.Startup.NodeInventoryReachPollSeconds <= 0 {
 		return fmt.Errorf("config.startup.node_inventory_reachability_poll_seconds must be > 0")
 	}
+	for _, node := range c.Startup.NodeInventoryReachRequiredNodes {
+		if strings.TrimSpace(node) == "" {
+			return fmt.Errorf("config.startup.node_inventory_reachability_required_nodes entries must not be empty")
+		}
+	}
+	for _, node := range c.Startup.LonghornCryptsetupExemptNodes {
+		if strings.TrimSpace(node) == "" {
+			return fmt.Errorf("config.startup.longhorn_cryptsetup_exempt_nodes entries must not be empty")
+		}
+	}
 	for node, labels := range c.Startup.RequiredNodeLabels {
 		if strings.TrimSpace(node) == "" {
 			return fmt.Errorf("config.startup.required_node_labels keys must not be empty")
@@ -77,6 +87,9 @@ func (c Config) Validate() error {
 			}
 		}
 	}
+	if c.Startup.RequiredNodeLabelsMode != "enforce" && c.Startup.RequiredNodeLabelsMode != "validate" {
+		return fmt.Errorf("config.startup.required_node_labels_mode must be enforce or validate")
+	}
 	if c.Startup.TimeSyncWaitSeconds <= 0 {
 		return fmt.Errorf("config.startup.time_sync_wait_seconds must be > 0")
 	}
@@ -193,6 +206,28 @@ func (c Config) Validate() error {
 			}
 		}
 	}
+	for i, check := range c.Startup.TCPServiceChecklist {
+		if strings.TrimSpace(check.Name) == "" {
+			return fmt.Errorf("config.startup.tcp_service_checklist[%d].name must not be empty", i)
+		}
+		if strings.TrimSpace(check.Host) == "" {
+			return fmt.Errorf("config.startup.tcp_service_checklist[%d].host must not be empty", i)
+		}
+		if strings.ContainsAny(check.Host, " \t\r\n/") {
+			return fmt.Errorf("config.startup.tcp_service_checklist[%d].host is invalid: %q", i, check.Host)
+		}
+		if check.Port <= 0 || check.Port > 65535 {
+			return fmt.Errorf("config.startup.tcp_service_checklist[%d].port must be in range 1-65535", i)
+		}
+		if check.TimeoutSeconds <= 0 {
+			return fmt.Errorf("config.startup.tcp_service_checklist[%d].timeout_seconds must be > 0", i)
+		}
+		if strings.TrimSpace(check.Namespace) != "" || strings.TrimSpace(check.Service) != "" {
+			if strings.TrimSpace(check.Namespace) == "" || strings.TrimSpace(check.Service) == "" {
+				return fmt.Errorf("config.startup.tcp_service_checklist[%d].namespace and service must be set together", i)
+			}
+		}
+	}
 	if c.Startup.CriticalServiceEndpointWaitSec <= 0 {
 		return fmt.Errorf("config.startup.critical_service_endpoint_wait_seconds must be > 0")
 	}
@@ -211,6 +246,9 @@ func (c Config) Validate() error {
 			return fmt.Errorf("config.startup.critical_service_endpoints entries must be namespace/service, got %q", entry)
 		}
 	}
+	if c.Startup.CriticalServiceStartupProbeRepair && c.Startup.CriticalServiceStartupProbeThreshold <= 0 {
+		return fmt.Errorf("config.startup.critical_service_startup_probe_failure_threshold must be > 0 when critical_service_startup_probe_repair is true")
+	}
 	if c.Startup.IngressChecklistWaitSeconds <= 0 {
 		return fmt.Errorf("config.startup.ingress_checklist_wait_seconds must be > 0")
 	}
@@ -233,21 +271,69 @@ func (c Config) Validate() error {
 	if c.Startup.NodeSSHAuthPollSeconds <= 0 {
 		return fmt.Errorf("config.startup.node_ssh_auth_poll_seconds must be > 0")
 	}
+	for _, node := range c.Startup.NodeSSHAuthRequiredNodes {
+		if strings.TrimSpace(node) == "" {
+			return fmt.Errorf("config.startup.node_ssh_auth_required_nodes entries must not be empty")
+		}
+	}
 	if c.Startup.FluxHealthWaitSeconds <= 0 {
 		return fmt.Errorf("config.startup.flux_health_wait_seconds must be > 0")
 	}
 	if c.Startup.FluxHealthPollSeconds <= 0 {
 		return fmt.Errorf("config.startup.flux_health_poll_seconds must be > 0")
 	}
+	for _, item := range c.Startup.FluxHealthRequiredKustomizations {
+		item = strings.TrimSpace(item)
+		if item == "" {
+			return fmt.Errorf("config.startup.flux_health_required_kustomizations entries must not be empty")
+		}
+		if strings.Count(item, "/") != 1 {
+			return fmt.Errorf("config.startup.flux_health_required_kustomizations entries must be namespace/name, got %q", item)
+		}
+	}
 	if c.Startup.WorkloadConvergenceWaitSeconds <= 0 {
 		return fmt.Errorf("config.startup.workload_convergence_wait_seconds must be > 0")
 	}
 	if c.Startup.WorkloadConvergencePollSeconds <= 0 {
 		return fmt.Errorf("config.startup.workload_convergence_poll_seconds must be > 0")
 	}
+	for _, ns := range c.Startup.WorkloadConvergenceRequiredNamespaces {
+		if strings.TrimSpace(ns) == "" {
+			return fmt.Errorf("config.startup.workload_convergence_required_namespaces entries must not be empty")
+		}
+	}
 	if c.Startup.StuckPodGraceSeconds <= 0 {
 		return fmt.Errorf("config.startup.stuck_pod_grace_seconds must be > 0")
 	}
+	if c.Startup.PostStartAutoHealSeconds <= 0 {
+		return fmt.Errorf("config.startup.post_start_auto_heal_seconds must be > 0")
+	}
+	if c.Startup.RecoveryCordonMaxSeconds <= 0 {
+		return fmt.Errorf("config.startup.recovery_cordon_max_seconds must be > 0")
+	}
+	if c.Startup.DeadNodeCleanupGraceSeconds <= 0 {
+		return fmt.Errorf("config.startup.dead_node_cleanup_grace_seconds must be > 0")
+	}
+	if strings.TrimSpace(c.Startup.HostSudoSecretNamespace) != "" || strings.TrimSpace(c.Startup.HostSudoSecretNameTemplate) != "" {
+		if strings.TrimSpace(c.Startup.HostSudoSecretNamespace) == "" {
+			return fmt.Errorf("config.startup.host_sudo_secret_namespace must not be empty when host sudo secret lookup is configured")
+		}
+		if strings.TrimSpace(c.Startup.HostSudoSecretNameTemplate) == "" {
+			return fmt.Errorf("config.startup.host_sudo_secret_name_template must not be empty when host sudo secret lookup is configured")
+		}
+		if strings.TrimSpace(c.Startup.HostSudoSecretPasswordKey) == "" {
+			return fmt.Errorf("config.startup.host_sudo_secret_password_key must not be empty when host sudo secret lookup is configured")
+		}
+	}
+	if c.Startup.HostPrivilegedCommandTimeoutSec <= 0 {
+		return fmt.Errorf("config.startup.host_privileged_command_timeout_seconds must be > 0")
+	}
+	if c.Startup.NodeRuntimeRestartWaitSeconds <= 0 {
+		return fmt.Errorf("config.startup.node_runtime_restart_wait_seconds must be > 0")
+	}
+	if c.Startup.NodeRuntimeRebootWaitSeconds <= 0 {
+		return fmt.Errorf("config.startup.node_runtime_reboot_wait_seconds must be > 0")
+	}
 	for _, probe := range c.Startup.PostStartProbes {
 		if strings.TrimSpace(probe) == "" {
 			return fmt.Errorf("config.startup.post_start_probes entries must not be empty")
@@ -277,6 +363,16 @@ func (c Config) Validate() error {
 			return fmt.Errorf("config.startup.ignore_workload_namespaces entries must not be empty")
 		}
 	}
+	for _, item := range c.Startup.FluxHealthRequiredKustomizations {
+		if containsTrimmed(c.Startup.IgnoreFluxKustomizations, item) {
+			return fmt.Errorf("config.startup.flux_health_required_kustomizations must not overlap ignore_flux_kustomizations (%q)", strings.TrimSpace(item))
+		}
+	}
+	for _, ns := range c.Startup.WorkloadConvergenceRequiredNamespaces {
+		if containsTrimmed(c.Startup.IgnoreWorkloadNamespaces, ns) {
+			return fmt.Errorf("config.startup.workload_convergence_required_namespaces must not overlap ignore_workload_namespaces (%q)", strings.TrimSpace(ns))
+		}
+	}
 	for _, node := range c.Startup.IgnoreUnavailableNodes {
 		if strings.TrimSpace(node) == "" {
 			return fmt.Errorf("config.startup.ignore_unavailable_nodes entries must not be empty")
@@ -292,6 +388,9 @@ func (c Config) Validate() error {
 		if c.UPS.Provider == "" {
 			return fmt.Errorf("config.ups.provider must not be empty when ups is enabled")
 		}
+		if c.UPS.OnBatteryGraceSeconds < 0 {
+			return fmt.Errorf("config.ups.on_battery_grace_seconds must be >= 0")
+		}
 		if c.UPS.Target == "" && len(c.UPS.Targets) == 0 {
 			return fmt.Errorf("config.ups.target or config.ups.targets must be set when ups is enabled")
 		}
@@ -306,6 +405,14 @@ func (c Config) Validate() error {
 			return fmt.Errorf("config.coordination.forward_shutdown_config must not be empty when forward_shutdown_host is set")
 		}
 	}
+	if c.Startup.AutoQuarantineSchedulingStorms {
+		if c.Startup.SchedulingStormEventThreshold <= 0 {
+			return fmt.Errorf("config.startup.scheduling_storm_event_threshold must be > 0 when auto_quarantine_scheduling_storms is enabled")
+		}
+		if c.Startup.SchedulingStormWindowSeconds <= 0 {
+			return fmt.Errorf("config.startup.scheduling_storm_window_seconds must be > 0 when auto_quarantine_scheduling_storms is enabled")
+		}
+	}
 	for _, peer := range c.Coordination.PeerHosts {
 		if strings.TrimSpace(peer) == "" {
 			return fmt.Errorf("config.coordination.peer_hosts entries must not be empty")
@@ -328,3 +435,20 @@ func (c Config) Validate() error {
 	}
 	return nil
 }
+
+// containsTrimmed runs one orchestration or CLI step.
+// Signature: containsTrimmed(entries []string, needle string) bool.
+// Why: startup config now supports both required and ignored recovery scopes, so
+// validation needs a single normalized overlap check for those lists.
+func containsTrimmed(entries []string, needle string) bool {
+	needle = strings.TrimSpace(needle)
+	if needle == "" {
+		return false
+	}
+	for _, entry := range entries {
+		if strings.TrimSpace(entry) == needle {
+			return true
+		}
+	}
+	return false
+}

internal/config/validate_matrix_test.go

@@ -30,6 +30,9 @@ func TestValidateRejectsInvalidFieldsMatrix(t *testing.T) {
 		{"bad_api_poll", func(c *Config) { c.Startup.APIPollSeconds = 0 }},
 		{"bad_min_battery_percent", func(c *Config) { c.Startup.MinimumBatteryPercent = 101 }},
 		{"bad_node_inventory_poll", func(c *Config) { c.Startup.NodeInventoryReachPollSeconds = 0 }},
+		{"bad_empty_node_inventory_required_node", func(c *Config) { c.Startup.NodeInventoryReachRequiredNodes = []string{"titan-0a", ""} }},
+		{"bad_empty_longhorn_cryptsetup_exempt_node", func(c *Config) { c.Startup.LonghornCryptsetupExemptNodes = []string{"titan-23", ""} }},
+		{"bad_required_node_labels_mode", func(c *Config) { c.Startup.RequiredNodeLabelsMode = "repair" }},
 		{"bad_time_sync_wait", func(c *Config) { c.Startup.TimeSyncWaitSeconds = 0 }},
 		{"bad_time_sync_poll", func(c *Config) { c.Startup.TimeSyncPollSeconds = 0 }},
 		{"bad_time_sync_quorum", func(c *Config) { c.Startup.TimeSyncMode = "quorum"; c.Startup.TimeSyncQuorum = 0 }},
@@ -50,6 +53,21 @@ func TestValidateRejectsInvalidFieldsMatrix(t *testing.T) {
 		{"bad_service_checklist_http_code", func(c *Config) {
 			c.Startup.ServiceChecklist = []ServiceChecklistCheck{{Name: "x", URL: "https://ok", TimeoutSeconds: 5, AcceptedStatuses: []int{999}}}
 		}},
+		{"bad_tcp_service_checklist_name", func(c *Config) {
+			c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Host: "mail.bstein.dev", Port: 25, TimeoutSeconds: 5}}
+		}},
+		{"bad_tcp_service_checklist_host", func(c *Config) {
+			c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail/bad", Port: 25, TimeoutSeconds: 5}}
+		}},
+		{"bad_tcp_service_checklist_port", func(c *Config) {
+			c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail.bstein.dev", Port: 70000, TimeoutSeconds: 5}}
+		}},
+		{"bad_tcp_service_checklist_timeout", func(c *Config) {
+			c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail.bstein.dev", Port: 25}}
+		}},
+		{"bad_tcp_service_checklist_partial_backend", func(c *Config) {
+			c.Startup.TCPServiceChecklist = []TCPServiceChecklistCheck{{Name: "mail", Host: "mail.bstein.dev", Port: 25, TimeoutSeconds: 5, Namespace: "mailu-mailserver"}}
+		}},
 		{"bad_critical_endpoint_wait", func(c *Config) { c.Startup.CriticalServiceEndpointWaitSec = 0 }},
 		{"bad_critical_endpoint_poll", func(c *Config) { c.Startup.CriticalServiceEndpointPollSec = 0 }},
 		{"bad_empty_critical_endpoints_when_required", func(c *Config) {
@@ -68,19 +86,42 @@ func TestValidateRejectsInvalidFieldsMatrix(t *testing.T) {
 		{"bad_ingress_ignore_entry", func(c *Config) { c.Startup.IngressChecklistIgnoreHosts = []string{"", "grafana.bstein.dev"} }},
 		{"bad_node_ssh_wait", func(c *Config) { c.Startup.NodeSSHAuthWaitSeconds = 0 }},
 		{"bad_node_ssh_poll", func(c *Config) { c.Startup.NodeSSHAuthPollSeconds = 0 }},
+		{"bad_empty_node_ssh_required_node", func(c *Config) { c.Startup.NodeSSHAuthRequiredNodes = []string{"titan-0a", ""} }},
 		{"bad_flux_wait", func(c *Config) { c.Startup.FluxHealthWaitSeconds = 0 }},
 		{"bad_flux_poll", func(c *Config) { c.Startup.FluxHealthPollSeconds = 0 }},
+		{"bad_empty_flux_required_kustomization", func(c *Config) { c.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/core", ""} }},
+		{"bad_malformed_flux_required_kustomization", func(c *Config) { c.Startup.FluxHealthRequiredKustomizations = []string{"flux-system-core"} }},
 		{"bad_workload_wait", func(c *Config) { c.Startup.WorkloadConvergenceWaitSeconds = 0 }},
 		{"bad_workload_poll", func(c *Config) { c.Startup.WorkloadConvergencePollSeconds = 0 }},
+		{"bad_empty_required_workload_namespace", func(c *Config) { c.Startup.WorkloadConvergenceRequiredNamespaces = []string{"monitoring", ""} }},
 		{"bad_stuck_pod_grace", func(c *Config) { c.Startup.StuckPodGraceSeconds = 0 }},
+		{"bad_post_start_auto_heal_seconds", func(c *Config) { c.Startup.PostStartAutoHealSeconds = 0 }},
+		{"bad_dead_node_cleanup_grace_seconds", func(c *Config) { c.Startup.DeadNodeCleanupGraceSeconds = 0 }},
 		{"bad_empty_post_start_probe_entry", func(c *Config) { c.Startup.PostStartProbes = []string{"https://ok", ""} }},
 		{"bad_empty_ignore_flux_entry", func(c *Config) { c.Startup.IgnoreFluxKustomizations = []string{"", "ns/name"} }},
 		{"bad_empty_ignore_workloads_entry", func(c *Config) { c.Startup.IgnoreWorkloads = []string{"", "ns/name"} }},
 		{"bad_empty_ignore_workload_namespaces_entry", func(c *Config) { c.Startup.IgnoreWorkloadNamespaces = []string{"", "vault"} }},
+		{"bad_overlap_flux_required_and_ignored", func(c *Config) {
+			c.Startup.FluxHealthRequiredKustomizations = []string{"flux-system/core"}
+			c.Startup.IgnoreFluxKustomizations = []string{"flux-system/core"}
+		}},
+		{"bad_overlap_workload_required_and_ignored", func(c *Config) {
+			c.Startup.WorkloadConvergenceRequiredNamespaces = []string{"monitoring"}
+			c.Startup.IgnoreWorkloadNamespaces = []string{"monitoring"}
+		}},
 		{"bad_empty_ignore_unavailable_nodes_entry", func(c *Config) { c.Startup.IgnoreUnavailableNodes = []string{"", "titan-22"} }},
 		{"bad_empty_vault_key_file", func(c *Config) { c.Startup.VaultUnsealKeyFile = "" }},
+		{"bad_scheduling_storm_threshold", func(c *Config) {
+			c.Startup.AutoQuarantineSchedulingStorms = true
+			c.Startup.SchedulingStormEventThreshold = 0
+		}},
+		{"bad_scheduling_storm_window", func(c *Config) {
+			c.Startup.AutoQuarantineSchedulingStorms = true
+			c.Startup.SchedulingStormWindowSeconds = 0
+		}},
 		{"bad_ssh_port_low", func(c *Config) { c.SSHPort = 0 }},
 		{"bad_ups_provider", func(c *Config) { c.UPS.Enabled = true; c.UPS.Provider = "" }},
+		{"bad_ups_on_battery_grace_negative", func(c *Config) { c.UPS.Enabled = true; c.UPS.OnBatteryGraceSeconds = -1 }},
 		{"bad_ups_target_empty", func(c *Config) { c.UPS.Enabled = true; c.UPS.Provider = "nut"; c.UPS.Target = ""; c.UPS.Targets = nil }},
 		{"bad_ups_targets_item_empty", func(c *Config) {
 			c.UPS.Enabled = true
@@ -121,12 +162,25 @@ func TestApplyDefaultsPopulatesZeroConfig(t *testing.T) {
 	if cfg.Startup.TimeSyncMode == "" || cfg.Startup.EtcdRestoreControlPlane == "" || cfg.Startup.VaultUnsealKeyFile == "" {
 		t.Fatalf("expected startup defaults to be set")
 	}
+	if cfg.Startup.RequiredNodeLabelsMode != "enforce" {
+		t.Fatalf("expected required node labels mode default enforce, got %q", cfg.Startup.RequiredNodeLabelsMode)
+	}
+	if cfg.Startup.PostStartAutoHealSeconds <= 0 || cfg.Startup.DeadNodeCleanupGraceSeconds <= 0 {
+		t.Fatalf("expected post-start auto-heal defaults to be set")
+	}
+	if cfg.Startup.NodeInventoryReachRequiredNodes == nil || cfg.Startup.NodeSSHAuthRequiredNodes == nil ||
+		cfg.Startup.FluxHealthRequiredKustomizations == nil || cfg.Startup.WorkloadConvergenceRequiredNamespaces == nil {
+		t.Fatalf("expected startup recovery scope slices to be initialized")
+	}
 	if cfg.Startup.CriticalServiceEndpointWaitSec <= 0 || cfg.Startup.CriticalServiceEndpointPollSec <= 0 {
 		t.Fatalf("expected critical service endpoint timing defaults to be set")
 	}
 	if len(cfg.Startup.CriticalServiceEndpoints) == 0 {
 		t.Fatalf("expected critical service endpoint defaults to be set")
 	}
+	if len(cfg.Startup.TCPServiceChecklist) == 0 {
+		t.Fatalf("expected TCP service checklist defaults to be set")
+	}
 	if cfg.Shutdown.SSHParallelism <= 0 || cfg.Shutdown.ScaleParallelism <= 0 || cfg.Shutdown.DrainParallelism <= 0 {
 		t.Fatalf("expected shutdown parallelism defaults to be set")
 	}

internal/metrics/exporter.go

@@ -27,6 +27,50 @@ type Sample struct {
 	UpdatedAt     time.Time
 }
 
+type GitOpsSnapshot struct {
+	UpdatedAt      time.Time
+	ScrapeSuccess  bool
+	Errors         map[string]string
+	FluxSources    []GitOpsFluxSource
+	Kustomizations []GitOpsKustomization
+	HelmReleases   []GitOpsHelmRelease
+}
+
+type GitOpsFluxSource struct {
+	Namespace string
+	Name      string
+	URL       string
+	Branch    string
+	Revision  string
+	Ready     bool
+	Reason    string
+	Suspended bool
+}
+
+type GitOpsKustomization struct {
+	Namespace       string
+	Name            string
+	Path            string
+	SourceNamespace string
+	SourceName      string
+	Revision        string
+	Ready           bool
+	Reason          string
+	Suspended       bool
+}
+
+type GitOpsHelmRelease struct {
+	Namespace  string
+	Name       string
+	Chart      string
+	Version    string
+	AppVersion string
+	Revision   string
+	Ready      bool
+	Reason     string
+	Suspended  bool
+}
+
 type Exporter struct {
 	mu                 sync.RWMutex
 	shutdownBudgetSec  int
@@ -34,6 +78,7 @@ type Exporter struct {
 	lastShutdownReason string
 	lastShutdownAt     time.Time
 	samples            map[string]Sample
+	gitops             GitOpsSnapshot
 }
 
 // New runs one orchestration or CLI step.
@@ -66,6 +111,22 @@ func (e *Exporter) UpdateSample(s Sample) {
 	e.samples[s.Name] = s
 }
 
+// UpdateGitOpsSnapshot records the most recent Flux object-state scrape.
+// Signature: (e *Exporter) UpdateGitOpsSnapshot(snapshot GitOpsSnapshot).
+// Why: Grafana needs object readiness and branch/revision state, while Flux's
+// controller metrics only expose controller health in this cluster.
+func (e *Exporter) UpdateGitOpsSnapshot(snapshot GitOpsSnapshot) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	if snapshot.UpdatedAt.IsZero() {
+		snapshot.UpdatedAt = time.Now().UTC()
+	}
+	if snapshot.Errors == nil {
+		snapshot.Errors = map[string]string{}
+	}
+	e.gitops = snapshot
+}
+
 // MarkShutdown runs one orchestration or CLI step.
 // Signature: (e *Exporter) MarkShutdown(reason string).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
@@ -164,11 +225,99 @@ func (e *Exporter) serveMetrics(w http.ResponseWriter, _ *http.Request) {
 		}
 		b.WriteString(fmt.Sprintf("ananke_ups_error%s %d\n", labels, boolNum(s.LastError != "")))
 	}
+	appendGitOpsMetrics(&b, e.gitops)
 	appendQualityGateMetrics(&b)
 
 	_, _ = w.Write([]byte(b.String()))
 }
 
+// appendGitOpsMetrics writes Flux object-state metrics collected by the
+// long-running daemon loop.
+// Signature: appendGitOpsMetrics(dst *strings.Builder, snapshot GitOpsSnapshot).
+// Why: this keeps the expensive Kubernetes API reads out of the HTTP scrape path
+// while still making current GitOps health cheap for Grafana to query.
+func appendGitOpsMetrics(dst *strings.Builder, snapshot GitOpsSnapshot) {
+	if dst.Len() > 0 {
+		dst.WriteString("\n")
+	}
+	dst.WriteString("# HELP ananke_gitops_last_scrape_timestamp_seconds Unix timestamp of the latest GitOps object-state scrape.\n")
+	dst.WriteString("# TYPE ananke_gitops_last_scrape_timestamp_seconds gauge\n")
+	if snapshot.UpdatedAt.IsZero() {
+		dst.WriteString("ananke_gitops_last_scrape_timestamp_seconds 0\n")
+	} else {
+		dst.WriteString(fmt.Sprintf("ananke_gitops_last_scrape_timestamp_seconds %d\n", snapshot.UpdatedAt.Unix()))
+	}
+	dst.WriteString("# HELP ananke_gitops_scrape_success Whether the latest GitOps object-state scrape completed without errors.\n")
+	dst.WriteString("# TYPE ananke_gitops_scrape_success gauge\n")
+	dst.WriteString(fmt.Sprintf("ananke_gitops_scrape_success %d\n", boolNum(snapshot.ScrapeSuccess)))
+	dst.WriteString("# HELP ananke_gitops_scrape_error Whether a GitOps resource family failed during the latest scrape.\n")
+	dst.WriteString("# TYPE ananke_gitops_scrape_error gauge\n")
+	resources := []string{"gitrepository", "kustomization", "helmrelease"}
+	for _, resource := range resources {
+		_, failed := snapshot.Errors[resource]
+		dst.WriteString(fmt.Sprintf("ananke_gitops_scrape_error{resource=%q} %d\n", resource, boolNum(failed)))
+	}
+
+	dst.WriteString("# HELP ananke_gitops_flux_source_info Current Flux GitRepository source metadata.\n")
+	dst.WriteString("# TYPE ananke_gitops_flux_source_info gauge\n")
+	dst.WriteString("# HELP ananke_gitops_flux_source_ready Whether a Flux GitRepository source is Ready.\n")
+	dst.WriteString("# TYPE ananke_gitops_flux_source_ready gauge\n")
+	dst.WriteString("# HELP ananke_gitops_flux_source_suspended Whether a Flux GitRepository source is suspended.\n")
+	dst.WriteString("# TYPE ananke_gitops_flux_source_suspended gauge\n")
+	sort.Slice(snapshot.FluxSources, func(i, j int) bool {
+		return snapshot.FluxSources[i].Namespace+"/"+snapshot.FluxSources[i].Name < snapshot.FluxSources[j].Namespace+"/"+snapshot.FluxSources[j].Name
+	})
+	for _, source := range snapshot.FluxSources {
+		infoLabels := fmt.Sprintf("{namespace=%q,name=%q,url=%q,branch=%q,revision=%q,ready=%q,reason=%q}",
+			safe(source.Namespace), safe(source.Name), safe(source.URL), safe(source.Branch),
+			safe(source.Revision), readyLabel(source.Ready), safe(defaultLabel(source.Reason, "unknown")))
+		baseLabels := fmt.Sprintf("{namespace=%q,name=%q}", safe(source.Namespace), safe(source.Name))
+		dst.WriteString(fmt.Sprintf("ananke_gitops_flux_source_info%s 1\n", infoLabels))
+		dst.WriteString(fmt.Sprintf("ananke_gitops_flux_source_ready%s %d\n", baseLabels, boolNum(source.Ready)))
+		dst.WriteString(fmt.Sprintf("ananke_gitops_flux_source_suspended%s %d\n", baseLabels, boolNum(source.Suspended)))
+	}
+
+	dst.WriteString("# HELP ananke_gitops_kustomization_info Current Flux Kustomization metadata.\n")
+	dst.WriteString("# TYPE ananke_gitops_kustomization_info gauge\n")
+	dst.WriteString("# HELP ananke_gitops_kustomization_ready Whether a Flux Kustomization is Ready.\n")
+	dst.WriteString("# TYPE ananke_gitops_kustomization_ready gauge\n")
+	dst.WriteString("# HELP ananke_gitops_kustomization_suspended Whether a Flux Kustomization is suspended.\n")
+	dst.WriteString("# TYPE ananke_gitops_kustomization_suspended gauge\n")
+	sort.Slice(snapshot.Kustomizations, func(i, j int) bool {
+		return snapshot.Kustomizations[i].Namespace+"/"+snapshot.Kustomizations[i].Name < snapshot.Kustomizations[j].Namespace+"/"+snapshot.Kustomizations[j].Name
+	})
+	for _, kustomization := range snapshot.Kustomizations {
+		infoLabels := fmt.Sprintf("{namespace=%q,name=%q,path=%q,source_namespace=%q,source_name=%q,revision=%q,ready=%q,reason=%q}",
+			safe(kustomization.Namespace), safe(kustomization.Name), safe(kustomization.Path),
+			safe(kustomization.SourceNamespace), safe(kustomization.SourceName), safe(kustomization.Revision),
+			readyLabel(kustomization.Ready), safe(defaultLabel(kustomization.Reason, "unknown")))
+		baseLabels := fmt.Sprintf("{namespace=%q,name=%q}", safe(kustomization.Namespace), safe(kustomization.Name))
+		dst.WriteString(fmt.Sprintf("ananke_gitops_kustomization_info%s 1\n", infoLabels))
+		dst.WriteString(fmt.Sprintf("ananke_gitops_kustomization_ready%s %d\n", baseLabels, boolNum(kustomization.Ready)))
+		dst.WriteString(fmt.Sprintf("ananke_gitops_kustomization_suspended%s %d\n", baseLabels, boolNum(kustomization.Suspended)))
+	}
+
+	dst.WriteString("# HELP ananke_gitops_helmrelease_info Current Flux HelmRelease metadata.\n")
+	dst.WriteString("# TYPE ananke_gitops_helmrelease_info gauge\n")
+	dst.WriteString("# HELP ananke_gitops_helmrelease_ready Whether a Flux HelmRelease is Ready.\n")
+	dst.WriteString("# TYPE ananke_gitops_helmrelease_ready gauge\n")
+	dst.WriteString("# HELP ananke_gitops_helmrelease_suspended Whether a Flux HelmRelease is suspended.\n")
+	dst.WriteString("# TYPE ananke_gitops_helmrelease_suspended gauge\n")
+	sort.Slice(snapshot.HelmReleases, func(i, j int) bool {
+		return snapshot.HelmReleases[i].Namespace+"/"+snapshot.HelmReleases[i].Name < snapshot.HelmReleases[j].Namespace+"/"+snapshot.HelmReleases[j].Name
+	})
+	for _, release := range snapshot.HelmReleases {
+		infoLabels := fmt.Sprintf("{namespace=%q,name=%q,chart=%q,version=%q,app_version=%q,revision=%q,ready=%q,reason=%q}",
+			safe(release.Namespace), safe(release.Name), safe(release.Chart), safe(release.Version),
+			safe(release.AppVersion), safe(release.Revision), readyLabel(release.Ready),
+			safe(defaultLabel(release.Reason, "unknown")))
+		baseLabels := fmt.Sprintf("{namespace=%q,name=%q}", safe(release.Namespace), safe(release.Name))
+		dst.WriteString(fmt.Sprintf("ananke_gitops_helmrelease_info%s 1\n", infoLabels))
+		dst.WriteString(fmt.Sprintf("ananke_gitops_helmrelease_ready%s %d\n", baseLabels, boolNum(release.Ready)))
+		dst.WriteString(fmt.Sprintf("ananke_gitops_helmrelease_suspended%s %d\n", baseLabels, boolNum(release.Suspended)))
+	}
+}
+
 // appendQualityGateMetrics runs one orchestration or CLI step.
 // Signature: appendQualityGateMetrics(dst *strings.Builder).
 // Why: quality-gate pass/fail telemetry should appear alongside UPS metrics so
@@ -228,3 +377,26 @@ func safe(in string) string {
 	out := strings.ReplaceAll(in, "\\", "\\\\")
 	return strings.ReplaceAll(out, "\"", "\\\"")
 }
+
+// readyLabel formats a boolean readiness state as a stable label value.
+// Signature: readyLabel(ready bool) string.
+// Why: info metrics need a human-readable status label without changing the
+// numeric ready gauges used for alerting and aggregation.
+func readyLabel(ready bool) string {
+	if ready {
+		return "true"
+	}
+	return "false"
+}
+
+// defaultLabel returns a safe fallback for empty metric label values.
+// Signature: defaultLabel(value string, fallback string) string.
+// Why: Flux status fields can be absent during startup, but Prometheus labels
+// should remain explicit rather than silently becoming empty strings.
+func defaultLabel(value string, fallback string) string {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return fallback
+	}
+	return value
+}

internal/service/daemon.go

@@ -32,6 +32,8 @@ type Daemon struct {
 	targets  []Target
 	log      *log.Logger
 	exporter *metrics.Exporter
+
+	postStartAutoHealOverride func(context.Context) error
 }
 
 var sshConfigCandidates = []string{
@@ -92,7 +94,12 @@ func (d *Daemon) Run(ctx context.Context) error {
 
 	lastGood := map[string]time.Time{}
 	lastOnBattery := map[string]bool{}
+	onBatterySince := map[string]time.Time{}
 	breachCount := map[string]int{}
+	lastAutoHeal := time.Time{}
+	lastGitOpsPoll := time.Time{}
+	gitOpsPollRunning := false
+	gitOpsDone := make(chan struct{}, 1)
 	for _, t := range d.targets {
 		lastGood[t.Name] = time.Now()
 	}
@@ -104,15 +111,21 @@ func (d *Daemon) Run(ctx context.Context) error {
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
+		case <-gitOpsDone:
+			gitOpsPollRunning = false
 		case <-t.C:
 			budget := d.orch.EstimatedEmergencyShutdownSeconds()
 			threshold := int(math.Ceil(float64(budget) * d.cfg.UPS.RuntimeSafetyFactor))
+			anyOnBattery := false
 
 			d.exporter.UpdateBudget(budget)
 
 			for _, target := range d.targets {
 				sample, err := target.Provider.Read(ctx)
 				if err != nil {
+					if lastOnBattery[target.Name] {
+						anyOnBattery = true
+					}
 					d.log.Printf("warning: ups read failed target=%s (%s): %v", target.Name, target.Target, err)
 					d.exporter.UpdateSample(metrics.Sample{
 						Name:         target.Name,
@@ -131,17 +144,45 @@ func (d *Daemon) Run(ctx context.Context) error {
 				}
 
 				lastGood[target.Name] = time.Now()
+				if sample.OnBattery {
+					anyOnBattery = true
+				}
+				wasOnBattery := lastOnBattery[target.Name]
+				if sample.OnBattery {
+					if !wasOnBattery || onBatterySince[target.Name].IsZero() {
+						onBatterySince[target.Name] = time.Now()
+					}
+				} else {
+					onBatterySince[target.Name] = time.Time{}
+				}
 				lastOnBattery[target.Name] = sample.OnBattery
 
-				trigger := sample.LowBattery || (sample.OnBattery && sample.RuntimeSeconds > 0 && sample.RuntimeSeconds <= threshold)
+				onBatteryElapsed := 0
+				if sample.OnBattery && !onBatterySince[target.Name].IsZero() {
+					onBatteryElapsed = int(time.Since(onBatterySince[target.Name]).Seconds())
+				}
+
+				trigger := false
+				triggerReason := ""
+				switch {
+				case sample.LowBattery:
+					trigger = true
+					triggerReason = fmt.Sprintf("ups-low-battery target=%s status=%s", target.Name, sample.RawStatus)
+				case sample.OnBattery && sample.RuntimeSeconds > 0 && sample.RuntimeSeconds <= threshold:
+					trigger = true
+					triggerReason = fmt.Sprintf("ups-threshold target=%s runtime=%ds threshold=%ds status=%s", target.Name, sample.RuntimeSeconds, threshold, sample.RawStatus)
+				case sample.OnBattery && d.cfg.UPS.OnBatteryGraceSeconds > 0 && onBatteryElapsed >= d.cfg.UPS.OnBatteryGraceSeconds:
+					trigger = true
+					triggerReason = fmt.Sprintf("ups-on-battery target=%s elapsed=%ds grace=%ds status=%s", target.Name, onBatteryElapsed, d.cfg.UPS.OnBatteryGraceSeconds, sample.RawStatus)
+				}
 				if trigger {
 					breachCount[target.Name]++
 				} else {
 					breachCount[target.Name] = 0
 				}
 
-				d.log.Printf("ups target=%s status=%s on_battery=%t runtime_s=%d threshold_s=%d budget_s=%d trigger=%t breach=%d",
+				d.log.Printf("ups target=%s status=%s on_battery=%t on_battery_s=%d grace_s=%d runtime_s=%d threshold_s=%d budget_s=%d trigger=%t breach=%d",
-					target.Name, sample.RawStatus, sample.OnBattery, sample.RuntimeSeconds, threshold, budget, trigger, breachCount[target.Name])
+					target.Name, sample.RawStatus, sample.OnBattery, onBatteryElapsed, d.cfg.UPS.OnBatteryGraceSeconds, sample.RuntimeSeconds, threshold, budget, trigger, breachCount[target.Name])
 
 				d.exporter.UpdateSample(metrics.Sample{
 					Name:          target.Name,
@@ -160,14 +201,60 @@ func (d *Daemon) Run(ctx context.Context) error {
 				})
 
 				if breachCount[target.Name] >= debounce {
-					reason := fmt.Sprintf("ups-threshold target=%s runtime=%ds threshold=%ds status=%s", target.Name, sample.RuntimeSeconds, threshold, sample.RawStatus)
+					return d.triggerShutdown(ctx, triggerReason)
-					return d.triggerShutdown(ctx, reason)
 				}
 			}
+
+			d.maybeRunPostStartAutoHeal(ctx, &lastAutoHeal, anyOnBattery)
+			gitOpsPollRunning = d.maybeStartGitOpsSnapshot(ctx, &lastGitOpsPoll, gitOpsPollRunning, gitOpsDone)
 		}
 	}
 }
 
+// maybeRunPostStartAutoHeal runs one orchestration or CLI step.
+// Signature: (d *Daemon) maybeRunPostStartAutoHeal(ctx context.Context, lastRun *time.Time, anyOnBattery bool).
+// Why: gives the long-running daemon a bounded path to repair post-start drift
+// like a later Vault reseal or stale dead-node deletions without waiting for a
+// fresh bootstrap run.
+func (d *Daemon) maybeRunPostStartAutoHeal(ctx context.Context, lastRun *time.Time, anyOnBattery bool) {
+	// Shared cluster repairs have one owner; peers still monitor their UPS and
+	// forward shutdown intent through the existing coordination path.
+	if d.cfg.Coordination.Role == "peer" {
+		return
+	}
+	interval := time.Duration(d.cfg.Startup.PostStartAutoHealSeconds) * time.Second
+	if interval <= 0 || anyOnBattery {
+		return
+	}
+	if d.orch == nil && d.postStartAutoHealOverride == nil {
+		return
+	}
+	now := time.Now()
+	if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < interval {
+		return
+	}
+	if lastRun != nil {
+		*lastRun = now
+	}
+	if err := d.runPostStartAutoHeal(ctx); err != nil {
+		d.log.Printf("warning: post-start auto-heal: %v", err)
+	}
+}
+
+// runPostStartAutoHeal runs one orchestration or CLI step.
+// Signature: (d *Daemon) runPostStartAutoHeal(ctx context.Context) error.
+// Why: keeps the daemon loop readable while allowing unit tests to inject a
+// deterministic repair hook without a live cluster.
+func (d *Daemon) runPostStartAutoHeal(ctx context.Context) error {
+	if d.postStartAutoHealOverride != nil {
+		return d.postStartAutoHealOverride(ctx)
+	}
+	if d.orch == nil {
+		return nil
+	}
+	return d.orch.RunPostStartAutoHeal(ctx)
+}
+
 // triggerShutdown runs one orchestration or CLI step.
 // Signature: (d *Daemon) triggerShutdown(ctx context.Context, reason string) error.
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.

internal/service/daemon_additional_test.go

@@ -165,6 +165,50 @@ func TestDaemonRunTriggersShutdownOnTelemetryTimeout(t *testing.T) {
 	}
 }
 
+// TestDaemonRunTriggersShutdownAfterOnBatteryGrace runs one orchestration or CLI step.
+// Signature: TestDaemonRunTriggersShutdownAfterOnBatteryGrace(t *testing.T).
+// Why: covers the sustained-on-battery trigger so short runtime estimates are not
+// the only path to a graceful shutdown during abrupt power loss.
+func TestDaemonRunTriggersShutdownAfterOnBatteryGrace(t *testing.T) {
+	stateDir := t.TempDir()
+	orch := newDaemonTestOrchestrator(t, stateDir)
+	d := &Daemon{
+		cfg: config.Config{
+			UPS: config.UPS{
+				Enabled:               true,
+				PollSeconds:           1,
+				DebounceCount:         1,
+				RuntimeSafetyFactor:   1.0,
+				OnBatteryGraceSeconds: 1,
+			},
+			State: config.State{
+				IntentPath: filepath.Join(stateDir, "intent.json"),
+			},
+			Shutdown: config.Shutdown{
+				EmergencySkipDrain: true,
+				EmergencySkipEtcd:  true,
+			},
+		},
+		orch: orch,
+		targets: []Target{
+			{
+				Name:   "Pyrphoros",
+				Target: "pyrphoros@localhost",
+				Provider: &daemonFakeProvider{
+					samples: []ups.Sample{{OnBattery: true, LowBattery: false, RuntimeSeconds: 9999, RawStatus: "OB"}},
+				},
+			},
+		},
+		log:      log.New(io.Discard, "", 0),
+		exporter: metrics.New(),
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	if err := d.Run(ctx); err != nil {
+		t.Fatalf("expected sustained-on-battery shutdown path to complete, got %v", err)
+	}
+}
+
 // TestForwardShutdownSucceedsWithSSHShim runs one orchestration or CLI step.
 // Signature: TestForwardShutdownSucceedsWithSSHShim(t *testing.T).
 // Why: covers forward-shutdown SSH execution path.

internal/service/daemon_poststart_autorepair_test.go

@@ -0,0 +1,89 @@
+package service
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"log"
+	"strings"
+
+	"scm.bstein.dev/bstein/ananke/internal/cluster"
+	"scm.bstein.dev/bstein/ananke/internal/execx"
+	"testing"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+)
+
+// TestDaemonMaybeRunPostStartAutoHeal runs one orchestration or CLI step.
+// Signature: TestDaemonMaybeRunPostStartAutoHeal(t *testing.T).
+// Why: covers the daemon-side interval and on-battery guards for the new
+// post-start repair loop.
+func TestDaemonMaybeRunPostStartAutoHeal(t *testing.T) {
+	calls := 0
+	d := &Daemon{
+		cfg: config.Config{
+			Startup: config.Startup{
+				PostStartAutoHealSeconds: 10,
+			},
+		},
+		postStartAutoHealOverride: func(context.Context) error {
+			calls++
+			return nil
+		},
+	}
+
+	d.cfg.Coordination.Role = "peer"
+	var last time.Time
+	d.maybeRunPostStartAutoHeal(context.Background(), &last, false)
+	if calls != 0 || !last.IsZero() {
+		t.Fatal("peer must not mutate shared cluster recovery state")
+	}
+	d.cfg.Coordination.Role = "coordinator"
+	d.maybeRunPostStartAutoHeal(context.Background(), &last, false)
+	if calls != 1 {
+		t.Fatalf("expected first auto-heal invocation, got %d", calls)
+	}
+
+	d.maybeRunPostStartAutoHeal(context.Background(), &last, false)
+	if calls != 1 {
+		t.Fatalf("expected interval guard to suppress second call, got %d", calls)
+	}
+
+	last = time.Now().Add(-11 * time.Second)
+	d.maybeRunPostStartAutoHeal(context.Background(), &last, true)
+	if calls != 1 {
+		t.Fatalf("expected on-battery guard to suppress call, got %d", calls)
+	}
+
+	last = time.Now().Add(-11 * time.Second)
+	d.maybeRunPostStartAutoHeal(context.Background(), &last, false)
+	if calls != 2 {
+		t.Fatalf("expected second allowed auto-heal call, got %d", calls)
+	}
+}
+
+// TestDaemonRecoveryFailureDoesNotStopMonitoring exercises optional repair isolation.
+// Signature: TestDaemonRecoveryFailureDoesNotStopMonitoring(t *testing.T).
+// Why: a missing or failed repair helper must not terminate UPS monitoring.
+func TestDaemonRecoveryFailureDoesNotStopMonitoring(t *testing.T) {
+	var messages bytes.Buffer
+	d := &Daemon{cfg: config.Config{Startup: config.Startup{PostStartAutoHealSeconds: 60}}, log: log.New(&messages, "", 0)}
+	var last time.Time
+	d.maybeRunPostStartAutoHeal(context.Background(), &last, false)
+	if !last.IsZero() {
+		t.Fatal("missing helper must not consume an interval")
+	}
+	if err := d.runPostStartAutoHeal(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	d.orch = cluster.New(config.Config{}, &execx.Runner{DryRun: true}, nil, d.log)
+	if err := d.runPostStartAutoHeal(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	d.postStartAutoHealOverride = func(context.Context) error { return errors.New("synthetic repair failure") }
+	d.maybeRunPostStartAutoHeal(context.Background(), &last, false)
+	if last.IsZero() || !strings.Contains(messages.String(), "synthetic repair failure") {
+		t.Fatal("failed repair must be logged and rate limited")
+	}
+}

internal/service/gitops_snapshot.go

@@ -0,0 +1,355 @@
+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"sort"
+	"strings"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+	"scm.bstein.dev/bstein/ananke/internal/metrics"
+)
+
+var gitOpsKubectlOutput = runGitOpsKubectlOutput
+
+type gitOpsResourceList struct {
+	Items []gitOpsResource `json:"items"`
+}
+
+type gitOpsResource struct {
+	Metadata struct {
+		Name      string `json:"name"`
+		Namespace string `json:"namespace"`
+	} `json:"metadata"`
+	Spec struct {
+		Suspend bool `json:"suspend"`
+		Ref     struct {
+			Branch string `json:"branch"`
+			Tag    string `json:"tag"`
+			SemVer string `json:"semver"`
+		} `json:"ref"`
+		URL       string `json:"url"`
+		Path      string `json:"path"`
+		SourceRef struct {
+			Name      string `json:"name"`
+			Namespace string `json:"namespace"`
+		} `json:"sourceRef"`
+		Chart struct {
+			Spec struct {
+				Chart   string `json:"chart"`
+				Version string `json:"version"`
+			} `json:"spec"`
+		} `json:"chart"`
+	} `json:"spec"`
+	Status struct {
+		Artifact struct {
+			Revision string `json:"revision"`
+		} `json:"artifact"`
+		LastAppliedRevision   string `json:"lastAppliedRevision"`
+		LastAttemptedRevision string `json:"lastAttemptedRevision"`
+		Conditions            []struct {
+			Type   string `json:"type"`
+			Status string `json:"status"`
+			Reason string `json:"reason"`
+		} `json:"conditions"`
+		History []struct {
+			ChartName    string `json:"chartName"`
+			ChartVersion string `json:"chartVersion"`
+			AppVersion   string `json:"appVersion"`
+			Digest       string `json:"digest"`
+		} `json:"history"`
+	} `json:"status"`
+}
+
+// maybeStartGitOpsSnapshot starts a bounded background scrape when the GitOps
+// sample interval has elapsed.
+// Signature: (d *Daemon) maybeStartGitOpsSnapshot(ctx context.Context, lastRun *time.Time, running bool, done chan<- struct{}) bool.
+// Why: Kubernetes API reads must not block UPS polling, especially during
+// emergency power events when the daemon's first job is still safe shutdown.
+func (d *Daemon) maybeStartGitOpsSnapshot(ctx context.Context, lastRun *time.Time, running bool, done chan<- struct{}) bool {
+	if !d.cfg.GitOps.Enabled || d.exporter == nil || running {
+		return running
+	}
+	interval := time.Duration(d.cfg.GitOps.PollSeconds) * time.Second
+	if interval <= 0 {
+		interval = 60 * time.Second
+	}
+	now := time.Now()
+	if lastRun != nil && !lastRun.IsZero() && now.Sub(*lastRun) < interval {
+		return running
+	}
+	if lastRun != nil {
+		*lastRun = now
+	}
+	cfg := d.cfg
+	exporter := d.exporter
+	logger := d.log
+	go func() {
+		defer func() {
+			select {
+			case done <- struct{}{}:
+			default:
+			}
+		}()
+		scrapeCtx, cancel := context.WithTimeout(ctx, 25*time.Second)
+		defer cancel()
+		snapshot := collectGitOpsSnapshot(scrapeCtx, cfg)
+		exporter.UpdateGitOpsSnapshot(snapshot)
+		if !snapshot.ScrapeSuccess && logger != nil {
+			logger.Printf("warning: gitops metrics scrape partial: %s", gitOpsErrorSummary(snapshot.Errors))
+		}
+	}()
+	return true
+}
+
+// collectGitOpsSnapshot queries Flux custom resources and converts them into
+// Prometheus-safe metric state.
+// Signature: collectGitOpsSnapshot(ctx context.Context, cfg config.Config) metrics.GitOpsSnapshot.
+// Why: the Grafana overview needs current branch/readiness/suspend state, which
+// is object status rather than controller operational telemetry.
+func collectGitOpsSnapshot(ctx context.Context, cfg config.Config) metrics.GitOpsSnapshot {
+	snapshot := metrics.GitOpsSnapshot{
+		UpdatedAt:     time.Now().UTC(),
+		ScrapeSuccess: true,
+		Errors:        map[string]string{},
+	}
+
+	if raw, err := gitOpsKubectlJSON(ctx, cfg, "get", "gitrepositories.source.toolkit.fluxcd.io", "-A", "-o", "json"); err != nil {
+		snapshot.Errors["gitrepository"] = err.Error()
+	} else if sources, err := parseGitRepositories(raw); err != nil {
+		snapshot.Errors["gitrepository"] = err.Error()
+	} else {
+		snapshot.FluxSources = sources
+	}
+
+	if raw, err := gitOpsKubectlJSON(ctx, cfg, "get", "kustomizations.kustomize.toolkit.fluxcd.io", "-A", "-o", "json"); err != nil {
+		snapshot.Errors["kustomization"] = err.Error()
+	} else if kustomizations, err := parseKustomizations(raw); err != nil {
+		snapshot.Errors["kustomization"] = err.Error()
+	} else {
+		snapshot.Kustomizations = kustomizations
+	}
+
+	if raw, err := gitOpsKubectlJSON(ctx, cfg, "get", "helmreleases.helm.toolkit.fluxcd.io", "-A", "-o", "json"); err != nil {
+		snapshot.Errors["helmrelease"] = err.Error()
+	} else if releases, err := parseHelmReleases(raw); err != nil {
+		snapshot.Errors["helmrelease"] = err.Error()
+	} else {
+		snapshot.HelmReleases = releases
+	}
+
+	snapshot.ScrapeSuccess = len(snapshot.Errors) == 0
+	return snapshot
+}
+
+// parseGitRepositories extracts Git source branch/revision/readiness from Flux
+// GitRepository JSON.
+// Signature: parseGitRepositories(raw []byte) ([]metrics.GitOpsFluxSource, error).
+// Why: keeping status parsing isolated makes it testable without a live Flux
+// API and protects the daemon loop from brittle JSON traversal code.
+func parseGitRepositories(raw []byte) ([]metrics.GitOpsFluxSource, error) {
+	var list gitOpsResourceList
+	if err := json.Unmarshal(raw, &list); err != nil {
+		return nil, fmt.Errorf("decode gitrepositories: %w", err)
+	}
+	out := make([]metrics.GitOpsFluxSource, 0, len(list.Items))
+	for _, item := range list.Items {
+		ready, reason := fluxReady(item.Status.Conditions)
+		out = append(out, metrics.GitOpsFluxSource{
+			Namespace: defaultString(item.Metadata.Namespace, "default"),
+			Name:      item.Metadata.Name,
+			URL:       item.Spec.URL,
+			Branch:    sourceBranch(item),
+			Revision:  item.Status.Artifact.Revision,
+			Ready:     ready,
+			Reason:    reason,
+			Suspended: item.Spec.Suspend,
+		})
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return out[i].Namespace+"/"+out[i].Name < out[j].Namespace+"/"+out[j].Name
+	})
+	return out, nil
+}
+
+// parseKustomizations extracts Kustomization health and source metadata from
+// Flux Kustomization JSON.
+// Signature: parseKustomizations(raw []byte) ([]metrics.GitOpsKustomization, error).
+// Why: Kustomization status drives both the overview summary and the detailed
+// GitOps table, so the parser keeps the metric contract in one place.
+func parseKustomizations(raw []byte) ([]metrics.GitOpsKustomization, error) {
+	var list gitOpsResourceList
+	if err := json.Unmarshal(raw, &list); err != nil {
+		return nil, fmt.Errorf("decode kustomizations: %w", err)
+	}
+	out := make([]metrics.GitOpsKustomization, 0, len(list.Items))
+	for _, item := range list.Items {
+		ready, reason := fluxReady(item.Status.Conditions)
+		namespace := defaultString(item.Metadata.Namespace, "default")
+		out = append(out, metrics.GitOpsKustomization{
+			Namespace:       namespace,
+			Name:            item.Metadata.Name,
+			Path:            item.Spec.Path,
+			SourceNamespace: defaultString(item.Spec.SourceRef.Namespace, namespace),
+			SourceName:      item.Spec.SourceRef.Name,
+			Revision:        firstNonEmpty(item.Status.LastAppliedRevision, item.Status.LastAttemptedRevision),
+			Ready:           ready,
+			Reason:          reason,
+			Suspended:       item.Spec.Suspend,
+		})
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return out[i].Namespace+"/"+out[i].Name < out[j].Namespace+"/"+out[j].Name
+	})
+	return out, nil
+}
+
+// parseHelmReleases extracts HelmRelease health and chart metadata from Flux
+// HelmRelease JSON.
+// Signature: parseHelmReleases(raw []byte) ([]metrics.GitOpsHelmRelease, error).
+// Why: HelmRelease status has different chart fields than Kustomizations, and
+// isolating it keeps dashboard metrics stable as Flux payloads evolve.
+func parseHelmReleases(raw []byte) ([]metrics.GitOpsHelmRelease, error) {
+	var list gitOpsResourceList
+	if err := json.Unmarshal(raw, &list); err != nil {
+		return nil, fmt.Errorf("decode helmreleases: %w", err)
+	}
+	out := make([]metrics.GitOpsHelmRelease, 0, len(list.Items))
+	for _, item := range list.Items {
+		ready, reason := fluxReady(item.Status.Conditions)
+		chart := item.Spec.Chart.Spec.Chart
+		version := item.Spec.Chart.Spec.Version
+		appVersion := ""
+		revision := firstNonEmpty(item.Status.LastAppliedRevision, item.Status.LastAttemptedRevision)
+		if len(item.Status.History) > 0 {
+			latest := item.Status.History[0]
+			chart = firstNonEmpty(latest.ChartName, chart)
+			version = firstNonEmpty(latest.ChartVersion, version)
+			appVersion = latest.AppVersion
+			revision = firstNonEmpty(latest.Digest, revision)
+		}
+		out = append(out, metrics.GitOpsHelmRelease{
+			Namespace:  defaultString(item.Metadata.Namespace, "default"),
+			Name:       item.Metadata.Name,
+			Chart:      chart,
+			Version:    version,
+			AppVersion: appVersion,
+			Revision:   revision,
+			Ready:      ready,
+			Reason:     reason,
+			Suspended:  item.Spec.Suspend,
+		})
+	}
+	sort.Slice(out, func(i, j int) bool {
+		return out[i].Namespace+"/"+out[i].Name < out[j].Namespace+"/"+out[j].Name
+	})
+	return out, nil
+}
+
+// gitOpsKubectlJSON runs kubectl with the configured kubeconfig and a short
+// request timeout.
+// Signature: gitOpsKubectlJSON(ctx context.Context, cfg config.Config, args ...string) ([]byte, error).
+// Why: every GitOps scrape should fail quickly instead of stealing time from
+// UPS safety polling during outage recovery.
+func gitOpsKubectlJSON(ctx context.Context, cfg config.Config, args ...string) ([]byte, error) {
+	finalArgs := []string{}
+	if strings.TrimSpace(cfg.Kubeconfig) != "" {
+		finalArgs = append(finalArgs, "--kubeconfig", cfg.Kubeconfig)
+	}
+	finalArgs = append(finalArgs, "--request-timeout=8s")
+	finalArgs = append(finalArgs, args...)
+	return gitOpsKubectlOutput(ctx, "kubectl", finalArgs...)
+}
+
+// runGitOpsKubectlOutput is the production command runner for GitOps snapshot
+// collection.
+// Signature: runGitOpsKubectlOutput(ctx context.Context, name string, args ...string) ([]byte, error).
+// Why: preserving stderr in errors makes live diagnosis possible when Flux CRD
+// reads fail on a recovering control plane.
+func runGitOpsKubectlOutput(ctx context.Context, name string, args ...string) ([]byte, error) {
+	cmd := exec.CommandContext(ctx, name, args...)
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		trimmed := strings.TrimSpace(string(out))
+		if trimmed == "" {
+			return nil, err
+		}
+		return nil, fmt.Errorf("%w: %s", err, trimmed)
+	}
+	return out, nil
+}
+
+// fluxReady resolves a Flux Ready condition into a boolean and reason.
+// Signature: fluxReady(conditions []struct { Type string; Status string; Reason string }) (bool, string).
+// Why: Flux resources all use Ready conditions, so one helper keeps status
+// interpretation consistent across GitRepository, Kustomization, and HelmRelease.
+func fluxReady(conditions []struct {
+	Type   string `json:"type"`
+	Status string `json:"status"`
+	Reason string `json:"reason"`
+}) (bool, string) {
+	for _, condition := range conditions {
+		if condition.Type == "Ready" {
+			return strings.EqualFold(condition.Status, "True"), defaultString(condition.Reason, "ReadyUnknown")
+		}
+	}
+	return false, "ReadyMissing"
+}
+
+// sourceBranch returns the most useful source reference label for a
+// GitRepository.
+// Signature: sourceBranch(item gitOpsResource) string.
+// Why: branch is preferred for Atlas, but tags and semver refs should still
+// render clearly if Flux is ever pinned differently.
+func sourceBranch(item gitOpsResource) string {
+	return firstNonEmpty(item.Spec.Ref.Branch, item.Spec.Ref.Tag, item.Spec.Ref.SemVer, "unknown")
+}
+
+// defaultString trims a value and returns a fallback when it is empty.
+// Signature: defaultString(value string, fallback string) string.
+// Why: Flux omits some optional fields; dashboards are easier to read with
+// explicit fallback labels.
+func defaultString(value string, fallback string) string {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return fallback
+	}
+	return value
+}
+
+// firstNonEmpty returns the first non-empty trimmed value.
+// Signature: firstNonEmpty(values ...string) string.
+// Why: Flux has several revision fields depending on resource kind and
+// reconciliation stage, so callers can express preferred fallback order.
+func firstNonEmpty(values ...string) string {
+	for _, value := range values {
+		value = strings.TrimSpace(value)
+		if value != "" {
+			return value
+		}
+	}
+	return ""
+}
+
+// gitOpsErrorSummary renders resource-family scrape errors for daemon logs.
+// Signature: gitOpsErrorSummary(errors map[string]string) string.
+// Why: a compact sorted summary keeps recurring scrape warnings readable in
+// journald during partial cluster recovery.
+func gitOpsErrorSummary(errors map[string]string) string {
+	if len(errors) == 0 {
+		return "none"
+	}
+	keys := make([]string, 0, len(errors))
+	for key := range errors {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	parts := make([]string, 0, len(keys))
+	for _, key := range keys {
+		parts = append(parts, key+"="+errors[key])
+	}
+	return strings.Join(parts, "; ")
+}

internal/service/testing_hooks_gitops.go

@@ -0,0 +1,127 @@
+package service
+
+import (
+	"context"
+	"log"
+	"time"
+
+	"scm.bstein.dev/bstein/ananke/internal/config"
+	"scm.bstein.dev/bstein/ananke/internal/metrics"
+)
+
+type TestHookGitOpsRunner func(context.Context, string, ...string) ([]byte, error)
+
+// TestHookParseGitRepositories exposes GitRepository parsing to the top-level
+// testing module.
+// Signature: TestHookParseGitRepositories(raw []byte) ([]metrics.GitOpsFluxSource, error).
+// Why: Ananke keeps split-module tests outside internal packages, but the
+// parser needs direct contract coverage without a live cluster.
+func TestHookParseGitRepositories(raw []byte) ([]metrics.GitOpsFluxSource, error) {
+	return parseGitRepositories(raw)
+}
+
+// TestHookParseKustomizations exposes Kustomization parsing to the top-level
+// testing module.
+// Signature: TestHookParseKustomizations(raw []byte) ([]metrics.GitOpsKustomization, error).
+// Why: dashboard-facing labels should be verified from representative Flux JSON
+// without relying on the current production cluster shape.
+func TestHookParseKustomizations(raw []byte) ([]metrics.GitOpsKustomization, error) {
+	return parseKustomizations(raw)
+}
+
+// TestHookParseHelmReleases exposes HelmRelease parsing to the top-level
+// testing module.
+// Signature: TestHookParseHelmReleases(raw []byte) ([]metrics.GitOpsHelmRelease, error).
+// Why: Helm status fields vary by reconciliation phase, and split-module tests
+// need a stable seam for parser coverage.
+func TestHookParseHelmReleases(raw []byte) ([]metrics.GitOpsHelmRelease, error) {
+	return parseHelmReleases(raw)
+}
+
+// TestHookCollectGitOpsSnapshotWithRunner runs snapshot collection with an
+// injected kubectl runner.
+// Signature: TestHookCollectGitOpsSnapshotWithRunner(ctx context.Context, cfg config.Config, runner TestHookGitOpsRunner) metrics.GitOpsSnapshot.
+// Why: this covers success and partial-failure collection paths without shelling
+// out to kubectl from unit tests.
+func TestHookCollectGitOpsSnapshotWithRunner(ctx context.Context, cfg config.Config, runner TestHookGitOpsRunner) metrics.GitOpsSnapshot {
+	original := gitOpsKubectlOutput
+	defer func() { gitOpsKubectlOutput = original }()
+	gitOpsKubectlOutput = runner
+	return collectGitOpsSnapshot(ctx, cfg)
+}
+
+// TestHookMaybeStartGitOpsSnapshot starts the daemon's background GitOps scrape
+// from split-module tests.
+// Signature: TestHookMaybeStartGitOpsSnapshot(ctx context.Context, cfg config.Config, exporter *metrics.Exporter, logger *log.Logger, lastRun *time.Time, running bool, done chan<- struct{}) bool.
+// Why: the daemon intentionally keeps scraping asynchronous so GitOps telemetry
+// cannot delay UPS polling; the behavior needs direct coverage.
+func TestHookMaybeStartGitOpsSnapshot(ctx context.Context, cfg config.Config, exporter *metrics.Exporter, logger *log.Logger, lastRun *time.Time, running bool, done chan<- struct{}) bool {
+	d := &Daemon{cfg: cfg, exporter: exporter, log: logger}
+	return d.maybeStartGitOpsSnapshot(ctx, lastRun, running, done)
+}
+
+// TestHookMaybeStartGitOpsSnapshotWithRunner starts a background GitOps scrape
+// with an injected runner and restores the production runner after completion.
+// Signature: TestHookMaybeStartGitOpsSnapshotWithRunner(ctx context.Context, cfg config.Config, exporter *metrics.Exporter, logger *log.Logger, lastRun *time.Time, running bool, done chan<- struct{}, runner TestHookGitOpsRunner) bool.
+// Why: the scrape is asynchronous, so split-module tests need a seam that keeps
+// fake kubectl behavior installed until the goroutine exits.
+func TestHookMaybeStartGitOpsSnapshotWithRunner(ctx context.Context, cfg config.Config, exporter *metrics.Exporter, logger *log.Logger, lastRun *time.Time, running bool, done chan<- struct{}, runner TestHookGitOpsRunner) bool {
+	original := gitOpsKubectlOutput
+	gitOpsKubectlOutput = runner
+	proxyDone := make(chan struct{}, 1)
+	d := &Daemon{cfg: cfg, exporter: exporter, log: logger}
+	started := d.maybeStartGitOpsSnapshot(ctx, lastRun, running, proxyDone)
+	if !started || running {
+		gitOpsKubectlOutput = original
+		return started
+	}
+	go func() {
+		<-proxyDone
+		gitOpsKubectlOutput = original
+		select {
+		case done <- struct{}{}:
+		default:
+		}
+	}()
+	return started
+}
+
+// TestHookRunGitOpsKubectlOutput exposes the production command runner to
+// split-module tests.
+// Signature: TestHookRunGitOpsKubectlOutput(ctx context.Context, name string, args ...string) ([]byte, error).
+// Why: stderr preservation and empty-stderr failure behavior are part of the
+// operator-facing diagnostics contract.
+func TestHookRunGitOpsKubectlOutput(ctx context.Context, name string, args ...string) ([]byte, error) {
+	return runGitOpsKubectlOutput(ctx, name, args...)
+}
+
+// TestHookGitOpsDefaultString exposes defaultString to split-module tests.
+// Signature: TestHookGitOpsDefaultString(value string, fallback string) string.
+// Why: fallback label behavior is intentionally tiny but important for readable
+// Grafana tables during startup.
+func TestHookGitOpsDefaultString(value string, fallback string) string {
+	return defaultString(value, fallback)
+}
+
+// TestHookGitOpsFirstNonEmpty exposes firstNonEmpty to split-module tests.
+// Signature: TestHookGitOpsFirstNonEmpty(values ...string) string.
+// Why: revision fallback order should remain deterministic as Flux status
+// payloads change.
+func TestHookGitOpsFirstNonEmpty(values ...string) string {
+	return firstNonEmpty(values...)
+}
+
+// TestHookGitOpsErrorSummary exposes gitOpsErrorSummary to split-module tests.
+// Signature: TestHookGitOpsErrorSummary(errors map[string]string) string.
+// Why: sorted scrape warnings are easier to inspect in journald during recovery.
+func TestHookGitOpsErrorSummary(errors map[string]string) string {
+	return gitOpsErrorSummary(errors)
+}
+
+// TestHookFluxReadyMissing exposes the missing Ready-condition branch to tests.
+// Signature: TestHookFluxReadyMissing() (bool, string).
+// Why: absent Ready conditions should be treated as not-ready and labelled
+// explicitly instead of being mistaken for healthy.
+func TestHookFluxReadyMissing() (bool, string) {
+	return fluxReady(nil)
+}

internal/state/intent.go

@@ -22,12 +22,23 @@ type Intent struct {
 	UpdatedAt time.Time `json:"updated_at"`
 }
 
-var writeIntentImpl = writeIntentDefault
+var (
+	readIntentImpl  = readIntentDefault
+	writeIntentImpl = writeIntentDefault
+)
 
 // ReadIntent runs one orchestration or CLI step.
 // Signature: ReadIntent(path string) (Intent, error).
 // Why: keeps behavior explicit so startup/shutdown workflows remain maintainable as services evolve.
 func ReadIntent(path string) (Intent, error) {
+	return readIntentImpl(path)
+}
+
+// readIntentDefault runs one orchestration or CLI step.
+// Signature: readIntentDefault(path string) (Intent, error).
+// Why: keeps production read behavior available while tests can override intent
+// reads deterministically without racing background file mutations.
+func readIntentDefault(path string) (Intent, error) {
 	b, err := os.ReadFile(path)
 	if err != nil {
 		if os.IsNotExist(err) {

internal/state/testhooks.go

@@ -22,6 +22,34 @@ func TestHookWriteIntentDefault(path string, in Intent) error {
 	return writeIntentDefault(path, in)
 }
 
+// TestHookReadIntentDefault runs one orchestration or CLI step.
+// Signature: TestHookReadIntentDefault(path string) (Intent, error).
+// Why: lets top-level tests delegate to production ReadIntent behavior while
+// selectively forcing deterministic read sequences for lifecycle branches.
+func TestHookReadIntentDefault(path string) (Intent, error) {
+	return readIntentDefault(path)
+}
+
+// TestHookSetReadIntentOverride runs one orchestration or CLI step.
+// Signature: TestHookSetReadIntentOverride(fn func(path string) (Intent, error)) (restore func()).
+// Why: enables deterministic intent-read failure injection without sleeping
+// goroutines that race slower CI agents.
+func TestHookSetReadIntentOverride(fn func(path string) (Intent, error)) (restore func()) {
+	testHookOverrideMu.Lock()
+	prev := readIntentImpl
+	if fn == nil {
+		readIntentImpl = readIntentDefault
+	} else {
+		readIntentImpl = fn
+	}
+	testHookOverrideMu.Unlock()
+	return func() {
+		testHookOverrideMu.Lock()
+		readIntentImpl = prev
+		testHookOverrideMu.Unlock()
+	}
+}
+
 // TestHookSetWriteIntentOverride runs one orchestration or CLI step.
 // Signature: TestHookSetWriteIntentOverride(fn func(path string, in Intent) error) (restore func()).
 // Why: enables deterministic intent-write failure injection from the top-level

scripts/ananke-self-update.sh

@@ -13,7 +13,7 @@ HOST_SHORT="$(hostname -s 2>/dev/null || hostname)"
 LOG_FILE="${ANANKE_UPDATE_LOG_FILE:-/var/log/ananke/update.log}"
 STATE_FILE="${ANANKE_UPDATE_STATE_FILE:-/var/lib/ananke/update-last.env}"
 LOCK_FILE="${ANANKE_UPDATE_LOCK_FILE:-/var/lock/ananke-update.lock}"
-ALLOW_QUALITY_FALLBACK="${ANANKE_UPDATE_ALLOW_QUALITY_FALLBACK:-1}"
+ALLOW_QUALITY_FALLBACK="${ANANKE_UPDATE_ALLOW_QUALITY_FALLBACK:-0}"
 QUALITY_GATE_MODE="${ANANKE_ENFORCE_QUALITY_GATE:-1}"
 
 mkdir -p "$(dirname "${LOG_FILE}")" "$(dirname "${STATE_FILE}")" "$(dirname "${LOCK_FILE}")"
@@ -103,6 +103,8 @@ if "${REPO_DIR}/scripts/install.sh"; then
   write_state "ok" "install-success" "${CURRENT_FROM_REV}" "${TARGET_TO_REV}"
   echo "[self-update] completed successfully"
   exit 0
+else
+  INSTALL_EXIT_CODE=$?
 fi
 
 if [[ "${ALLOW_QUALITY_FALLBACK}" == "1" || "${ALLOW_QUALITY_FALLBACK}" == "true" ]]; then
@@ -113,3 +115,8 @@ if [[ "${ALLOW_QUALITY_FALLBACK}" == "1" || "${ALLOW_QUALITY_FALLBACK}" == "true
   echo "[self-update] completed via fallback mode"
   exit 0
 fi
+
+# A failed command used as an if condition does not trigger errexit or ERR.
+write_state "failed" "install-failed;rc=${INSTALL_EXIT_CODE}" "${CURRENT_FROM_REV}" "${TARGET_TO_REV}"
+echo "[self-update] installer failed rc=${INSTALL_EXIT_CODE}; fallback disabled"
+exit "${INSTALL_EXIT_CODE}"

scripts/install-artifacts.sh

@@ -0,0 +1,116 @@
+# Binary, config template, and systemd artifact helpers for the installer.
+
+resolve_build_target() {
+  if [[ -d "${REPO_DIR}/cmd/ananke" ]]; then
+    echo "./cmd/ananke"
+    return 0
+  fi
+  return 1
+}
+
+install_config_template() {
+  local template="$1"
+  local dest="$2"
+  local src legacy
+  local -a modern_candidates=()
+  local -a legacy_candidates=()
+
+  case "${template}" in
+    coordinator)
+      modern_candidates=("configs/ananke.coordinator.yaml" "configs/ananke.titan-db.yaml")
+      legacy_candidates=("configs/hecate.titan-db.yaml")
+      ;;
+    peer)
+      modern_candidates=("configs/ananke.peer.yaml" "configs/ananke.tethys.yaml")
+      legacy_candidates=("configs/hecate.tethys.yaml")
+      ;;
+    example)
+      modern_candidates=("configs/ananke.example.yaml")
+      legacy_candidates=("configs/hecate.example.yaml")
+      ;;
+    *)
+      echo "[install] unknown config template key: ${template}" >&2
+      return 1
+      ;;
+  esac
+
+  for src in "${modern_candidates[@]}"; do
+    if [[ -f "${src}" ]]; then
+      install -m 0640 "${src}" "${dest}"
+      return 0
+    fi
+  done
+
+  for legacy in "${legacy_candidates[@]}"; do
+    if [[ -f "${legacy}" ]]; then
+      src="$(mktemp)"
+      legacy_path_rewrite "${legacy}" "${src}"
+      install -m 0640 "${src}" "${dest}"
+      rm -f "${src}"
+      return 0
+    fi
+  done
+
+  echo "[install] missing config template sources for '${template}'. modern=[${modern_candidates[*]}] legacy=[${legacy_candidates[*]}]" >&2
+  return 1
+}
+
+install_systemd_units() {
+  local tmp
+
+  while IFS='|' read -r target_name modern_name legacy_name; do
+    local modern_src="deploy/systemd/${modern_name}"
+    local legacy_src="deploy/systemd/${legacy_name}"
+    local target="${SYSTEMD_DIR}/${target_name}"
+
+    if [[ -f "${modern_src}" ]]; then
+      install -m 0644 "${modern_src}" "${target}"
+      continue
+    fi
+
+    if [[ -f "${legacy_src}" ]]; then
+      tmp="$(mktemp)"
+      legacy_path_rewrite "${legacy_src}" "${tmp}"
+      install -m 0644 "${tmp}" "${target}"
+      rm -f "${tmp}"
+      continue
+    fi
+
+    echo "[install] missing both modern and legacy systemd unit sources for ${target_name}" >&2
+    return 1
+  done <<'EOF_UNITS'
+ananke.service|ananke.service|hecate.service
+ananke-bootstrap.service|ananke-bootstrap.service|hecate-bootstrap.service
+ananke-update.service|ananke-update.service|hecate-update.service
+ananke-update.timer|ananke-update.timer|hecate-update.timer
+EOF_UNITS
+}
+
+install_self_update_script() {
+  local modern_src="scripts/ananke-self-update.sh"
+  local legacy_src="scripts/hecate-self-update.sh"
+  local target="${LIB_DIR}/ananke-self-update.sh"
+  local tmp
+
+  if [[ -f "${modern_src}" ]]; then
+    install -m 0755 "${modern_src}" "${target}"
+    return 0
+  fi
+
+  if [[ -f "${legacy_src}" ]]; then
+    tmp="$(mktemp)"
+    legacy_path_rewrite "${legacy_src}" "${tmp}"
+    sed -Ei \
+      -e 's/HECATE_/ANANKE_/g' \
+      -e 's/hecate-self-update/ananke-self-update/g' \
+      -e 's#/opt/hecate#/opt/ananke#g' \
+      -e 's#bstein/hecate\.git#bstein/ananke.git#g' \
+      "${tmp}"
+    install -m 0755 "${tmp}" "${target}"
+    rm -f "${tmp}"
+    return 0
+  fi
+
+  echo "[install] missing both modern and legacy self-update scripts." >&2
+  return 1
+}

scripts/install-config-migration.sh

@@ -0,0 +1,352 @@
+# Config migration helpers for the Ananke host installer.
+
+read_ananke_role() {
+  if [[ ! -f "${CONF_DIR}/ananke.yaml" ]]; then
+    echo "coordinator"
+    return 0
+  fi
+  local role
+  role="$(awk '/^[[:space:]]*role:[[:space:]]*/ {print $2; exit}' "${CONF_DIR}/ananke.yaml" 2>/dev/null || true)"
+  if [[ -z "${role}" ]]; then
+    role="coordinator"
+  fi
+  echo "${role}"
+}
+
+migration_yaml_lookup() {
+  local key="$1"
+  awk -F': *' -v k="${key}" '$1 == k {print $2; exit}' "${CONF_DIR}/ananke.yaml" 2>/dev/null || true
+}
+
+first_control_plane_name() {
+  awk '
+    /^control_planes:[[:space:]]*$/ {in_list=1; next}
+    in_list && /^[[:space:]]*-[[:space:]]*/ {gsub(/^[[:space:]]*-[[:space:]]*/, "", $0); print $0; exit}
+    in_list && /^[^[:space:]]/ {in_list=0}
+  ' "${CONF_DIR}/ananke.yaml" 2>/dev/null || true
+}
+
+lookup_node_host() {
+  local node="$1"
+  awk -F': *' -v n="${node}" '$1 == "  " n {print $2; exit}' "${CONF_DIR}/ananke.yaml" 2>/dev/null || true
+}
+
+migrate_ananke_config() {
+  if [[ ! -f "${CONF_DIR}/ananke.yaml" ]]; then
+    return 0
+  fi
+
+  local changed=0
+  local role_hint
+  role_hint="$(read_ananke_role)"
+  if grep -Eq 'default_budget_seconds:[[:space:]]*300' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei 's/(default_budget_seconds:[[:space:]]*)300/\11380/' "${CONF_DIR}/ananke.yaml"
+    echo "[install] migrated default_budget_seconds 300 -> 1380 in ${CONF_DIR}/ananke.yaml"
+    changed=1
+  fi
+  if grep -Eq 'runtime_safety_factor:[[:space:]]*1\.10' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei 's/(runtime_safety_factor:[[:space:]]*)1\.10/\11.25/' "${CONF_DIR}/ananke.yaml"
+    echo "[install] migrated runtime_safety_factor 1.10 -> 1.25 in ${CONF_DIR}/ananke.yaml"
+    changed=1
+  fi
+  if grep -Eq '^ssh_node_users:[[:space:]]*$' "${CONF_DIR}/ananke.yaml" \
+    && grep -Eq '^  titan-24:[[:space:]]*tethys[[:space:]]*$' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei 's/^  titan-24:[[:space:]]*tethys[[:space:]]*$/  titan-24: atlas/' "${CONF_DIR}/ananke.yaml"
+    echo "[install] migrated ssh_node_users titan-24 override to atlas"
+    changed=1
+  fi
+  if grep -Eq '^  command_timeout_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml" \
+    && ! grep -Eq '^  startup_guard_max_age_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei '/^  command_timeout_seconds:[[:space:]]*[0-9]+/a\  startup_guard_max_age_seconds: 900' "${CONF_DIR}/ananke.yaml"
+    echo "[install] added coordination.startup_guard_max_age_seconds=900"
+    changed=1
+  fi
+  if grep -Eq '^[[:space:]]*poweroff_enabled:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei \
+      -e '/^[[:space:]]*poweroff_enabled:[[:space:]]*(true|false)/d' \
+      -e '/^[[:space:]]*poweroff_delay_seconds:[[:space:]]*[0-9]+/d' \
+      -e '/^[[:space:]]*poweroff_local_host:[[:space:]]*(true|false)/d' \
+      -e '/^[[:space:]]*extra_poweroff_hosts:[[:space:]]*(\[\])?[[:space:]]*$/d' \
+      "${CONF_DIR}/ananke.yaml"
+    echo "[install] removed deprecated host-poweroff shutdown config keys"
+    changed=1
+  fi
+  if grep -Eq '^  minimum_battery_percent:[[:space:]]*[0-9.]+' "${CONF_DIR}/ananke.yaml" \
+    && ! grep -Eq '^  require_node_inventory_reachability:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei '/^  minimum_battery_percent:[[:space:]]*[0-9.]+/a\  require_node_inventory_reachability: true\n  node_inventory_reachability_wait_seconds: 300\n  node_inventory_reachability_poll_seconds: 5' "${CONF_DIR}/ananke.yaml"
+    echo "[install] added startup node inventory reachability gate defaults"
+    changed=1
+  fi
+  if grep -Eq '^  required_node_labels:[[:space:]]*$' "${CONF_DIR}/ananke.yaml" \
+    && ! grep -Eq '^  longhorn_cryptsetup_exempt_nodes:' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei '/^  required_node_labels:[[:space:]]*$/i\  longhorn_cryptsetup_exempt_nodes:\n    - titan-23' "${CONF_DIR}/ananke.yaml"
+    echo "[install] added startup.longhorn_cryptsetup_exempt_nodes default (titan-23)"
+    changed=1
+  fi
+  if grep -Eq '^state:[[:space:]]*$' "${CONF_DIR}/ananke.yaml" \
+    && ! grep -Eq '^  reports_dir:[[:space:]]*/var/lib/ananke/reports' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei '/^  dir:[[:space:]]*\/var\/lib\/ananke$/a\  reports_dir: /var/lib/ananke/reports' "${CONF_DIR}/ananke.yaml"
+    echo "[install] added state.reports_dir default"
+    changed=1
+  fi
+  if ! grep -Eq '^  peer_hosts:' "${CONF_DIR}/ananke.yaml"; then
+    if [[ "${role_hint}" == "peer" ]] && grep -Eq '^  forward_shutdown_host:[[:space:]]*[A-Za-z0-9._-]+' "${CONF_DIR}/ananke.yaml"; then
+      local peer_host
+      peer_host="$(awk -F': *' '/^  forward_shutdown_host:[[:space:]]*/ {print $2; exit}' "${CONF_DIR}/ananke.yaml" 2>/dev/null || true)"
+      if [[ -n "${peer_host}" ]]; then
+        sed -Ei '/^  forward_shutdown_config:[[:space:]]*.*$/a\  peer_hosts:\n    - '"${peer_host}"'' "${CONF_DIR}/ananke.yaml"
+        echo "[install] added coordination.peer_hosts from forward_shutdown_host (${peer_host})"
+        changed=1
+      fi
+    elif [[ "${role_hint}" == "coordinator" ]] && grep -Eq '^  titan-24:[[:space:]]*[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' "${CONF_DIR}/ananke.yaml"; then
+      sed -Ei '/^  forward_shutdown_config:[[:space:]]*.*$/a\  peer_hosts:\n    - titan-24' "${CONF_DIR}/ananke.yaml"
+      echo "[install] added coordination.peer_hosts default (titan-24) for coordinator role"
+      changed=1
+    else
+      sed -Ei '/^  forward_shutdown_config:[[:space:]]*.*$/a\  peer_hosts: []' "${CONF_DIR}/ananke.yaml"
+      echo "[install] added coordination.peer_hosts empty default"
+      changed=1
+    fi
+  fi
+  local default_restore_cp
+  default_restore_cp="$(first_control_plane_name)"
+  if [[ -z "${default_restore_cp}" ]]; then
+    default_restore_cp="titan-0a"
+  fi
+  if grep -Eq '^  api_poll_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml" \
+    && ! grep -Eq '^  auto_etcd_restore_on_api_failure:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei '/^  api_poll_seconds:[[:space:]]*[0-9]+/a\  require_time_sync: true\n  time_sync_wait_seconds: 240\n  time_sync_poll_seconds: 5\n  reconcile_access_on_boot: true\n  auto_etcd_restore_on_api_failure: true\n  etcd_restore_control_plane: '"${default_restore_cp}"'' "${CONF_DIR}/ananke.yaml"
+    echo "[install] added startup.auto_etcd_restore_on_api_failure + startup.etcd_restore_control_plane defaults"
+    changed=1
+  fi
+  if grep -Eq '^  api_poll_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml" \
+    && ! grep -Eq '^  require_time_sync:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei '/^  api_poll_seconds:[[:space:]]*[0-9]+/a\  require_time_sync: true\n  time_sync_wait_seconds: 240\n  time_sync_poll_seconds: 5\n  reconcile_access_on_boot: true' "${CONF_DIR}/ananke.yaml"
+    echo "[install] added startup time sync + access reconciliation defaults"
+    changed=1
+  fi
+  if grep -Eq '^  time_sync_poll_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml" \
+    && ! grep -Eq '^  time_sync_mode:[[:space:]]*(strict|quorum)' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei '/^  time_sync_poll_seconds:[[:space:]]*[0-9]+/a\  time_sync_mode: quorum\n  time_sync_quorum: 2' "${CONF_DIR}/ananke.yaml"
+    echo "[install] added startup time sync quorum defaults"
+    changed=1
+  fi
+  if grep -Eq '^  etcd_restore_control_plane:[[:space:]]*[A-Za-z0-9._-]+' "${CONF_DIR}/ananke.yaml" \
+    && ! grep -Eq '^  require_storage_ready:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei '/^  etcd_restore_control_plane:[[:space:]]*[A-Za-z0-9._-]+/a\  require_storage_ready: true\n  storage_ready_wait_seconds: 420\n  storage_ready_poll_seconds: 5\n  storage_min_ready_nodes: 2\n  storage_critical_pvcs:\n    - vault/data-vault-0\n    - postgres/postgres-data-postgres-0\n    - gitea/gitea-data\n    - sso/keycloak-data' "${CONF_DIR}/ananke.yaml"
+    echo "[install] added startup storage readiness defaults"
+    changed=1
+  fi
+  if grep -Eq '^  storage_critical_pvcs:[[:space:]]*$' "${CONF_DIR}/ananke.yaml" \
+    && ! grep -Eq '^  require_post_start_probes:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei '/^    - sso\/keycloak-data$/a\  require_post_start_probes: true\n  post_start_probe_wait_seconds: 240\n  post_start_probe_poll_seconds: 5\n  post_start_probes:\n    - https://scm.bstein.dev/api/healthz\n    - https://metrics.bstein.dev/api/health\n  require_service_checklist: true\n  service_checklist_wait_seconds: 420\n  service_checklist_poll_seconds: 5\n  service_checklist_stability_seconds: 120\n  service_checklist:\n    - name: gitea-api\n      url: https://scm.bstein.dev/api/healthz\n      accepted_statuses: [200]\n      body_contains: pass\n      timeout_seconds: 12\n    - name: grafana-api\n      url: https://metrics.bstein.dev/api/health\n      accepted_statuses: [200]\n      body_contains: '\''\"database\":\"ok\"'\''\n      timeout_seconds: 12\n  vault_unseal_key_file: /var/lib/ananke/vault-unseal.key' "${CONF_DIR}/ananke.yaml"
+    echo "[install] added startup post-start probe + vault key fallback defaults"
+    changed=1
+  fi
+  if grep -Eq '^    - https://sso.bstein.dev/realms/atlas/.well-known/openid-configuration$' "${CONF_DIR}/ananke.yaml"; then
+    sed -Ei '/^    - https:\/\/sso\.bstein\.dev\/realms\/atlas\/\.well-known\/openid-configuration$/d' "${CONF_DIR}/ananke.yaml"
+    echo "[install] removed sso OIDC probe from startup.post_start_probes (returns 404 in current deployment)"
+    changed=1
+  fi
+  if ! grep -Eq '^  vault_unseal_key_file:[[:space:]]*/var/lib/ananke/vault-unseal.key' "${CONF_DIR}/ananke.yaml"; then
+    if grep -Eq '^startup:[[:space:]]*$' "${CONF_DIR}/ananke.yaml" && grep -Eq '^  post_start_probes:[[:space:]]*$' "${CONF_DIR}/ananke.yaml"; then
+      sed -Ei '/^    - https:\/\/metrics\.bstein\.dev\/api\/health$/a\  vault_unseal_key_file: /var/lib/ananke/vault-unseal.key' "${CONF_DIR}/ananke.yaml"
+      echo "[install] added startup.vault_unseal_key_file default"
+      changed=1
+    fi
+  fi
+  if ! grep -Eq '^  vault_unseal_breakglass_timeout_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml"; then
+    if grep -Eq '^  vault_unseal_key_file:[[:space:]]*/var/lib/ananke/vault-unseal.key' "${CONF_DIR}/ananke.yaml"; then
+      sed -Ei '/^  vault_unseal_key_file:[[:space:]]*\/var\/lib\/ananke\/vault-unseal.key$/a\  vault_unseal_breakglass_command: ""\n  vault_unseal_breakglass_timeout_seconds: 15' "${CONF_DIR}/ananke.yaml"
+      echo "[install] added startup break-glass fallback defaults"
+      changed=1
+    fi
+  fi
+
+  install_cluster_inventory_defaults "${role_hint}" && changed=1
+
+  if [[ "${changed}" -eq 1 ]]; then
+    chmod 0640 "${CONF_DIR}/ananke.yaml" || true
+  fi
+}
+
+install_cluster_inventory_defaults() {
+  local role="$1"
+  local changed=0
+  local inventory_block=""
+  local managed_block=""
+  local workers_block
+  workers_block='workers:
+  - titan-04
+  - titan-05
+  - titan-06
+  - titan-07
+  - titan-08
+  - titan-09
+  - titan-10
+  - titan-11
+  - titan-12
+  - titan-13
+  - titan-14
+  - titan-15
+  - titan-17
+  - titan-18
+  - titan-19
+  - titan-20
+  - titan-21
+  - titan-22
+  - titan-24'
+
+  if [[ "${role}" == "coordinator" || "${role}" == "peer" ]]; then
+    inventory_block='ssh_node_hosts:
+  titan-db: 192.168.22.10
+  titan-0a: 192.168.22.11
+  titan-0b: 192.168.22.12
+  titan-0c: 192.168.22.13
+  titan-04: 192.168.22.30
+  titan-05: 192.168.22.31
+  titan-06: 192.168.22.32
+  titan-07: 192.168.22.33
+  titan-08: 192.168.22.34
+  titan-09: 192.168.22.35
+  titan-10: 192.168.22.36
+  titan-11: 192.168.22.37
+  titan-12: 192.168.22.40
+  titan-13: 192.168.22.41
+  titan-14: 192.168.22.42
+  titan-15: 192.168.22.43
+  titan-17: 192.168.22.45
+  titan-18: 192.168.22.46
+  titan-19: 192.168.22.47
+  titan-20: 192.168.22.20
+  titan-21: 192.168.22.21
+  titan-22: 192.168.22.22
+  titan-24: 192.168.22.26'
+    managed_block='ssh_managed_nodes:
+  - titan-db
+  - titan-0a
+  - titan-0b
+  - titan-0c
+  - titan-04
+  - titan-05
+  - titan-06
+  - titan-07
+  - titan-08
+  - titan-09
+  - titan-10
+  - titan-11
+  - titan-12
+  - titan-13
+  - titan-14
+  - titan-15
+  - titan-17
+  - titan-18
+  - titan-19
+  - titan-20
+  - titan-21
+  - titan-22
+  - titan-24'
+  fi
+
+  if [[ -n "${inventory_block}" ]] && grep -Eq '^ssh_node_hosts:[[:space:]]*\{\}[[:space:]]*$' "${CONF_DIR}/ananke.yaml"; then
+    perl -0pi -e 's#ssh_node_hosts:\s*\{\}\n#'"${inventory_block}"'\n#s' "${CONF_DIR}/ananke.yaml"
+    echo "[install] hydrated ssh_node_hosts inventory for role=${role}"
+    changed=1
+  fi
+  if grep -Eq '^workers:[[:space:]]*\[\][[:space:]]*$' "${CONF_DIR}/ananke.yaml"; then
+    perl -0pi -e 's#workers:\s*\[\]\n#'"${workers_block}"'\n#s' "${CONF_DIR}/ananke.yaml"
+    echo "[install] hydrated workers inventory for startup/shutdown orchestration"
+    changed=1
+  fi
+
+  if [[ -n "${managed_block}" ]]; then
+    if grep -Eq '^ssh_managed_nodes:[[:space:]]*\[\][[:space:]]*$' "${CONF_DIR}/ananke.yaml"; then
+      perl -0pi -e 's#ssh_managed_nodes:\s*\[\]\n#'"${managed_block}"'\n#s' "${CONF_DIR}/ananke.yaml"
+      echo "[install] hydrated ssh_managed_nodes inventory for role=${role}"
+      changed=1
+    fi
+    if ! grep -Eq '^  - titan-04$' "${CONF_DIR}/ananke.yaml" || ! grep -Eq '^  - titan-21$' "${CONF_DIR}/ananke.yaml"; then
+      perl -0pi -e 's#ssh_managed_nodes:\n(?:  - [^\n]*\n)*#'"${managed_block}"'\n#s' "${CONF_DIR}/ananke.yaml"
+      echo "[install] refreshed ssh_managed_nodes coverage for role=${role}"
+      changed=1
+    fi
+  fi
+
+  if [[ "${role}" == "peer" ]]; then
+    install_peer_inventory_defaults && changed=1
+  fi
+
+  [[ "${changed}" -eq 1 ]]
+}
+
+install_peer_inventory_defaults() {
+  local changed=0
+  if grep -Eq '^ssh_managed_nodes:[[:space:]]*$' "${CONF_DIR}/ananke.yaml" \
+    && grep -Eq '^  - titan-db$' "${CONF_DIR}/ananke.yaml" \
+    && grep -Eq '^  - titan-24$' "${CONF_DIR}/ananke.yaml" \
+    && ! grep -Eq '^  - titan-0a$' "${CONF_DIR}/ananke.yaml"; then
+    perl -0pi -e 's#ssh_managed_nodes:\n  - titan-db\n  - titan-24\n#ssh_managed_nodes:\n  - titan-db\n  - titan-0a\n  - titan-0b\n  - titan-0c\n  - titan-04\n  - titan-05\n  - titan-06\n  - titan-07\n  - titan-08\n  - titan-09\n  - titan-10\n  - titan-11\n  - titan-12\n  - titan-13\n  - titan-14\n  - titan-15\n  - titan-17\n  - titan-18\n  - titan-19\n  - titan-20\n  - titan-21\n  - titan-22\n  - titan-24\n#s' "${CONF_DIR}/ananke.yaml"
+    echo "[install] expanded peer ssh_managed_nodes for bootstrap fallback coverage"
+    changed=1
+  fi
+
+  if ! grep -Eq '^  - services/keycloak$' "${CONF_DIR}/ananke.yaml" || ! grep -Eq '^  - infrastructure/cert-manager$' "${CONF_DIR}/ananke.yaml" || ! grep -Eq '^  - services/oauth2-proxy$' "${CONF_DIR}/ananke.yaml"; then
+    perl -0pi -e 's#local_bootstrap_paths:\n(?:  - [^\n]*\n)*#local_bootstrap_paths:\n  - infrastructure/core\n  - clusters/atlas/flux-system\n  - infrastructure/sources/helm\n  - infrastructure/metallb\n  - infrastructure/traefik\n  - infrastructure/cert-manager\n  - infrastructure/vault-csi\n  - infrastructure/vault-injector\n  - services/vault\n  - infrastructure/postgres\n  - services/gitea\n  - services/keycloak\n  - services/oauth2-proxy\n#s' "${CONF_DIR}/ananke.yaml"
+    echo "[install] refreshed peer local_bootstrap_paths for full fallback bootstrap parity"
+    changed=1
+  fi
+  [[ "${changed}" -eq 1 ]]
+}
+
+sanitize_migrated_ananke_config() {
+  local cfg="${CONF_DIR}/ananke.yaml"
+  [[ -f "${cfg}" ]] || return 0
+
+  local tmp changed=0
+  tmp="$(mktemp)"
+
+  # If a legacy migration bug appended root-level node entries after
+  # ssh_managed_nodes, drop those orphan entries until the next top-level key.
+  awk '
+    BEGIN {in_managed=0}
+    /^ssh_managed_nodes:[[:space:]]*$/ {in_managed=1; print; next}
+    {
+      if (in_managed) {
+        if ($0 ~ /^  - /) {print; next}
+        if ($0 ~ /^- /) {next}
+        if ($0 ~ /^[A-Za-z0-9_]+:[[:space:]]*/) {in_managed=0}
+      }
+      print
+    }
+  ' "${cfg}" > "${tmp}"
+
+  if ! cmp -s "${cfg}" "${tmp}"; then
+    mv "${tmp}" "${cfg}"
+    changed=1
+    echo "[install] sanitized malformed ssh_managed_nodes block in ${cfg}"
+  else
+    rm -f "${tmp}"
+  fi
+
+  if grep -Eq '^[[:space:]]*forward_shutdown_config:[[:space:]]*/etc/ananke/hecate.yaml[[:space:]]*$' "${cfg}"; then
+    sed -Ei 's#(^[[:space:]]*forward_shutdown_config:[[:space:]]*)/etc/ananke/hecate.yaml#\1/etc/ananke/ananke.yaml#' "${cfg}"
+    changed=1
+    echo "[install] migrated coordination.forward_shutdown_config to /etc/ananke/ananke.yaml"
+  fi
+
+  if [[ "${changed}" -eq 1 ]]; then
+    chmod 0640 "${cfg}" || true
+  fi
+}
+
+install_inventory_fragment() {
+  if [[ -z "${INVENTORY_FRAGMENT:-}" ]]; then
+    return 0
+  fi
+  if [[ ! -f "${INVENTORY_FRAGMENT}" ]]; then
+    echo "[install] ANANKE_INVENTORY_FRAGMENT does not exist: ${INVENTORY_FRAGMENT}" >&2
+    exit 1
+  fi
+  install -m 0640 "${INVENTORY_FRAGMENT}" "${CONF_DIR}/ananke.inventory.yaml"
+  echo "[install] installed Terraform-generated inventory fragment to ${CONF_DIR}/ananke.inventory.yaml"
+}

scripts/install-host-bootstrap.sh

@@ -0,0 +1,239 @@
+# Host bootstrap helpers for the Ananke installer.
+
+resolve_nut_ups_name() {
+  if [[ -n "${NUT_UPS_NAME}" ]]; then
+    return 0
+  fi
+
+  if [[ -f "${CONF_DIR}/ananke.yaml" ]]; then
+    local target=""
+    target="$(grep -Eo 'target:[[:space:]]*[A-Za-z0-9._-]+@localhost' "${CONF_DIR}/ananke.yaml" | head -n 1 | awk '{print $2}')"
+    if [[ -n "${target}" ]]; then
+      NUT_UPS_NAME="${target%@localhost}"
+      echo "[install] inferred NUT UPS name from config: ${NUT_UPS_NAME}"
+      return 0
+    fi
+  fi
+
+  NUT_UPS_NAME="pyrphoros"
+  echo "[install] defaulting NUT UPS name to ${NUT_UPS_NAME}"
+}
+
+ensure_ananke_kubeconfig() {
+  local kubeconfig_path
+  kubeconfig_path="$(migration_yaml_lookup "kubeconfig")"
+  if [[ -z "${kubeconfig_path}" ]]; then
+    kubeconfig_path="/etc/ananke/kubeconfig"
+  fi
+  install -d -m 0750 "$(dirname "${kubeconfig_path}")"
+
+  if [[ -s "${kubeconfig_path}" ]] && KUBECONFIG="${kubeconfig_path}" kubectl version --request-timeout=5s >/dev/null 2>&1; then
+    return 0
+  fi
+
+  if [[ -r /etc/rancher/k3s/k3s.yaml ]]; then
+    install -m 0600 /etc/rancher/k3s/k3s.yaml "${kubeconfig_path}"
+    echo "[install] refreshed kubeconfig from local /etc/rancher/k3s/k3s.yaml"
+    if KUBECONFIG="${kubeconfig_path}" kubectl version --request-timeout=5s >/dev/null 2>&1; then
+      return 0
+    fi
+  fi
+
+  local cp_name cp_host ssh_user ssh_port ssh_cfg ssh_key
+  cp_name="$(first_control_plane_name)"
+  if [[ -z "${cp_name}" ]]; then
+    echo "[install] warning: cannot infer control plane name; kubeconfig bootstrap skipped"
+    return 0
+  fi
+  cp_host="$(lookup_node_host "${cp_name}")"
+  if [[ -z "${cp_host}" ]]; then
+    cp_host="${cp_name}"
+  fi
+  ssh_user="$(migration_yaml_lookup "ssh_user")"
+  ssh_port="$(migration_yaml_lookup "ssh_port")"
+  ssh_cfg="$(migration_yaml_lookup "ssh_config_file")"
+  ssh_key="$(migration_yaml_lookup "ssh_identity_file")"
+  if [[ -z "${ssh_port}" ]]; then
+    ssh_port="2277"
+  fi
+
+  local target
+  target="${cp_host}"
+  if [[ -n "${ssh_user}" ]]; then
+    target="${ssh_user}@${cp_host}"
+  fi
+  local ssh_args=(
+    -o BatchMode=yes
+    -o ConnectTimeout=8
+    -o StrictHostKeyChecking=accept-new
+  )
+  if [[ -n "${ssh_cfg}" && -f "${ssh_cfg}" ]]; then
+    ssh_args+=(-F "${ssh_cfg}")
+  fi
+  if [[ -n "${ssh_key}" && -f "${ssh_key}" ]]; then
+    ssh_args+=(-i "${ssh_key}")
+  fi
+  if [[ -n "${ssh_port}" ]]; then
+    ssh_args+=(-p "${ssh_port}")
+  fi
+
+  local remote_cfg
+  if remote_cfg="$(ssh "${ssh_args[@]}" "${target}" "sudo cat /etc/rancher/k3s/k3s.yaml" 2>/dev/null)"; then
+    printf '%s\n' "${remote_cfg}" > "${kubeconfig_path}"
+    sed -Ei "s#server:[[:space:]]*https://127\\.0\\.0\\.1:6443#server: https://${cp_host}:6443#g" "${kubeconfig_path}" || true
+    chmod 0600 "${kubeconfig_path}"
+    echo "[install] bootstrapped kubeconfig from control plane ${cp_name} (${cp_host})"
+    if KUBECONFIG="${kubeconfig_path}" kubectl version --request-timeout=5s >/dev/null 2>&1; then
+      return 0
+    fi
+  else
+    echo "[install] warning: failed to fetch kubeconfig from ${cp_name} (${cp_host})"
+  fi
+
+  echo "[install] warning: kubeconfig at ${kubeconfig_path} is still not validated; local startup fallback may fail"
+}
+
+ensure_ananke_ssh_identity() {
+  local key_path key_dir key_user key_comment
+  key_path="$(migration_yaml_lookup "ssh_identity_file")"
+  if [[ -z "${key_path}" ]]; then
+    key_path="/home/atlas/.ssh/id_ed25519"
+  fi
+  key_dir="$(dirname "${key_path}")"
+  key_comment="ananke-$(hostname)-forward"
+
+  key_user="root"
+  if [[ "${key_path}" == /home/*/* ]]; then
+    key_user="${key_path#/home/}"
+    key_user="${key_user%%/*}"
+  fi
+
+  if ! id "${key_user}" >/dev/null 2>&1; then
+    echo "[install] warning: ssh identity owner ${key_user} does not exist; skipping key bootstrap for ${key_path}"
+    return 0
+  fi
+
+  install -d -m 0700 -o "${key_user}" -g "${key_user}" "${key_dir}"
+  if [[ ! -s "${key_path}" ]]; then
+    echo "[install] generating missing SSH identity at ${key_path}"
+    if [[ "${key_user}" == "root" ]]; then
+      ssh-keygen -q -t ed25519 -N '' -C "${key_comment}" -f "${key_path}"
+    else
+      runuser -u "${key_user}" -- ssh-keygen -q -t ed25519 -N '' -C "${key_comment}" -f "${key_path}"
+    fi
+  fi
+  chown "${key_user}:${key_user}" "${key_path}" "${key_path}.pub" 2>/dev/null || true
+  chmod 0600 "${key_path}" || true
+  chmod 0644 "${key_path}.pub" || true
+}
+
+ensure_apt_packages() {
+  local missing=()
+  for pkg in "$@"; do
+    if ! dpkg -s "${pkg}" >/dev/null 2>&1; then
+      missing+=("${pkg}")
+    fi
+  done
+  if [[ ${#missing[@]} -eq 0 ]]; then
+    return 0
+  fi
+  echo "[install] apt install: ${missing[*]}"
+  export DEBIAN_FRONTEND=noninteractive
+  apt-get update -y
+  apt-get install -y "${missing[@]}"
+}
+
+install_kubectl_if_missing() {
+  if command -v kubectl >/dev/null 2>&1; then
+    return 0
+  fi
+  ensure_apt_packages kubernetes-client || true
+  if command -v kubectl >/dev/null 2>&1; then
+    return 0
+  fi
+  echo "[install] installing kubectl via upstream binary"
+  local arch
+  arch="$(uname -m)"
+  case "${arch}" in
+    x86_64) arch="amd64" ;;
+    aarch64|arm64) arch="arm64" ;;
+    *) echo "Unsupported arch for kubectl install: ${arch}" >&2; return 1 ;;
+  esac
+  local version
+  version="$(curl -fsSL https://dl.k8s.io/release/stable.txt)"
+  curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/${version}/bin/linux/${arch}/kubectl"
+  chmod 0755 /usr/local/bin/kubectl
+}
+
+ensure_dependencies() {
+  if [[ "${INSTALL_DEPS}" -eq 0 ]]; then
+    echo "[install] skipping dependency installation"
+    return 0
+  fi
+  if ! command -v apt-get >/dev/null 2>&1; then
+    echo "This installer currently supports apt-based hosts only." >&2
+    exit 1
+  fi
+  ensure_apt_packages ca-certificates curl git openssh-client jq nut-client nut-server nut-monitor golang-go
+  install_kubectl_if_missing
+}
+
+configure_nut() {
+  if [[ "${MANAGE_NUT}" != "1" ]]; then
+    echo "[install] skipping NUT configuration (ANANKE_MANAGE_NUT=${MANAGE_NUT})"
+    return 0
+  fi
+
+  echo "[install] configuring NUT + udev for UPS ${NUT_UPS_NAME} (${NUT_VENDOR_ID}:${NUT_PRODUCT_ID})"
+  install -d -m 0755 /etc/nut /etc/udev/rules.d
+
+  cat > /etc/nut/nut.conf <<EOF
+MODE=standalone
+EOF
+
+  cat > /etc/nut/ups.conf <<EOF
+[${NUT_UPS_NAME}]
+  driver = usbhid-ups
+  port = auto
+  vendorid = ${NUT_VENDOR_ID}
+  productid = ${NUT_PRODUCT_ID}
+  pollinterval = 5
+EOF
+
+  cat > /etc/nut/upsd.users <<EOF
+[${NUT_MONITOR_USER}]
+  password = ${NUT_MONITOR_PASSWORD}
+  upsmon primary
+EOF
+  chmod 0640 /etc/nut/upsd.users
+  if getent group nut >/dev/null 2>&1; then
+    chown root:nut /etc/nut/upsd.users
+  else
+    chown root:root /etc/nut/upsd.users
+  fi
+
+  cat > /etc/nut/upsmon.conf <<EOF
+RUN_AS_USER nut
+MONITOR ${NUT_UPS_NAME}@localhost 1 ${NUT_MONITOR_USER} ${NUT_MONITOR_PASSWORD} primary
+MINSUPPLIES 1
+SHUTDOWNCMD "/sbin/shutdown -h +0"
+POLLFREQ 5
+POLLFREQALERT 5
+HOSTSYNC 15
+DEADTIME 15
+POWERDOWNFLAG /etc/killpower
+EOF
+
+  cat > /etc/udev/rules.d/99-ananke-ups.rules <<EOF
+# Managed by ananke install.sh: ensure UPS USB HID devices are readable by NUT
+ACTION=="add|change", SUBSYSTEM=="usb", ATTR{idVendor}=="${NUT_VENDOR_ID}", ATTR{idProduct}=="${NUT_PRODUCT_ID}", MODE:="0660", GROUP:="nut"
+EOF
+
+  udevadm control --reload-rules || true
+  udevadm trigger --subsystem-match=usb --attr-match=idVendor="${NUT_VENDOR_ID}" --attr-match=idProduct="${NUT_PRODUCT_ID}" || true
+
+  systemctl enable nut-driver-enumerator.service nut-server.service nut-monitor.service >/dev/null 2>&1 || true
+  systemctl restart nut-driver-enumerator.service >/dev/null 2>&1 || true
+  systemctl restart "nut-driver@${NUT_UPS_NAME}.service" >/dev/null 2>&1 || true
+  systemctl restart nut-server.service nut-monitor.service >/dev/null 2>&1 || true
+}

scripts/install-legacy-migration.sh

@@ -0,0 +1,98 @@
+# Legacy Hecate migration helpers for the Ananke installer.
+
+legacy_path_rewrite() {
+  local src="$1"
+  local dst="$2"
+  sed \
+    -e 's#/etc/hecate/hecate.yaml#/etc/ananke/ananke.yaml#g' \
+    -e 's#/etc/hecate/kubeconfig#/etc/ananke/kubeconfig#g' \
+    -e 's#/var/lib/hecate/vault-unseal.key#/var/lib/ananke/vault-unseal.key#g' \
+    -e 's#/var/lib/hecate/hecate.lock#/var/lib/ananke/ananke.lock#g' \
+    -e 's#/opt/hecate#/opt/ananke#g' \
+    -e 's#/etc/hecate#/etc/ananke#g' \
+    -e 's#/var/lib/hecate#/var/lib/ananke#g' \
+    -e 's#/usr/local/bin/hecate#/usr/local/bin/ananke#g' \
+    -e 's#/usr/local/lib/hecate#/usr/local/lib/ananke#g' \
+    -e 's/hecate.yaml/ananke.yaml/g' \
+    -e 's/hecate.lock/ananke.lock/g' \
+    -e 's/hecate/ananke/g' \
+    -e 's/Hecate/Ananke/g' \
+    -e 's#hecate\.lock#ananke.lock#g' \
+    "${src}" > "${dst}"
+}
+
+migrate_legacy_hecate_install() {
+  local legacy_conf_dir="/etc/hecate"
+  local legacy_state_dir="/var/lib/hecate"
+  local legacy_systemd_dir="/etc/systemd/system"
+
+  install -d -m 0750 "${CONF_DIR}"
+  install -d -m 0750 "${STATE_DIR}"
+
+  if [[ ! -f "${CONF_DIR}/ananke.yaml" && -f "${legacy_conf_dir}/hecate.yaml" ]]; then
+    echo "[install] migrating legacy config ${legacy_conf_dir}/hecate.yaml -> ${CONF_DIR}/ananke.yaml"
+    legacy_path_rewrite "${legacy_conf_dir}/hecate.yaml" "${CONF_DIR}/ananke.yaml"
+    chmod 0640 "${CONF_DIR}/ananke.yaml"
+  fi
+
+  if [[ ! -f "${CONF_DIR}/kubeconfig" && -f "${legacy_conf_dir}/kubeconfig" ]]; then
+    echo "[install] migrating legacy kubeconfig ${legacy_conf_dir}/kubeconfig -> ${CONF_DIR}/kubeconfig"
+    install -m 0600 "${legacy_conf_dir}/kubeconfig" "${CONF_DIR}/kubeconfig"
+  fi
+
+  if [[ ! -f "${STATE_DIR}/vault-unseal.key" && -f "${legacy_state_dir}/vault-unseal.key" ]]; then
+    echo "[install] migrating legacy vault key ${legacy_state_dir}/vault-unseal.key -> ${STATE_DIR}/vault-unseal.key"
+    install -m 0600 "${legacy_state_dir}/vault-unseal.key" "${STATE_DIR}/vault-unseal.key"
+  fi
+
+  if [[ ! -f "${STATE_DIR}/runs.json" && -f "${legacy_state_dir}/runs.json" ]]; then
+    echo "[install] migrating legacy run history ${legacy_state_dir}/runs.json -> ${STATE_DIR}/runs.json"
+    install -m 0640 "${legacy_state_dir}/runs.json" "${STATE_DIR}/runs.json"
+  fi
+
+  if [[ ! -f "${STATE_DIR}/intent.json" && -f "${legacy_state_dir}/intent.json" ]]; then
+    echo "[install] migrating legacy intent state ${legacy_state_dir}/intent.json -> ${STATE_DIR}/intent.json"
+    install -m 0640 "${legacy_state_dir}/intent.json" "${STATE_DIR}/intent.json"
+  fi
+
+  if [[ ! -f "${STATE_DIR}/ananke.lock" && -f "${legacy_state_dir}/hecate.lock" ]]; then
+    echo "[install] migrating legacy lock ${legacy_state_dir}/hecate.lock -> ${STATE_DIR}/ananke.lock"
+    install -m 0640 "${legacy_state_dir}/hecate.lock" "${STATE_DIR}/ananke.lock"
+  fi
+
+  if [[ -d "${legacy_systemd_dir}" ]]; then
+    if ls "${legacy_systemd_dir}"/hecate*.service >/dev/null 2>&1 || ls "${legacy_systemd_dir}"/hecate*.timer >/dev/null 2>&1; then
+      echo "[install] detected legacy hecate systemd unit files; will retire after ananke install"
+    fi
+  fi
+}
+
+retire_legacy_hecate_install() {
+  local ts backup_dir
+  ts="$(date +%Y%m%d%H%M%S)"
+  backup_dir="/var/backups/ananke-legacy-hecate-${ts}"
+
+  systemctl disable --now hecate.service hecate-bootstrap.service hecate-update.timer >/dev/null 2>&1 || true
+  systemctl stop hecate-update.service >/dev/null 2>&1 || true
+
+  if [[ -d /etc/hecate || -d /var/lib/hecate || -d /usr/local/lib/hecate || -d /opt/hecate ]]; then
+    install -d -m 0750 "${backup_dir}"
+    [[ -d /etc/hecate ]] && cp -a /etc/hecate "${backup_dir}/" || true
+    [[ -d /var/lib/hecate ]] && cp -a /var/lib/hecate "${backup_dir}/" || true
+    [[ -d /usr/local/lib/hecate ]] && cp -a /usr/local/lib/hecate "${backup_dir}/" || true
+    [[ -d /opt/hecate ]] && cp -a /opt/hecate "${backup_dir}/" || true
+    [[ -f /usr/local/bin/hecate ]] && install -m 0755 /usr/local/bin/hecate "${backup_dir}/hecate.bin" || true
+    echo "[install] backed up legacy hecate assets to ${backup_dir}"
+  fi
+
+  rm -f \
+    /etc/systemd/system/hecate.service \
+    /etc/systemd/system/hecate-bootstrap.service \
+    /etc/systemd/system/hecate-update.service \
+    /etc/systemd/system/hecate-update.timer
+  rm -f /usr/local/bin/hecate
+  rm -rf /usr/local/lib/hecate
+  rm -rf /opt/hecate
+  rm -rf /etc/hecate
+  rm -rf /var/lib/hecate
+}

scripts/install.sh

@@ -14,6 +14,7 @@ SYSTEMD_DIR="/etc/systemd/system"
 LIB_DIR="/usr/local/lib/ananke"
 START_NOW=1
 INSTALL_DEPS=1
+BINARY_ONLY=0
 ENABLE_BOOTSTRAP="${ANANKE_ENABLE_BOOTSTRAP:-auto}"
 MANAGE_NUT="${ANANKE_MANAGE_NUT:-1}"
 NUT_UPS_NAME="${ANANKE_NUT_UPS_NAME:-}"
@@ -22,10 +23,15 @@ NUT_PRODUCT_ID="${ANANKE_NUT_PRODUCT_ID:-0601}"
 NUT_MONITOR_USER="${ANANKE_NUT_MONITOR_USER:-monuser}"
 NUT_MONITOR_PASSWORD="${ANANKE_NUT_MONITOR_PASSWORD:-anankeupsmon}"
 FORCE_CONFIG_TEMPLATE="${ANANKE_FORCE_CONFIG_TEMPLATE:-}"
+INVENTORY_FRAGMENT="${ANANKE_INVENTORY_FRAGMENT:-}"
 ENFORCE_QUALITY_GATE="${ANANKE_ENFORCE_QUALITY_GATE:-1}"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
+    --binary-only)
+      BINARY_ONLY=1
+      shift
+      ;;
     --no-start)
       START_NOW=0
       shift
@@ -41,832 +47,19 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-resolve_nut_ups_name() {
+source "${REPO_DIR}/scripts/install-config-migration.sh"
-  if [[ -n "${NUT_UPS_NAME}" ]]; then
+source "${REPO_DIR}/scripts/install-host-bootstrap.sh"
-    return 0
+source "${REPO_DIR}/scripts/install-legacy-migration.sh"
-  fi
+source "${REPO_DIR}/scripts/install-artifacts.sh"
-
+
-  if [[ -f "${CONF_DIR}/ananke.yaml" ]]; then
+if [[ "${BINARY_ONLY}" == "1" && ! -f "${BIN_DIR}/ananke" ]]; then
-    local target=""
+  echo "[install] binary-only requires an existing installation" >&2
-    target="$(grep -Eo 'target:[[:space:]]*[A-Za-z0-9._-]+@localhost' "${CONF_DIR}/ananke.yaml" | head -n 1 | awk '{print $2}')"
+  exit 1
-    if [[ -n "${target}" ]]; then
+fi
-      NUT_UPS_NAME="${target%@localhost}"
-      echo "[install] inferred NUT UPS name from config: ${NUT_UPS_NAME}"
-      return 0
-    fi
-  fi
-
-  NUT_UPS_NAME="pyrphoros"
-  echo "[install] defaulting NUT UPS name to ${NUT_UPS_NAME}"
-}
-
-read_ananke_role() {
-  if [[ ! -f "${CONF_DIR}/ananke.yaml" ]]; then
-    echo "coordinator"
-    return 0
-  fi
-  local role
-  role="$(awk '/^[[:space:]]*role:[[:space:]]*/ {print $2; exit}' "${CONF_DIR}/ananke.yaml" 2>/dev/null || true)"
-  if [[ -z "${role}" ]]; then
-    role="coordinator"
-  fi
-  echo "${role}"
-}
-
-migration_yaml_lookup() {
-  local key="$1"
-  awk -F': *' -v k="${key}" '$1 == k {print $2; exit}' "${CONF_DIR}/ananke.yaml" 2>/dev/null || true
-}
-
-first_control_plane_name() {
-  awk '
-    /^control_planes:[[:space:]]*$/ {in_list=1; next}
-    in_list && /^[[:space:]]*-[[:space:]]*/ {gsub(/^[[:space:]]*-[[:space:]]*/, "", $0); print $0; exit}
-    in_list && /^[^[:space:]]/ {in_list=0}
-  ' "${CONF_DIR}/ananke.yaml" 2>/dev/null || true
-}
-
-lookup_node_host() {
-  local node="$1"
-  awk -F': *' -v n="${node}" '$1 == "  " n {print $2; exit}' "${CONF_DIR}/ananke.yaml" 2>/dev/null || true
-}
-
-ensure_ananke_kubeconfig() {
-  local kubeconfig_path
-  kubeconfig_path="$(migration_yaml_lookup "kubeconfig")"
-  if [[ -z "${kubeconfig_path}" ]]; then
-    kubeconfig_path="/etc/ananke/kubeconfig"
-  fi
-  install -d -m 0750 "$(dirname "${kubeconfig_path}")"
-
-  if [[ -s "${kubeconfig_path}" ]] && KUBECONFIG="${kubeconfig_path}" kubectl version --request-timeout=5s >/dev/null 2>&1; then
-    return 0
-  fi
-
-  if [[ -r /etc/rancher/k3s/k3s.yaml ]]; then
-    install -m 0600 /etc/rancher/k3s/k3s.yaml "${kubeconfig_path}"
-    echo "[install] refreshed kubeconfig from local /etc/rancher/k3s/k3s.yaml"
-    if KUBECONFIG="${kubeconfig_path}" kubectl version --request-timeout=5s >/dev/null 2>&1; then
-      return 0
-    fi
-  fi
-
-  local cp_name cp_host ssh_user ssh_port ssh_cfg ssh_key
-  cp_name="$(first_control_plane_name)"
-  if [[ -z "${cp_name}" ]]; then
-    echo "[install] warning: cannot infer control plane name; kubeconfig bootstrap skipped"
-    return 0
-  fi
-  cp_host="$(lookup_node_host "${cp_name}")"
-  if [[ -z "${cp_host}" ]]; then
-    cp_host="${cp_name}"
-  fi
-  ssh_user="$(migration_yaml_lookup "ssh_user")"
-  ssh_port="$(migration_yaml_lookup "ssh_port")"
-  ssh_cfg="$(migration_yaml_lookup "ssh_config_file")"
-  ssh_key="$(migration_yaml_lookup "ssh_identity_file")"
-  if [[ -z "${ssh_port}" ]]; then
-    ssh_port="2277"
-  fi
-
-  local target
-  target="${cp_host}"
-  if [[ -n "${ssh_user}" ]]; then
-    target="${ssh_user}@${cp_host}"
-  fi
-  local ssh_args=(
-    -o BatchMode=yes
-    -o ConnectTimeout=8
-    -o StrictHostKeyChecking=accept-new
-  )
-  if [[ -n "${ssh_cfg}" && -f "${ssh_cfg}" ]]; then
-    ssh_args+=(-F "${ssh_cfg}")
-  fi
-  if [[ -n "${ssh_key}" && -f "${ssh_key}" ]]; then
-    ssh_args+=(-i "${ssh_key}")
-  fi
-  if [[ -n "${ssh_port}" ]]; then
-    ssh_args+=(-p "${ssh_port}")
-  fi
-
-  local remote_cfg
-  if remote_cfg="$(ssh "${ssh_args[@]}" "${target}" "sudo cat /etc/rancher/k3s/k3s.yaml" 2>/dev/null)"; then
-    printf '%s\n' "${remote_cfg}" > "${kubeconfig_path}"
-    sed -Ei "s#server:[[:space:]]*https://127\\.0\\.0\\.1:6443#server: https://${cp_host}:6443#g" "${kubeconfig_path}" || true
-    chmod 0600 "${kubeconfig_path}"
-    echo "[install] bootstrapped kubeconfig from control plane ${cp_name} (${cp_host})"
-    if KUBECONFIG="${kubeconfig_path}" kubectl version --request-timeout=5s >/dev/null 2>&1; then
-      return 0
-    fi
-  else
-    echo "[install] warning: failed to fetch kubeconfig from ${cp_name} (${cp_host})"
-  fi
-
-  echo "[install] warning: kubeconfig at ${kubeconfig_path} is still not validated; local startup fallback may fail"
-}
-
-ensure_ananke_ssh_identity() {
-  local key_path key_dir key_user key_comment
-  key_path="$(migration_yaml_lookup "ssh_identity_file")"
-  if [[ -z "${key_path}" ]]; then
-    key_path="/home/atlas/.ssh/id_ed25519"
-  fi
-  key_dir="$(dirname "${key_path}")"
-  key_comment="ananke-$(hostname)-forward"
-
-  key_user="root"
-  if [[ "${key_path}" == /home/*/* ]]; then
-    key_user="${key_path#/home/}"
-    key_user="${key_user%%/*}"
-  fi
-
-  if ! id "${key_user}" >/dev/null 2>&1; then
-    echo "[install] warning: ssh identity owner ${key_user} does not exist; skipping key bootstrap for ${key_path}"
-    return 0
-  fi
-
-  install -d -m 0700 -o "${key_user}" -g "${key_user}" "${key_dir}"
-  if [[ ! -s "${key_path}" ]]; then
-    echo "[install] generating missing SSH identity at ${key_path}"
-    if [[ "${key_user}" == "root" ]]; then
-      ssh-keygen -q -t ed25519 -N '' -C "${key_comment}" -f "${key_path}"
-    else
-      runuser -u "${key_user}" -- ssh-keygen -q -t ed25519 -N '' -C "${key_comment}" -f "${key_path}"
-    fi
-  fi
-  chown "${key_user}:${key_user}" "${key_path}" "${key_path}.pub" 2>/dev/null || true
-  chmod 0600 "${key_path}" || true
-  chmod 0644 "${key_path}.pub" || true
-}
-
-migrate_ananke_config() {
-  if [[ ! -f "${CONF_DIR}/ananke.yaml" ]]; then
-    return 0
-  fi
-
-  local changed=0
-  local role_hint
-  role_hint="$(read_ananke_role)"
-  if grep -Eq 'default_budget_seconds:[[:space:]]*300' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei 's/(default_budget_seconds:[[:space:]]*)300/\11380/' "${CONF_DIR}/ananke.yaml"
-    echo "[install] migrated default_budget_seconds 300 -> 1380 in ${CONF_DIR}/ananke.yaml"
-    changed=1
-  fi
-  if grep -Eq 'runtime_safety_factor:[[:space:]]*1\.10' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei 's/(runtime_safety_factor:[[:space:]]*)1\.10/\11.25/' "${CONF_DIR}/ananke.yaml"
-    echo "[install] migrated runtime_safety_factor 1.10 -> 1.25 in ${CONF_DIR}/ananke.yaml"
-    changed=1
-  fi
-  if grep -Eq '^ssh_node_users:[[:space:]]*$' "${CONF_DIR}/ananke.yaml" \
-    && grep -Eq '^  titan-24:[[:space:]]*tethys[[:space:]]*$' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei 's/^  titan-24:[[:space:]]*tethys[[:space:]]*$/  titan-24: atlas/' "${CONF_DIR}/ananke.yaml"
-    echo "[install] migrated ssh_node_users titan-24 override to atlas"
-    changed=1
-  fi
-  if grep -Eq '^  command_timeout_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml" \
-    && ! grep -Eq '^  startup_guard_max_age_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei '/^  command_timeout_seconds:[[:space:]]*[0-9]+/a\  startup_guard_max_age_seconds: 900' "${CONF_DIR}/ananke.yaml"
-    echo "[install] added coordination.startup_guard_max_age_seconds=900"
-    changed=1
-  fi
-  if grep -Eq '^[[:space:]]*poweroff_enabled:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei \
-      -e '/^[[:space:]]*poweroff_enabled:[[:space:]]*(true|false)/d' \
-      -e '/^[[:space:]]*poweroff_delay_seconds:[[:space:]]*[0-9]+/d' \
-      -e '/^[[:space:]]*poweroff_local_host:[[:space:]]*(true|false)/d' \
-      -e '/^[[:space:]]*extra_poweroff_hosts:[[:space:]]*(\[\])?[[:space:]]*$/d' \
-      "${CONF_DIR}/ananke.yaml"
-    echo "[install] removed deprecated host-poweroff shutdown config keys"
-    changed=1
-  fi
-  if grep -Eq '^  minimum_battery_percent:[[:space:]]*[0-9.]+' "${CONF_DIR}/ananke.yaml" \
-    && ! grep -Eq '^  require_node_inventory_reachability:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei '/^  minimum_battery_percent:[[:space:]]*[0-9.]+/a\  require_node_inventory_reachability: true\n  node_inventory_reachability_wait_seconds: 300\n  node_inventory_reachability_poll_seconds: 5' "${CONF_DIR}/ananke.yaml"
-    echo "[install] added startup node inventory reachability gate defaults"
-    changed=1
-  fi
-  if grep -Eq '^state:[[:space:]]*$' "${CONF_DIR}/ananke.yaml" \
-    && ! grep -Eq '^  reports_dir:[[:space:]]*/var/lib/ananke/reports' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei '/^  dir:[[:space:]]*\/var\/lib\/ananke$/a\  reports_dir: /var/lib/ananke/reports' "${CONF_DIR}/ananke.yaml"
-    echo "[install] added state.reports_dir default"
-    changed=1
-  fi
-  if ! grep -Eq '^  peer_hosts:' "${CONF_DIR}/ananke.yaml"; then
-    if [[ "${role_hint}" == "peer" ]] && grep -Eq '^  forward_shutdown_host:[[:space:]]*[A-Za-z0-9._-]+' "${CONF_DIR}/ananke.yaml"; then
-      local peer_host
-      peer_host="$(awk -F': *' '/^  forward_shutdown_host:[[:space:]]*/ {print $2; exit}' "${CONF_DIR}/ananke.yaml" 2>/dev/null || true)"
-      if [[ -n "${peer_host}" ]]; then
-        sed -Ei '/^  forward_shutdown_config:[[:space:]]*.*$/a\  peer_hosts:\n    - '"${peer_host}"'' "${CONF_DIR}/ananke.yaml"
-        echo "[install] added coordination.peer_hosts from forward_shutdown_host (${peer_host})"
-        changed=1
-      fi
-    elif [[ "${role_hint}" == "coordinator" ]] && grep -Eq '^  titan-24:[[:space:]]*[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' "${CONF_DIR}/ananke.yaml"; then
-      sed -Ei '/^  forward_shutdown_config:[[:space:]]*.*$/a\  peer_hosts:\n    - titan-24' "${CONF_DIR}/ananke.yaml"
-      echo "[install] added coordination.peer_hosts default (titan-24) for coordinator role"
-      changed=1
-    else
-      sed -Ei '/^  forward_shutdown_config:[[:space:]]*.*$/a\  peer_hosts: []' "${CONF_DIR}/ananke.yaml"
-      echo "[install] added coordination.peer_hosts empty default"
-      changed=1
-    fi
-  fi
-  local default_restore_cp
-  default_restore_cp="$(first_control_plane_name)"
-  if [[ -z "${default_restore_cp}" ]]; then
-    default_restore_cp="titan-0a"
-  fi
-  if grep -Eq '^  api_poll_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml" \
-    && ! grep -Eq '^  auto_etcd_restore_on_api_failure:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei '/^  api_poll_seconds:[[:space:]]*[0-9]+/a\  require_time_sync: true\n  time_sync_wait_seconds: 240\n  time_sync_poll_seconds: 5\n  reconcile_access_on_boot: true\n  auto_etcd_restore_on_api_failure: true\n  etcd_restore_control_plane: '"${default_restore_cp}"'' "${CONF_DIR}/ananke.yaml"
-    echo "[install] added startup.auto_etcd_restore_on_api_failure + startup.etcd_restore_control_plane defaults"
-    changed=1
-  fi
-  if grep -Eq '^  api_poll_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml" \
-    && ! grep -Eq '^  require_time_sync:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei '/^  api_poll_seconds:[[:space:]]*[0-9]+/a\  require_time_sync: true\n  time_sync_wait_seconds: 240\n  time_sync_poll_seconds: 5\n  reconcile_access_on_boot: true' "${CONF_DIR}/ananke.yaml"
-    echo "[install] added startup time sync + access reconciliation defaults"
-    changed=1
-  fi
-  if grep -Eq '^  time_sync_poll_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml" \
-    && ! grep -Eq '^  time_sync_mode:[[:space:]]*(strict|quorum)' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei '/^  time_sync_poll_seconds:[[:space:]]*[0-9]+/a\  time_sync_mode: quorum\n  time_sync_quorum: 2' "${CONF_DIR}/ananke.yaml"
-    echo "[install] added startup time sync quorum defaults"
-    changed=1
-  fi
-  if grep -Eq '^  etcd_restore_control_plane:[[:space:]]*[A-Za-z0-9._-]+' "${CONF_DIR}/ananke.yaml" \
-    && ! grep -Eq '^  require_storage_ready:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei '/^  etcd_restore_control_plane:[[:space:]]*[A-Za-z0-9._-]+/a\  require_storage_ready: true\n  storage_ready_wait_seconds: 420\n  storage_ready_poll_seconds: 5\n  storage_min_ready_nodes: 2\n  storage_critical_pvcs:\n    - vault/data-vault-0\n    - postgres/postgres-data-postgres-0\n    - gitea/gitea-data\n    - sso/keycloak-data' "${CONF_DIR}/ananke.yaml"
-    echo "[install] added startup storage readiness defaults"
-    changed=1
-  fi
-  if grep -Eq '^  storage_critical_pvcs:[[:space:]]*$' "${CONF_DIR}/ananke.yaml" \
-    && ! grep -Eq '^  require_post_start_probes:[[:space:]]*(true|false)' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei '/^    - sso\/keycloak-data$/a\  require_post_start_probes: true\n  post_start_probe_wait_seconds: 240\n  post_start_probe_poll_seconds: 5\n  post_start_probes:\n    - https://scm.bstein.dev/api/healthz\n    - https://metrics.bstein.dev/api/health\n  require_service_checklist: true\n  service_checklist_wait_seconds: 420\n  service_checklist_poll_seconds: 5\n  service_checklist_stability_seconds: 120\n  service_checklist:\n    - name: gitea-api\n      url: https://scm.bstein.dev/api/healthz\n      accepted_statuses: [200]\n      body_contains: pass\n      timeout_seconds: 12\n    - name: grafana-api\n      url: https://metrics.bstein.dev/api/health\n      accepted_statuses: [200]\n      body_contains: '\''\"database\":\"ok\"'\''\n      timeout_seconds: 12\n  vault_unseal_key_file: /var/lib/ananke/vault-unseal.key' "${CONF_DIR}/ananke.yaml"
-    echo "[install] added startup post-start probe + vault key fallback defaults"
-    changed=1
-  fi
-  if grep -Eq '^    - https://sso.bstein.dev/realms/atlas/.well-known/openid-configuration$' "${CONF_DIR}/ananke.yaml"; then
-    sed -Ei '/^    - https:\/\/sso\.bstein\.dev\/realms\/atlas\/\.well-known\/openid-configuration$/d' "${CONF_DIR}/ananke.yaml"
-    echo "[install] removed sso OIDC probe from startup.post_start_probes (returns 404 in current deployment)"
-    changed=1
-  fi
-  if ! grep -Eq '^  vault_unseal_key_file:[[:space:]]*/var/lib/ananke/vault-unseal.key' "${CONF_DIR}/ananke.yaml"; then
-    if grep -Eq '^startup:[[:space:]]*$' "${CONF_DIR}/ananke.yaml" && grep -Eq '^  post_start_probes:[[:space:]]*$' "${CONF_DIR}/ananke.yaml"; then
-      sed -Ei '/^    - https:\/\/metrics\.bstein\.dev\/api\/health$/a\  vault_unseal_key_file: /var/lib/ananke/vault-unseal.key' "${CONF_DIR}/ananke.yaml"
-      echo "[install] added startup.vault_unseal_key_file default"
-      changed=1
-    fi
-  fi
-  if ! grep -Eq '^  vault_unseal_breakglass_timeout_seconds:[[:space:]]*[0-9]+' "${CONF_DIR}/ananke.yaml"; then
-    if grep -Eq '^  vault_unseal_key_file:[[:space:]]*/var/lib/ananke/vault-unseal.key' "${CONF_DIR}/ananke.yaml"; then
-      sed -Ei '/^  vault_unseal_key_file:[[:space:]]*\/var\/lib\/ananke\/vault-unseal.key$/a\  vault_unseal_breakglass_command: ""\n  vault_unseal_breakglass_timeout_seconds: 15' "${CONF_DIR}/ananke.yaml"
-      echo "[install] added startup break-glass fallback defaults"
-      changed=1
-    fi
-  fi
-
-  local role
-  role="$(read_ananke_role)"
-  local inventory_block
-  local managed_block
-  local workers_block
-  workers_block='workers:
-  - titan-04
-  - titan-05
-  - titan-06
-  - titan-07
-  - titan-08
-  - titan-09
-  - titan-10
-  - titan-11
-  - titan-12
-  - titan-13
-  - titan-14
-  - titan-15
-  - titan-17
-  - titan-18
-  - titan-19
-  - titan-20
-  - titan-21
-  - titan-22
-  - titan-24'
-  if [[ "${role}" == "coordinator" ]]; then
-    inventory_block='ssh_node_hosts:
-  titan-db: 192.168.22.10
-  titan-0a: 192.168.22.11
-  titan-0b: 192.168.22.12
-  titan-0c: 192.168.22.13
-  titan-04: 192.168.22.30
-  titan-05: 192.168.22.31
-  titan-06: 192.168.22.32
-  titan-07: 192.168.22.33
-  titan-08: 192.168.22.34
-  titan-09: 192.168.22.35
-  titan-10: 192.168.22.36
-  titan-11: 192.168.22.37
-  titan-12: 192.168.22.40
-  titan-13: 192.168.22.41
-  titan-14: 192.168.22.42
-  titan-15: 192.168.22.43
-  titan-17: 192.168.22.45
-  titan-18: 192.168.22.46
-  titan-19: 192.168.22.47
-  titan-20: 192.168.22.20
-  titan-21: 192.168.22.21
-  titan-22: 192.168.22.22
-  titan-24: 192.168.22.26'
-    managed_block='ssh_managed_nodes:
-  - titan-db
-  - titan-0a
-  - titan-0b
-  - titan-0c
-  - titan-04
-  - titan-05
-  - titan-06
-  - titan-07
-  - titan-08
-  - titan-09
-  - titan-10
-  - titan-11
-  - titan-12
-  - titan-13
-  - titan-14
-  - titan-15
-  - titan-17
-  - titan-18
-  - titan-19
-  - titan-20
-  - titan-21
-  - titan-22
-  - titan-24'
-  elif [[ "${role}" == "peer" ]]; then
-    inventory_block='ssh_node_hosts:
-  titan-db: 192.168.22.10
-  titan-0a: 192.168.22.11
-  titan-0b: 192.168.22.12
-  titan-0c: 192.168.22.13
-  titan-04: 192.168.22.30
-  titan-05: 192.168.22.31
-  titan-06: 192.168.22.32
-  titan-07: 192.168.22.33
-  titan-08: 192.168.22.34
-  titan-09: 192.168.22.35
-  titan-10: 192.168.22.36
-  titan-11: 192.168.22.37
-  titan-12: 192.168.22.40
-  titan-13: 192.168.22.41
-  titan-14: 192.168.22.42
-  titan-15: 192.168.22.43
-  titan-17: 192.168.22.45
-  titan-18: 192.168.22.46
-  titan-19: 192.168.22.47
-  titan-20: 192.168.22.20
-  titan-21: 192.168.22.21
-  titan-22: 192.168.22.22
-  titan-24: 192.168.22.26'
-    managed_block='ssh_managed_nodes:
-  - titan-db
-  - titan-0a
-  - titan-0b
-  - titan-0c
-  - titan-04
-  - titan-05
-  - titan-06
-  - titan-07
-  - titan-08
-  - titan-09
-  - titan-10
-  - titan-11
-  - titan-12
-  - titan-13
-  - titan-14
-  - titan-15
-  - titan-17
-  - titan-18
-  - titan-19
-  - titan-20
-  - titan-21
-  - titan-22
-  - titan-24'
-  fi
-
-  if [[ -n "${inventory_block}" ]]; then
-    if grep -Eq '^ssh_node_hosts:[[:space:]]*\{\}[[:space:]]*$' "${CONF_DIR}/ananke.yaml"; then
-      perl -0pi -e 's#ssh_node_hosts:\s*\{\}\n#'"${inventory_block}"'\n#s' "${CONF_DIR}/ananke.yaml"
-      echo "[install] hydrated ssh_node_hosts inventory for role=${role}"
-      changed=1
-    fi
-  fi
-  if grep -Eq '^workers:[[:space:]]*\[\][[:space:]]*$' "${CONF_DIR}/ananke.yaml"; then
-    perl -0pi -e 's#workers:\s*\[\]\n#'"${workers_block}"'\n#s' "${CONF_DIR}/ananke.yaml"
-    echo "[install] hydrated workers inventory for startup/shutdown orchestration"
-    changed=1
-  fi
-
-  if [[ -n "${managed_block}" ]]; then
-    if grep -Eq '^ssh_managed_nodes:[[:space:]]*\[\][[:space:]]*$' "${CONF_DIR}/ananke.yaml"; then
-      perl -0pi -e 's#ssh_managed_nodes:\s*\[\]\n#'"${managed_block}"'\n#s' "${CONF_DIR}/ananke.yaml"
-      echo "[install] hydrated ssh_managed_nodes inventory for role=${role}"
-      changed=1
-    fi
-    if ! grep -Eq '^  - titan-04$' "${CONF_DIR}/ananke.yaml" || ! grep -Eq '^  - titan-21$' "${CONF_DIR}/ananke.yaml"; then
-      perl -0pi -e 's#ssh_managed_nodes:\n(?:  - [^\n]*\n)*#'"${managed_block}"'\n#s' "${CONF_DIR}/ananke.yaml"
-      echo "[install] refreshed ssh_managed_nodes coverage for role=${role}"
-      changed=1
-    fi
-  fi
-
-  if [[ "${role}" == "peer" ]]; then
-    if grep -Eq '^ssh_managed_nodes:[[:space:]]*$' "${CONF_DIR}/ananke.yaml" \
-      && grep -Eq '^  - titan-db$' "${CONF_DIR}/ananke.yaml" \
-      && grep -Eq '^  - titan-24$' "${CONF_DIR}/ananke.yaml" \
-      && ! grep -Eq '^  - titan-0a$' "${CONF_DIR}/ananke.yaml"; then
-      perl -0pi -e 's#ssh_managed_nodes:\n  - titan-db\n  - titan-24\n#ssh_managed_nodes:\n  - titan-db\n  - titan-0a\n  - titan-0b\n  - titan-0c\n  - titan-04\n  - titan-05\n  - titan-06\n  - titan-07\n  - titan-08\n  - titan-09\n  - titan-10\n  - titan-11\n  - titan-12\n  - titan-13\n  - titan-14\n  - titan-15\n  - titan-17\n  - titan-18\n  - titan-19\n  - titan-20\n  - titan-21\n  - titan-22\n  - titan-24\n#s' "${CONF_DIR}/ananke.yaml"
-      echo "[install] expanded peer ssh_managed_nodes for bootstrap fallback coverage"
-      changed=1
-    fi
-
-    if ! grep -Eq '^  - services/keycloak$' "${CONF_DIR}/ananke.yaml" || ! grep -Eq '^  - infrastructure/cert-manager$' "${CONF_DIR}/ananke.yaml" || ! grep -Eq '^  - services/oauth2-proxy$' "${CONF_DIR}/ananke.yaml"; then
-      perl -0pi -e 's#local_bootstrap_paths:\n(?:  - [^\n]*\n)*#local_bootstrap_paths:\n  - infrastructure/core\n  - clusters/atlas/flux-system\n  - infrastructure/sources/helm\n  - infrastructure/metallb\n  - infrastructure/traefik\n  - infrastructure/cert-manager\n  - infrastructure/vault-csi\n  - infrastructure/vault-injector\n  - services/vault\n  - infrastructure/postgres\n  - services/gitea\n  - services/keycloak\n  - services/oauth2-proxy\n#s' "${CONF_DIR}/ananke.yaml"
-      echo "[install] refreshed peer local_bootstrap_paths for full fallback bootstrap parity"
-      changed=1
-    fi
-
-  fi
-
-  if [[ "${changed}" -eq 1 ]]; then
-    chmod 0640 "${CONF_DIR}/ananke.yaml" || true
-  fi
-}
-
-sanitize_migrated_ananke_config() {
-  local cfg="${CONF_DIR}/ananke.yaml"
-  [[ -f "${cfg}" ]] || return 0
-
-  local tmp changed=0
-  tmp="$(mktemp)"
-
-  # Legacy migration bug guard:
-  # If root-level "- node" entries were accidentally appended after ssh_managed_nodes,
-  # drop those orphan entries until the next top-level key.
-  awk '
-    BEGIN {in_managed=0}
-    /^ssh_managed_nodes:[[:space:]]*$/ {in_managed=1; print; next}
-    {
-      if (in_managed) {
-        if ($0 ~ /^  - /) {print; next}
-        if ($0 ~ /^- /) {next}
-        if ($0 ~ /^[A-Za-z0-9_]+:[[:space:]]*/) {in_managed=0}
-      }
-      print
-    }
-  ' "${cfg}" > "${tmp}"
-
-  if ! cmp -s "${cfg}" "${tmp}"; then
-    mv "${tmp}" "${cfg}"
-    changed=1
-    echo "[install] sanitized malformed ssh_managed_nodes block in ${cfg}"
-  else
-    rm -f "${tmp}"
-  fi
-
-  if grep -Eq '^[[:space:]]*forward_shutdown_config:[[:space:]]*/etc/ananke/hecate.yaml[[:space:]]*$' "${cfg}"; then
-    sed -Ei 's#(^[[:space:]]*forward_shutdown_config:[[:space:]]*)/etc/ananke/hecate.yaml#\1/etc/ananke/ananke.yaml#' "${cfg}"
-    changed=1
-    echo "[install] migrated coordination.forward_shutdown_config to /etc/ananke/ananke.yaml"
-  fi
-
-  if [[ "${changed}" -eq 1 ]]; then
-    chmod 0640 "${cfg}" || true
-  fi
-}
-
-ensure_apt_packages() {
-  local missing=()
-  for pkg in "$@"; do
-    if ! dpkg -s "${pkg}" >/dev/null 2>&1; then
-      missing+=("${pkg}")
-    fi
-  done
-  if [[ ${#missing[@]} -eq 0 ]]; then
-    return 0
-  fi
-  echo "[install] apt install: ${missing[*]}"
-  export DEBIAN_FRONTEND=noninteractive
-  apt-get update -y
-  apt-get install -y "${missing[@]}"
-}
-
-install_kubectl_if_missing() {
-  if command -v kubectl >/dev/null 2>&1; then
-    return 0
-  fi
-  ensure_apt_packages kubernetes-client || true
-  if command -v kubectl >/dev/null 2>&1; then
-    return 0
-  fi
-  echo "[install] installing kubectl via upstream binary"
-  local arch
-  arch="$(uname -m)"
-  case "${arch}" in
-    x86_64) arch="amd64" ;;
-    aarch64|arm64) arch="arm64" ;;
-    *) echo "Unsupported arch for kubectl install: ${arch}" >&2; return 1 ;;
-  esac
-  local version
-  version="$(curl -fsSL https://dl.k8s.io/release/stable.txt)"
-  curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/${version}/bin/linux/${arch}/kubectl"
-  chmod 0755 /usr/local/bin/kubectl
-}
-
-ensure_dependencies() {
-  if [[ "${INSTALL_DEPS}" -eq 0 ]]; then
-    echo "[install] skipping dependency installation"
-    return 0
-  fi
-  if ! command -v apt-get >/dev/null 2>&1; then
-    echo "This installer currently supports apt-based hosts only." >&2
-    exit 1
-  fi
-  ensure_apt_packages ca-certificates curl git openssh-client jq nut-client nut-server nut-monitor golang-go
-  install_kubectl_if_missing
-}
-
-legacy_path_rewrite() {
-  local src="$1"
-  local dst="$2"
-  sed \
-    -e 's#/etc/hecate/hecate.yaml#/etc/ananke/ananke.yaml#g' \
-    -e 's#/etc/hecate/kubeconfig#/etc/ananke/kubeconfig#g' \
-    -e 's#/var/lib/hecate/vault-unseal.key#/var/lib/ananke/vault-unseal.key#g' \
-    -e 's#/var/lib/hecate/hecate.lock#/var/lib/ananke/ananke.lock#g' \
-    -e 's#/opt/hecate#/opt/ananke#g' \
-    -e 's#/etc/hecate#/etc/ananke#g' \
-    -e 's#/var/lib/hecate#/var/lib/ananke#g' \
-    -e 's#/usr/local/bin/hecate#/usr/local/bin/ananke#g' \
-    -e 's#/usr/local/lib/hecate#/usr/local/lib/ananke#g' \
-    -e 's/hecate.yaml/ananke.yaml/g' \
-    -e 's/hecate.lock/ananke.lock/g' \
-    -e 's/hecate/ananke/g' \
-    -e 's/Hecate/Ananke/g' \
-    -e 's#hecate\.lock#ananke.lock#g' \
-    "${src}" > "${dst}"
-}
-
-migrate_legacy_hecate_install() {
-  local legacy_conf_dir="/etc/hecate"
-  local legacy_state_dir="/var/lib/hecate"
-  local legacy_systemd_dir="/etc/systemd/system"
-
-  install -d -m 0750 "${CONF_DIR}"
-  install -d -m 0750 "${STATE_DIR}"
-
-  if [[ ! -f "${CONF_DIR}/ananke.yaml" && -f "${legacy_conf_dir}/hecate.yaml" ]]; then
-    echo "[install] migrating legacy config ${legacy_conf_dir}/hecate.yaml -> ${CONF_DIR}/ananke.yaml"
-    legacy_path_rewrite "${legacy_conf_dir}/hecate.yaml" "${CONF_DIR}/ananke.yaml"
-    chmod 0640 "${CONF_DIR}/ananke.yaml"
-  fi
-
-  if [[ ! -f "${CONF_DIR}/kubeconfig" && -f "${legacy_conf_dir}/kubeconfig" ]]; then
-    echo "[install] migrating legacy kubeconfig ${legacy_conf_dir}/kubeconfig -> ${CONF_DIR}/kubeconfig"
-    install -m 0600 "${legacy_conf_dir}/kubeconfig" "${CONF_DIR}/kubeconfig"
-  fi
-
-  if [[ ! -f "${STATE_DIR}/vault-unseal.key" && -f "${legacy_state_dir}/vault-unseal.key" ]]; then
-    echo "[install] migrating legacy vault key ${legacy_state_dir}/vault-unseal.key -> ${STATE_DIR}/vault-unseal.key"
-    install -m 0600 "${legacy_state_dir}/vault-unseal.key" "${STATE_DIR}/vault-unseal.key"
-  fi
-
-  if [[ ! -f "${STATE_DIR}/runs.json" && -f "${legacy_state_dir}/runs.json" ]]; then
-    echo "[install] migrating legacy run history ${legacy_state_dir}/runs.json -> ${STATE_DIR}/runs.json"
-    install -m 0640 "${legacy_state_dir}/runs.json" "${STATE_DIR}/runs.json"
-  fi
-
-  if [[ ! -f "${STATE_DIR}/intent.json" && -f "${legacy_state_dir}/intent.json" ]]; then
-    echo "[install] migrating legacy intent state ${legacy_state_dir}/intent.json -> ${STATE_DIR}/intent.json"
-    install -m 0640 "${legacy_state_dir}/intent.json" "${STATE_DIR}/intent.json"
-  fi
-
-  if [[ ! -f "${STATE_DIR}/ananke.lock" && -f "${legacy_state_dir}/hecate.lock" ]]; then
-    echo "[install] migrating legacy lock ${legacy_state_dir}/hecate.lock -> ${STATE_DIR}/ananke.lock"
-    install -m 0640 "${legacy_state_dir}/hecate.lock" "${STATE_DIR}/ananke.lock"
-  fi
-
-  if [[ -d "${legacy_systemd_dir}" ]]; then
-    if ls "${legacy_systemd_dir}"/hecate*.service >/dev/null 2>&1 || ls "${legacy_systemd_dir}"/hecate*.timer >/dev/null 2>&1; then
-      echo "[install] detected legacy hecate systemd unit files; will retire after ananke install"
-    fi
-  fi
-}
-
-retire_legacy_hecate_install() {
-  local ts backup_dir
-  ts="$(date +%Y%m%d%H%M%S)"
-  backup_dir="/var/backups/ananke-legacy-hecate-${ts}"
-
-  systemctl disable --now hecate.service hecate-bootstrap.service hecate-update.timer >/dev/null 2>&1 || true
-  systemctl stop hecate-update.service >/dev/null 2>&1 || true
-
-  if [[ -d /etc/hecate || -d /var/lib/hecate || -d /usr/local/lib/hecate || -d /opt/hecate ]]; then
-    install -d -m 0750 "${backup_dir}"
-    [[ -d /etc/hecate ]] && cp -a /etc/hecate "${backup_dir}/" || true
-    [[ -d /var/lib/hecate ]] && cp -a /var/lib/hecate "${backup_dir}/" || true
-    [[ -d /usr/local/lib/hecate ]] && cp -a /usr/local/lib/hecate "${backup_dir}/" || true
-    [[ -d /opt/hecate ]] && cp -a /opt/hecate "${backup_dir}/" || true
-    [[ -f /usr/local/bin/hecate ]] && install -m 0755 /usr/local/bin/hecate "${backup_dir}/hecate.bin" || true
-    echo "[install] backed up legacy hecate assets to ${backup_dir}"
-  fi
-
-  rm -f \
-    /etc/systemd/system/hecate.service \
-    /etc/systemd/system/hecate-bootstrap.service \
-    /etc/systemd/system/hecate-update.service \
-    /etc/systemd/system/hecate-update.timer
-  rm -f /usr/local/bin/hecate
-  rm -rf /usr/local/lib/hecate
-  rm -rf /opt/hecate
-  rm -rf /etc/hecate
-  rm -rf /var/lib/hecate
-}
-
-resolve_build_target() {
-  if [[ -d "${REPO_DIR}/cmd/ananke" ]]; then
-    echo "./cmd/ananke"
-    return 0
-  fi
-  return 1
-}
-
-install_config_template() {
-  local template="$1"
-  local dest="$2"
-  local src legacy
-  local -a modern_candidates=()
-  local -a legacy_candidates=()
-
-  case "${template}" in
-    coordinator)
-      modern_candidates=("configs/ananke.coordinator.yaml" "configs/ananke.titan-db.yaml")
-      legacy_candidates=("configs/hecate.titan-db.yaml")
-      ;;
-    peer)
-      modern_candidates=("configs/ananke.peer.yaml" "configs/ananke.tethys.yaml")
-      legacy_candidates=("configs/hecate.tethys.yaml")
-      ;;
-    example)
-      modern_candidates=("configs/ananke.example.yaml")
-      legacy_candidates=("configs/hecate.example.yaml")
-      ;;
-    *)
-      echo "[install] unknown config template key: ${template}" >&2
-      return 1
-      ;;
-  esac
-
-  for src in "${modern_candidates[@]}"; do
-    if [[ -f "${src}" ]]; then
-      install -m 0640 "${src}" "${dest}"
-      return 0
-    fi
-  done
-
-  for legacy in "${legacy_candidates[@]}"; do
-    if [[ -f "${legacy}" ]]; then
-      src="$(mktemp)"
-      legacy_path_rewrite "${legacy}" "${src}"
-      install -m 0640 "${src}" "${dest}"
-      rm -f "${src}"
-      return 0
-    fi
-  done
-
-  echo "[install] missing config template sources for '${template}'. modern=[${modern_candidates[*]}] legacy=[${legacy_candidates[*]}]" >&2
-  return 1
-}
-
-install_systemd_units() {
-  local source_map
-  local tmp
-
-  while IFS='|' read -r target_name modern_name legacy_name; do
-    local modern_src="deploy/systemd/${modern_name}"
-    local legacy_src="deploy/systemd/${legacy_name}"
-    local target="${SYSTEMD_DIR}/${target_name}"
-
-    if [[ -f "${modern_src}" ]]; then
-      install -m 0644 "${modern_src}" "${target}"
-      continue
-    fi
-
-    if [[ -f "${legacy_src}" ]]; then
-      tmp="$(mktemp)"
-      legacy_path_rewrite "${legacy_src}" "${tmp}"
-      install -m 0644 "${tmp}" "${target}"
-      rm -f "${tmp}"
-      continue
-    fi
-
-    echo "[install] missing both modern and legacy systemd unit sources for ${target_name}" >&2
-    return 1
-  done <<'EOF_UNITS'
-ananke.service|ananke.service|hecate.service
-ananke-bootstrap.service|ananke-bootstrap.service|hecate-bootstrap.service
-ananke-update.service|ananke-update.service|hecate-update.service
-ananke-update.timer|ananke-update.timer|hecate-update.timer
-EOF_UNITS
-}
-
-install_self_update_script() {
-  local modern_src="scripts/ananke-self-update.sh"
-  local legacy_src="scripts/hecate-self-update.sh"
-  local target="${LIB_DIR}/ananke-self-update.sh"
-  local tmp
-
-  if [[ -f "${modern_src}" ]]; then
-    install -m 0755 "${modern_src}" "${target}"
-    return 0
-  fi
-
-  if [[ -f "${legacy_src}" ]]; then
-    tmp="$(mktemp)"
-    legacy_path_rewrite "${legacy_src}" "${tmp}"
-    sed -Ei \
-      -e 's/HECATE_/ANANKE_/g' \
-      -e 's/hecate-self-update/ananke-self-update/g' \
-      -e 's#/opt/hecate#/opt/ananke#g' \
-      -e 's#bstein/hecate\.git#bstein/ananke.git#g' \
-      "${tmp}"
-    install -m 0755 "${tmp}" "${target}"
-    rm -f "${tmp}"
-    return 0
-  fi
-
-  echo "[install] missing both modern and legacy self-update scripts." >&2
-  return 1
-}
-
-configure_nut() {
-  if [[ "${MANAGE_NUT}" != "1" ]]; then
-    echo "[install] skipping NUT configuration (ANANKE_MANAGE_NUT=${MANAGE_NUT})"
-    return 0
-  fi
-
-  echo "[install] configuring NUT + udev for UPS ${NUT_UPS_NAME} (${NUT_VENDOR_ID}:${NUT_PRODUCT_ID})"
-  install -d -m 0755 /etc/nut /etc/udev/rules.d
-
-  cat > /etc/nut/nut.conf <<EOF
-MODE=standalone
-EOF
-
-  cat > /etc/nut/ups.conf <<EOF
-[${NUT_UPS_NAME}]
-  driver = usbhid-ups
-  port = auto
-  vendorid = ${NUT_VENDOR_ID}
-  productid = ${NUT_PRODUCT_ID}
-  pollinterval = 5
-EOF
-
-  cat > /etc/nut/upsd.users <<EOF
-[${NUT_MONITOR_USER}]
-  password = ${NUT_MONITOR_PASSWORD}
-  upsmon primary
-EOF
-  chmod 0640 /etc/nut/upsd.users
-  if getent group nut >/dev/null 2>&1; then
-    chown root:nut /etc/nut/upsd.users
-  else
-    chown root:root /etc/nut/upsd.users
-  fi
-
-  cat > /etc/nut/upsmon.conf <<EOF
-RUN_AS_USER nut
-MONITOR ${NUT_UPS_NAME}@localhost 1 ${NUT_MONITOR_USER} ${NUT_MONITOR_PASSWORD} primary
-MINSUPPLIES 1
-SHUTDOWNCMD "/sbin/shutdown -h +0"
-POLLFREQ 5
-POLLFREQALERT 5
-HOSTSYNC 15
-DEADTIME 15
-POWERDOWNFLAG /etc/killpower
-EOF
-
-  cat > /etc/udev/rules.d/99-ananke-ups.rules <<EOF
-# Managed by ananke install.sh: ensure UPS USB HID devices are readable by NUT
-ACTION=="add|change", SUBSYSTEM=="usb", ATTR{idVendor}=="${NUT_VENDOR_ID}", ATTR{idProduct}=="${NUT_PRODUCT_ID}", MODE:="0660", GROUP:="nut"
-EOF
-
-  udevadm control --reload-rules || true
-  udevadm trigger --subsystem-match=usb --attr-match=idVendor="${NUT_VENDOR_ID}" --attr-match=idProduct="${NUT_PRODUCT_ID}" || true
-
-  systemctl enable nut-driver-enumerator.service nut-server.service nut-monitor.service >/dev/null 2>&1 || true
-  systemctl restart nut-driver-enumerator.service >/dev/null 2>&1 || true
-  systemctl restart "nut-driver@${NUT_UPS_NAME}.service" >/dev/null 2>&1 || true
-  systemctl restart nut-server.service nut-monitor.service >/dev/null 2>&1 || true
-}
-
 ensure_dependencies
-migrate_legacy_hecate_install
+if [[ "${BINARY_ONLY}" != "1" ]]; then
+  migrate_legacy_hecate_install
+fi
 
 if [[ "${ENFORCE_QUALITY_GATE}" == "1" ]]; then
   echo "[install] running quality gate"
@@ -887,8 +80,26 @@ go build -o dist/ananke "${BUILD_TARGET}"
 
 echo "[install] installing binary"
 install -d -m 0755 "${BIN_DIR}"
+if [[ "${BINARY_ONLY}" == "1" && -f "${BIN_DIR}/ananke" ]]; then
+  install -d -m 0700 "${LIB_DIR}/rollback"
+  install -m 0700 "${BIN_DIR}/ananke" "${LIB_DIR}/rollback/ananke.previous"
+fi
 install -m 0755 dist/ananke "${BIN_DIR}/ananke"
 
+# Targeted recovery fixes must not overwrite host config, NUT, or bootstrap units.
+if [[ "${BINARY_ONLY}" == "1" ]]; then
+  if [[ "${START_NOW}" == "1" ]]; then
+    if ! systemctl restart ananke.service || ! systemctl is-active --quiet ananke.service; then
+      echo "[install] service failed; restoring previous binary" >&2
+      install -m 0755 "${LIB_DIR}/rollback/ananke.previous" "${BIN_DIR}/ananke"
+      systemctl restart ananke.service
+      exit 1
+    fi
+  fi
+  echo "[install] binary-only update complete; host configuration unchanged"
+  exit 0
+fi
+
 echo "[install] installing config + state dirs"
 install -d -m 0750 "${CONF_DIR}"
 install -d -m 0750 "${STATE_DIR}"
@@ -922,6 +133,7 @@ else
 fi
 migrate_ananke_config
 sanitize_migrated_ananke_config
+install_inventory_fragment
 ensure_ananke_ssh_identity
 ensure_ananke_kubeconfig
 

scripts/lint.sh

@@ -6,13 +6,44 @@ cd "${REPO_DIR}"
 export PATH="$(go env GOPATH)/bin:${PATH}"
 STATICCHECK_VERSION="${ANANKE_STATICCHECK_VERSION:-2025.1.1}"
 
+run_with_retry() {
+  local attempts="$1"
+  shift
+  local try=1
+  local delay=3
+  local rc=0
+  while true; do
+    "$@" && return 0
+    rc=$?
+    if [[ "${try}" -ge "${attempts}" ]]; then
+      return "${rc}"
+    fi
+    echo "[lint] retry ${try}/${attempts} after rc=${rc}: $*" >&2
+    sleep "${delay}"
+    delay=$((delay * 2))
+    try=$((try + 1))
+  done
+}
+
 if ! command -v staticcheck >/dev/null 2>&1 || ! staticcheck -version 2>/dev/null | grep -q "${STATICCHECK_VERSION}"; then
   echo "[lint] installing staticcheck ${STATICCHECK_VERSION}"
-  go install "honnef.co/go/tools/cmd/staticcheck@${STATICCHECK_VERSION}"
+  run_with_retry 4 go install "honnef.co/go/tools/cmd/staticcheck@${STATICCHECK_VERSION}"
 fi
 
 echo "[lint] go vet"
 go vet ./...
 
 echo "[lint] staticcheck (pedantic code-smell pass)"
-staticcheck ./...
+set +e
+staticcheck_output="$(staticcheck ./... 2>&1)"
+staticcheck_rc=$?
+set -e
+if [[ "${staticcheck_rc}" -ne 0 ]]; then
+  printf '%s\n' "${staticcheck_output}" >&2
+  if grep -q "Staticcheck was built with go" <<<"${staticcheck_output}" \
+    && [[ "${ANANKE_STATICCHECK_REQUIRED:-0}" != "1" ]]; then
+    echo "[lint] warning: skipping staticcheck because the host staticcheck binary was built with an older Go toolchain" >&2
+  else
+    exit "${staticcheck_rc}"
+  fi
+fi

scripts/publish_quality_metrics.py

@@ -14,7 +14,7 @@ import urllib.error
 import urllib.request
 
 
-DEFAULT_PUSHGATEWAY_URL = "http://platform-quality-gateway.monitoring.svc.cluster.local:9091"
+DEFAULT_PUSHGATEWAY_URL = "http://platform-quality-gateway.monitoring.svc.cluster.local:9091"  # NOSONAR: in-cluster Pushgateway is HTTP-only.
 SOURCE_SCAN_ROOTS = ("cmd", "internal", "scripts", "testing")
 SOURCE_EXTENSIONS = {".go", ".py", ".sh"}
 QUALITY_SUCCESS_STATES = {"ok", "pass", "passed", "success", "compliant"}
@@ -77,6 +77,17 @@ def _fetch_existing_counter(pushgateway_url: str, metric: str, labels: dict[str,
     return 0.0
 
 
+def _series_exists(pushgateway_url: str, metric: str, labels: dict[str, str], timeout_seconds: float) -> bool:
+    """Return whether Pushgateway already has a series for this build."""
+    text = _read_http(f"{pushgateway_url.rstrip('/')}/metrics", timeout_seconds)
+    for line in text.splitlines():
+        if not line.startswith(metric + "{"):
+            continue
+        if all(f'{key}="{value}"' in line for key, value in labels.items()):
+            return True
+    return False
+
+
 def _build_payload(
     suite: str,
     trigger: str,
@@ -89,37 +100,66 @@ def _build_payload(
     tests_skipped: int,
     test_cases: list[tuple[str, str]],
     coverage_percent: float,
+    source_files_total: int,
     source_lines_over_500: int,
+    branch: str,
+    build_number: str,
+    jenkins_job: str,
     checks: dict[str, str],
 ) -> str:
+    build_labels = {
+        "suite": suite,
+        "branch": branch,
+        "build_number": build_number or "unknown",
+        "jenkins_job": jenkins_job,
+    }
+    base_labels = {
+        "suite": suite,
+        "branch": branch or "unknown",
+    }
+    test_case_base_labels = {
+        "suite": suite,
+        "branch": branch,
+        "build_number": build_number or "unknown",
+        "jenkins_job": jenkins_job,
+    }
     lines = [
         "# TYPE platform_quality_gate_runs_total counter",
-        f'platform_quality_gate_runs_total{{suite="{suite}",status="ok"}} {ok_count}',
+        f"platform_quality_gate_runs_total{_label_str({**base_labels, 'status': 'ok'})} {ok_count}",
-        f'platform_quality_gate_runs_total{{suite="{suite}",status="failed"}} {failed_count}',
+        f"platform_quality_gate_runs_total{_label_str({**base_labels, 'status': 'failed'})} {failed_count}",
         "# TYPE ananke_quality_gate_tests_total gauge",
-        f'ananke_quality_gate_tests_total{{suite="{suite}",result="passed"}} {tests_passed}',
+        f"ananke_quality_gate_tests_total{_label_str({**base_labels, 'result': 'passed'})} {tests_passed}",
-        f'ananke_quality_gate_tests_total{{suite="{suite}",result="failed"}} {tests_failed}',
+        f"ananke_quality_gate_tests_total{_label_str({**base_labels, 'result': 'failed'})} {tests_failed}",
-        f'ananke_quality_gate_tests_total{{suite="{suite}",result="error"}} {tests_errors}',
+        f"ananke_quality_gate_tests_total{_label_str({**base_labels, 'result': 'error'})} {tests_errors}",
-        f'ananke_quality_gate_tests_total{{suite="{suite}",result="skipped"}} {tests_skipped}',
+        f"ananke_quality_gate_tests_total{_label_str({**base_labels, 'result': 'skipped'})} {tests_skipped}",
         "# TYPE ananke_quality_gate_coverage_percent gauge",
-        f'ananke_quality_gate_coverage_percent{{suite="{suite}"}} {coverage_percent:.3f}',
+        f"ananke_quality_gate_coverage_percent{_label_str(base_labels)} {coverage_percent:.3f}",
         "# TYPE platform_quality_gate_workspace_line_coverage_percent gauge",
-        f'platform_quality_gate_workspace_line_coverage_percent{{suite="{suite}"}} {coverage_percent:.3f}',
+        f"platform_quality_gate_workspace_line_coverage_percent{_label_str(base_labels)} {coverage_percent:.3f}",
+        "# TYPE platform_quality_gate_source_files_total gauge",
+        f"platform_quality_gate_source_files_total{_label_str(base_labels)} {source_files_total}",
         "# TYPE platform_quality_gate_source_lines_over_500_total gauge",
-        f'platform_quality_gate_source_lines_over_500_total{{suite="{suite}"}} {source_lines_over_500}',
+        f"platform_quality_gate_source_lines_over_500_total{_label_str(base_labels)} {source_lines_over_500}",
-        "# TYPE platform_quality_gate_test_case_result gauge",
+        "# TYPE platform_quality_gate_build_info gauge",
+        f"platform_quality_gate_build_info{_label_str(build_labels)} 1",
         "# TYPE ananke_quality_gate_checks_total gauge",
         "# TYPE ananke_quality_gate_publish_info gauge",
-        f'ananke_quality_gate_publish_info{_label_str({"suite": suite, "trigger": trigger})} 1',
+        f'ananke_quality_gate_publish_info{_label_str({**base_labels, "trigger": trigger})} 1',
     ]
     lines.extend(
-        f'platform_quality_gate_test_case_result{{suite="{suite}",test="{_escape_label(test_name)}",status="{_escape_label(test_status)}"}} 1'
+        f"ananke_quality_gate_checks_total{_label_str({**base_labels, 'check': check_name, 'result': check_status})} 1"
-        for test_name, test_status in test_cases
-    )
-    lines.extend(
-        f'ananke_quality_gate_checks_total{{suite="{suite}",check="{check_name}",result="{check_status}"}} 1'
         for check_name, check_status in checks.items()
     )
+    lines.append("# TYPE platform_quality_gate_test_case_result gauge")
+    if test_cases:
+        lines.extend(
+            f"platform_quality_gate_test_case_result{_label_str({**test_case_base_labels, 'test': test_name, 'status': test_status})} 1"
+            for test_name, test_status in test_cases
+        )
+    else:
+        lines.append(
+            f"platform_quality_gate_test_case_result{_label_str({**test_case_base_labels, 'test': '__no_test_cases__', 'status': 'skipped'})} 1"
+        )
     return "\n".join(lines) + "\n"
 
 
@@ -136,8 +176,15 @@ def _read_coverage_percent(path: str) -> float:
         return 0.0
 
 
-def _count_source_files_over_limit(repo_root: Path, max_lines: int = 500) -> int:
+def _resolve_repo_path(repo_root: Path, path: str) -> Path:
-    count = 0
+    """Resolve quality-gate artifact paths relative to the repository root."""
+    candidate = Path(path)
+    if candidate.is_absolute():
+        return candidate
+    return repo_root / candidate
+
+
+def _iter_source_files(repo_root: Path):
     for rel_root in SOURCE_SCAN_ROOTS:
         base = repo_root / rel_root
         if not base.exists():
@@ -147,12 +194,37 @@ def _count_source_files_over_limit(repo_root: Path, max_lines: int = 500) -> int
                 continue
             if path.suffix not in SOURCE_EXTENSIONS:
                 continue
-            lines = len(path.read_text(encoding="utf-8", errors="ignore").splitlines())
+            if path.name.endswith("_test.go") or path.name.endswith(".test.py"):
-            if lines > max_lines:
+                continue
-                count += 1
+            yield path
+
+
+def _count_source_files(repo_root: Path) -> int:
+    return sum(1 for _ in _iter_source_files(repo_root))
+
+
+def _count_source_files_over_limit(repo_root: Path, max_lines: int = 500) -> int:
+    count = 0
+    for path in _iter_source_files(repo_root):
+        lines = len(path.read_text(encoding="utf-8", errors="ignore").splitlines())
+        if lines > max_lines:
+            count += 1
     return count
 
 
+def _unit_tests_failed(output_path: Path, coverage_percent: float) -> bool:
+    if coverage_percent <= 0 or not output_path.exists():
+        return True
+    text = output_path.read_text(encoding="utf-8", errors="ignore")
+    start_marker = "[quality] unit tests + workspace coverage profile"
+    end_marker = "[quality] hygiene: doc contracts"
+    if start_marker in text:
+        text = text.split(start_marker, 1)[1]
+    if end_marker in text:
+        text = text.split(end_marker, 1)[0]
+    return bool(re.search(r"^(--- FAIL:|FAIL\\b)", text, flags=re.M))
+
+
 def _parse_go_test_counts(output_path: Path) -> dict[str, int]:
     if not output_path.exists():
         return {"passed": 0, "failed": 0, "errors": 0, "skipped": 0}
@@ -166,14 +238,37 @@ def _parse_go_test_counts(output_path: Path) -> dict[str, int]:
 
 
 def _parse_go_test_cases(output_path: Path) -> list[tuple[str, str]]:
+    """Parse per-test status records from go test output text."""
     if not output_path.exists():
         return []
     text = output_path.read_text(encoding="utf-8", errors="ignore")
     cases: list[tuple[str, str]] = []
-    for match in re.finditer(r"^---\s+(PASS|FAIL|SKIP):\s+(\S+)", text, flags=re.M):
+    patterns = {
-        raw_status, test_name = match.groups()
+        "passed": re.compile(r"^--- PASS: ([^\s(]+)", flags=re.M),
-        status = {"PASS": "passed", "FAIL": "failed", "SKIP": "skipped"}.get(raw_status, "error")
+        "failed": re.compile(r"^--- FAIL: ([^\s(]+)", flags=re.M),
-        cases.append((test_name.strip(), status))
+        "skipped": re.compile(r"^--- SKIP: ([^\s(]+)", flags=re.M),
+    }
+    for status, pattern in patterns.items():
+        for test_name in pattern.findall(text):
+            cleaned = str(test_name).strip()
+            if cleaned:
+                cases.append((cleaned, status))
+    if cases:
+        return cases
+
+    # Fallback for non-verbose `go test` output where individual test names are absent.
+    package_cases: list[tuple[str, str]] = []
+    for package_name in re.findall(r"^ok\s+([^\s]+)", text, flags=re.M):
+        cleaned = str(package_name).strip()
+        if cleaned:
+            package_cases.append((f"package::{cleaned}", "passed"))
+    for package_name in re.findall(r"^FAIL\s+([^\s]+)", text, flags=re.M):
+        cleaned = str(package_name).strip()
+        if cleaned:
+            package_cases.append((f"package::{cleaned}", "failed"))
+    if package_cases:
+        deduped = list(dict.fromkeys(package_cases))
+        return deduped
     return cases
 
 
@@ -224,17 +319,23 @@ def _sonarqube_check_status(build_dir: Path) -> str:
 
 
 def _supply_chain_check_status(build_dir: Path) -> str:
+    required = os.getenv("QUALITY_GATE_IRONBANK_REQUIRED", "0").strip().lower() in {"1", "true", "yes", "on"}
     report = _load_json(Path(os.getenv("QUALITY_GATE_IRONBANK_REPORT", str(build_dir / "ironbank-compliance.json"))))
     if not report:
-        return "not_applicable"
+        return "failed" if required else "not_applicable"
     compliant = report.get("compliant")
     if isinstance(compliant, bool):
         return "ok" if compliant else "failed"
     status_candidates = [report.get("status"), report.get("result"), report.get("compliance")]
     for value in status_candidates:
         if isinstance(value, str):
-            return "ok" if value.strip().lower() in QUALITY_SUCCESS_STATES else "failed"
+            normalized = value.strip().lower()
-    return "failed"
+            if normalized in QUALITY_SUCCESS_STATES:
+                return "ok"
+            if normalized in {"n/a", "na", "not_applicable", "not-applicable", "skipped", "skip"}:
+                return "failed" if required else "not_applicable"
+            return "failed" if required else "not_applicable"
+    return "failed" if required else "not_applicable"
 
 
 def parse_args(argv: list[str]) -> argparse.Namespace:
@@ -278,10 +379,21 @@ def main(argv: list[str] | None = None) -> int:
     args = parse_args(argv or sys.argv[1:])
     repo_root = Path(__file__).resolve().parents[1]
     build_dir = repo_root / "build"
+    gate_rc = _read_exit_code(
+        _resolve_repo_path(repo_root, os.getenv("ANANKE_QUALITY_EXIT_CODE_PATH", str(build_dir / "quality-gate.rc")))
+    )
+    current_ok = 1 if gate_rc == 0 else 0
+    current_failed = 0 if gate_rc == 0 else 1
 
+    branch = os.getenv("BRANCH_NAME") or os.getenv("GIT_BRANCH") or "unknown"
+    if branch.startswith("origin/"):
+        branch = branch[len("origin/") :]
+    build_number = os.getenv("BUILD_NUMBER", "")
+    jenkins_job = os.getenv("JOB_NAME", "ananke")
     remote_ok = 0
     remote_failed = 0
     remote_error = ""
+    already_recorded = False
     try:
         remote_ok = int(
             _fetch_existing_counter(
@@ -299,21 +411,44 @@ def main(argv: list[str] | None = None) -> int:
                 args.timeout_seconds,
             )
         )
+        already_recorded = bool(build_number) and _series_exists(
+            args.pushgateway_url,
+            "platform_quality_gate_build_info",
+            {
+                "job": args.job_name,
+                "suite": args.suite,
+                "branch": branch or "unknown",
+                "build_number": build_number or "unknown",
+                "jenkins_job": jenkins_job,
+            },
+            args.timeout_seconds,
+        )
     except Exception as exc:
         remote_error = str(exc)
 
-    resolved_ok = max(args.local_ok, remote_ok)
+    resolved_ok = remote_ok
-    resolved_failed = max(args.local_failed, remote_failed)
+    resolved_failed = remote_failed
-    coverage_percent = _read_coverage_percent(args.coverage_percent_file)
+    if remote_error:
+        resolved_ok = args.local_ok
+        resolved_failed = args.local_failed
+    elif not already_recorded:
+        resolved_ok += current_ok
+        resolved_failed += current_failed
+    coverage_percent = _read_coverage_percent(str(_resolve_repo_path(repo_root, args.coverage_percent_file)))
+    source_files_total = _count_source_files(repo_root)
     source_lines_over_500 = _count_source_files_over_limit(repo_root, max_lines=500)
-    test_output = Path(os.getenv("ANANKE_QUALITY_OUTPUT_FILE", str(build_dir / "quality-gate.out")))
+    quality_output = _resolve_repo_path(
-    tests = _parse_go_test_counts(test_output)
+        repo_root,
-    test_cases = _parse_go_test_cases(test_output)
+        os.getenv("ANANKE_QUALITY_OUTPUT_FILE", str(build_dir / "quality-gate.out")),
-    gate_rc = _read_exit_code(Path(os.getenv("ANANKE_QUALITY_EXIT_CODE_PATH", str(build_dir / "quality-gate.rc"))))
+    )
-    docs_status = _read_status(Path(os.getenv("ANANKE_QUALITY_DOCS_STATUS_PATH", str(build_dir / "docs-naming.status"))))
+    tests = _parse_go_test_counts(quality_output)
-    gate_failed = gate_rc != 0
+    test_cases = _parse_go_test_cases(quality_output)
+    docs_status = _read_status(
+        _resolve_repo_path(repo_root, os.getenv("ANANKE_QUALITY_DOCS_STATUS_PATH", str(build_dir / "docs-naming.status")))
+    )
+    unit_tests_failed = _unit_tests_failed(quality_output, coverage_percent)
     checks = {
-        "tests": "failed" if gate_failed or tests["failed"] > 0 else "ok",
+        "tests": "failed" if unit_tests_failed or tests["failed"] > 0 or tests["errors"] > 0 else "ok",
         "coverage": "ok" if coverage_percent >= 95.0 else "failed",
         "loc": "ok" if source_lines_over_500 == 0 else "failed",
         "docs_naming": docs_status,
@@ -332,7 +467,11 @@ def main(argv: list[str] | None = None) -> int:
         tests_skipped=tests["skipped"],
         test_cases=test_cases,
         coverage_percent=coverage_percent,
+        source_files_total=source_files_total,
         source_lines_over_500=source_lines_over_500,
+        branch=branch,
+        build_number=build_number,
+        jenkins_job=jenkins_job,
         checks=checks,
     )
 
@@ -345,7 +484,8 @@ def main(argv: list[str] | None = None) -> int:
 
     summary = (
         f"[quality] published Pushgateway metrics suite={args.suite} job={args.job_name} ok={resolved_ok} "
-        f"failed={resolved_failed} coverage={coverage_percent:.3f} source_lines_over_500={source_lines_over_500}"
+        f"failed={resolved_failed} coverage={coverage_percent:.3f} source_files_total={source_files_total} "
+        f"source_lines_over_500={source_lines_over_500}"
     )
     if remote_error:
         summary += f" remote_read_error={remote_error}"

scripts/publish_quality_metrics_test.py

@@ -3,8 +3,11 @@
 from __future__ import annotations
 
 import http.server
+from pathlib import Path
 import socketserver
+import tempfile
 import threading
+from unittest import mock
 import unittest
 
 import publish_quality_metrics as publisher
@@ -58,7 +61,34 @@ class PublishQualityMetricsTest(unittest.TestCase):
         self.server.server_close()
         self.thread.join(timeout=5)
 
-    def test_publish_uses_remote_high_water_mark(self) -> None:
+    def test_relative_artifact_paths_are_repo_rooted(self) -> None:
+        tmp_dir = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp_dir.cleanup)
+        repo_root = Path(tmp_dir.name) / "ananke-repo"
+        absolute_coverage_path = Path(tmp_dir.name) / "coverage-percent.txt"
+
+        self.assertEqual(
+            publisher._resolve_repo_path(repo_root, "build/coverage-percent.txt"),
+            repo_root / "build" / "coverage-percent.txt",
+        )
+        self.assertEqual(
+            publisher._resolve_repo_path(repo_root, str(absolute_coverage_path)),
+            absolute_coverage_path,
+        )
+
+    def _env_for_gate_status(self, status: int = 0) -> dict[str, str]:
+        tmp_dir = tempfile.TemporaryDirectory()
+        self.addCleanup(tmp_dir.cleanup)
+        rc_path = Path(tmp_dir.name) / "quality-gate.rc"
+        rc_path.write_text(f"{status}\n", encoding="utf-8")
+        return {
+            "ANANKE_QUALITY_EXIT_CODE_PATH": str(rc_path),
+            "ANANKE_QUALITY_COVERAGE_PERCENT_FILE": str(Path(tmp_dir.name) / "coverage.txt"),
+            "ANANKE_QUALITY_OUTPUT_FILE": str(Path(tmp_dir.name) / "quality-gate.out"),
+            "ANANKE_QUALITY_DOCS_STATUS_PATH": str(Path(tmp_dir.name) / "docs-naming.status"),
+        }
+
+    def test_publish_adds_current_run_to_remote_counters(self) -> None:
         _GatewayHandler.metrics_text = "\n".join(
             [
                 '# TYPE platform_quality_gate_runs_total counter',
@@ -67,59 +97,103 @@ class PublishQualityMetricsTest(unittest.TestCase):
             ]
         )
 
-        exit_code = publisher.main(
+        with mock.patch.dict("os.environ", {**self._env_for_gate_status(0), "BRANCH_NAME": "main"}):
-            [
+            exit_code = publisher.main(
-                "--pushgateway-url",
+                [
-                self.base_url,
+                    "--pushgateway-url",
-                "--job-name",
+                    self.base_url,
-                "platform-quality-ci",
+                    "--job-name",
-                "--suite",
+                    "platform-quality-ci",
-                "ananke",
+                    "--suite",
-                "--trigger",
+                    "ananke",
-                "host",
+                    "--trigger",
-                "--local-ok",
+                    "host",
-                "5",
+                    "--local-ok",
-                "--local-failed",
+                    "5",
-                "2",
+                    "--local-failed",
-            ]
+                    "2",
-        )
+                ]
+            )
 
         self.assertEqual(exit_code, 0)
         self.assertEqual(len(_GatewayHandler.posts), 1)
         path, body = _GatewayHandler.posts[0]
         self.assertEqual(path, "/metrics/job/platform-quality-ci/suite/ananke")
-        self.assertIn('platform_quality_gate_runs_total{suite="ananke",status="ok"} 7', body)
+        self.assertIn('platform_quality_gate_runs_total{suite="ananke",branch="main",status="ok"} 8', body)
-        self.assertIn('platform_quality_gate_runs_total{suite="ananke",status="failed"} 2', body)
+        self.assertIn('platform_quality_gate_runs_total{suite="ananke",branch="main",status="failed"} 1', body)
-        self.assertIn('ananke_quality_gate_publish_info{suite="ananke",trigger="host"} 1', body)
+        self.assertIn('ananke_quality_gate_publish_info{suite="ananke",branch="main",trigger="host"} 1', body)
-        self.assertIn('ananke_quality_gate_coverage_percent{suite="ananke"}', body)
+        self.assertIn('ananke_quality_gate_coverage_percent{suite="ananke",branch="main"}', body)
-        self.assertIn('platform_quality_gate_workspace_line_coverage_percent{suite="ananke"}', body)
+        self.assertIn('platform_quality_gate_workspace_line_coverage_percent{suite="ananke",branch="main"}', body)
-        self.assertIn('platform_quality_gate_source_lines_over_500_total{suite="ananke"}', body)
+        self.assertIn('platform_quality_gate_source_files_total{suite="ananke",branch="main"}', body)
+        self.assertIn('platform_quality_gate_source_lines_over_500_total{suite="ananke",branch="main"}', body)
+        self.assertIn('ananke_quality_gate_checks_total{suite="ananke",branch="main",check="tests",result=', body)
+
+    def test_publish_does_not_double_count_same_build(self) -> None:
+        _GatewayHandler.metrics_text = "\n".join(
+            [
+                'platform_quality_gate_runs_total{job="platform-quality-ci",suite="ananke",status="ok"} 7',
+                'platform_quality_gate_runs_total{job="platform-quality-ci",suite="ananke",status="failed"} 1',
+                'platform_quality_gate_build_info{job="platform-quality-ci",suite="ananke",branch="main",build_number="78",jenkins_job="ananke"} 1',
+            ]
+        )
+        with mock.patch.dict(
+            "os.environ",
+            {
+                **self._env_for_gate_status(0),
+                "BRANCH_NAME": "main",
+                "BUILD_NUMBER": "78",
+                "JOB_NAME": "ananke",
+            },
+        ):
+            exit_code = publisher.main(
+                [
+                    "--pushgateway-url",
+                    self.base_url,
+                    "--job-name",
+                    "platform-quality-ci",
+                    "--suite",
+                    "ananke",
+                    "--trigger",
+                    "host",
+                    "--local-ok",
+                    "1",
+                    "--local-failed",
+                    "0",
+                ]
+            )
+
+        self.assertEqual(exit_code, 0)
+        _, body = _GatewayHandler.posts[0]
+        self.assertIn('platform_quality_gate_runs_total{suite="ananke",branch="main",status="ok"} 7', body)
+        self.assertIn('platform_quality_gate_runs_total{suite="ananke",branch="main",status="failed"} 1', body)
 
     def test_publish_falls_back_to_local_counters_when_metrics_read_fails(self) -> None:
         _GatewayHandler.fail_metrics_read = True
 
-        exit_code = publisher.main(
+        with mock.patch.dict("os.environ", {**self._env_for_gate_status(0), "GIT_BRANCH": "origin/main"}):
-            [
+            exit_code = publisher.main(
-                "--pushgateway-url",
+                [
-                self.base_url,
+                    "--pushgateway-url",
-                "--job-name",
+                    self.base_url,
-                "platform-quality-ci",
+                    "--job-name",
-                "--suite",
+                    "platform-quality-ci",
-                "ananke",
+                    "--suite",
-                "--local-ok",
+                    "ananke",
-                "11",
+                    "--local-ok",
-                "--local-failed",
+                    "11",
-                "3",
+                    "--local-failed",
-            ]
+                    "3",
-        )
+                ]
+            )
 
         self.assertEqual(exit_code, 0)
         self.assertEqual(len(_GatewayHandler.posts), 1)
         _, body = _GatewayHandler.posts[0]
-        self.assertIn('platform_quality_gate_runs_total{suite="ananke",status="ok"} 11', body)
+        self.assertIn('platform_quality_gate_runs_total{suite="ananke",branch="main",status="ok"} 11', body)
-        self.assertIn('platform_quality_gate_runs_total{suite="ananke",status="failed"} 3', body)
+        self.assertIn('platform_quality_gate_runs_total{suite="ananke",branch="main",status="failed"} 3', body)
-        self.assertIn('platform_quality_gate_workspace_line_coverage_percent{suite="ananke"}', body)
+        self.assertIn('platform_quality_gate_workspace_line_coverage_percent{suite="ananke",branch="main"}', body)
-        self.assertIn('platform_quality_gate_source_lines_over_500_total{suite="ananke"}', body)
+        self.assertIn('platform_quality_gate_source_files_total{suite="ananke",branch="main"}', body)
+        self.assertIn('platform_quality_gate_source_lines_over_500_total{suite="ananke",branch="main"}', body)
 
 
 if __name__ == "__main__":

scripts/quality_gate.sh

@@ -56,12 +56,13 @@ read_quality_counter() {
 write_quality_metrics() {
   local exit_code="$1"
 
-  local metrics_dir state_dir
+  local metrics_dir state_dir write_metrics
   metrics_dir="$(dirname "${QUALITY_METRICS_FILE}")"
   state_dir="$(dirname "${QUALITY_STATE_FILE}")"
+  write_metrics="${QUALITY_METRICS_ENABLED}"
   mkdir -p "${state_dir}" >/dev/null 2>&1 || return 0
-  if [[ "${QUALITY_METRICS_ENABLED}" == "1" ]]; then
+  if [[ "${write_metrics}" == "1" ]]; then
-    mkdir -p "${metrics_dir}" >/dev/null 2>&1 || return 0
+    mkdir -p "${metrics_dir}" >/dev/null 2>&1 || write_metrics=0
   fi
 
   local ok failed total last_success now success_percent
@@ -84,12 +85,12 @@ write_quality_metrics() {
   QUALITY_SUCCESS_PERCENT="${success_percent}"
 
   local tmp_metrics="" tmp_state
-  if [[ "${QUALITY_METRICS_ENABLED}" == "1" ]]; then
+  if [[ "${write_metrics}" == "1" ]]; then
     tmp_metrics="$(mktemp "${metrics_dir}/quality-gate.prom.XXXXXX")"
   fi
   tmp_state="$(mktemp "${state_dir}/quality-gate.state.XXXXXX")"
 
-  if [[ "${QUALITY_METRICS_ENABLED}" == "1" ]]; then
+  if [[ "${write_metrics}" == "1" ]]; then
     cat > "${tmp_metrics}" <<EOF
 # HELP ananke_quality_gate_runs_total Total Ananke quality gate runs by status.
 # TYPE ananke_quality_gate_runs_total counter
@@ -115,7 +116,7 @@ last_run=${now}
 EOF
 
   mv -f "${tmp_state}" "${QUALITY_STATE_FILE}"
-  if [[ "${QUALITY_METRICS_ENABLED}" == "1" ]]; then
+  if [[ "${write_metrics}" == "1" ]]; then
     mv -f "${tmp_metrics}" "${QUALITY_METRICS_FILE}"
   fi
 }
@@ -146,6 +147,8 @@ publish_quality_metrics() {
 quality_gate_finalize() {
   local exit_code="$1"
   set +e
+  printf '%s\n' "${exit_code}" > "${BUILD_DIR}/quality-gate.rc" 2>/dev/null || true
+  cd "${REPO_DIR}" 2>/dev/null || true
   write_quality_metrics "${exit_code}" || true
   publish_quality_metrics || true
   exit "${exit_code}"
@@ -154,19 +157,18 @@ quality_gate_finalize() {
 trap 'quality_gate_finalize $?' EXIT
 
 cd "${REPO_DIR}"
+# Pin the resolved toolchain for child coverage commands. Auto-switching from an
+# older host Go can otherwise look for the removed covdata tool (Go issue 75031).
+if [[ "$(go env GOTOOLCHAIN)" == "auto" ]]; then
+  export GOTOOLCHAIN="$(go env GOVERSION)"
+fi
 mkdir -p "${BUILD_DIR}"
 rm -f "${COVERAGE_PROFILE}" "${COVERAGE_PERCENT_FILE}"
 printf 'failed\n' > "${BUILD_DIR}/docs-naming.status"
 
-echo "[quality] unit tests + workspace coverage profile"
+echo "[quality] dependency download"
 export GOPROXY="${GOPROXY:-https://proxy.golang.org,direct}"
 run_with_retry 4 go mod download
-run_with_retry 3 go test -coverprofile="${COVERAGE_PROFILE}" ./...
-coverage_percent="$(go tool cover -func="${COVERAGE_PROFILE}" | awk '/^total:/ {gsub("%","",$3); print $3}')"
-if [[ -z "${coverage_percent}" ]]; then
-  coverage_percent="0"
-fi
-printf '%s\n' "${coverage_percent}" > "${COVERAGE_PERCENT_FILE}"
 
 echo "[quality] hygiene: doc contracts"
 cd testing
@@ -189,6 +191,17 @@ echo "[quality] lint"
 echo "[quality] installer template contracts"
 ./scripts/verify_install_templates.sh
 
+echo "[quality] unit tests + workspace coverage profile"
+run_with_retry 3 go test -coverprofile="${COVERAGE_PROFILE}" ./...
+coverage_percent="$(go tool cover -func="${COVERAGE_PROFILE}" | awk '/^total:/ {gsub("%","",$3); print $3}')"
+if [[ -z "${coverage_percent}" ]]; then
+  coverage_percent="0"
+fi
+printf '%s\n' "${coverage_percent}" > "${COVERAGE_PERCENT_FILE}"
+
 echo "[quality] per-file coverage gate (95%)"
 cd testing
-ANANKE_ENFORCE_COVERAGE=1 ANANKE_PER_FILE_COVERAGE_TARGET=95 go test ./coverage -run TestPerFileCoverageReport -count=1 -v
+ANANKE_ENFORCE_COVERAGE=1 \
+  ANANKE_PER_FILE_COVERAGE_TARGET=95 \
+  ANANKE_PER_FILE_COVERAGE_PERCENT_FILE="${COVERAGE_PERCENT_FILE}" \
+  go test ./coverage -run TestPerFileCoverageReport -count=1 -v

testing/config/config_quality_matrix_test.go

@@ -51,15 +51,33 @@ func TestHookServiceCatalogAndMergeContracts(t *testing.T) {
 		t.Fatalf("expected critical endpoint defaults")
 	}
 	foundMonitoring := false
+	foundMailAdmin := false
 	for _, entry := range critical {
 		if entry == "monitoring/grafana" {
 			foundMonitoring = true
-			break
+		}
+		if entry == "mailu-mailserver/mailu-admin" {
+			foundMailAdmin = true
 		}
 	}
 	if !foundMonitoring {
 		t.Fatalf("expected monitoring/grafana critical endpoint default")
 	}
+	if !foundMailAdmin {
+		t.Fatalf("expected mailu-mailserver/mailu-admin critical endpoint default")
+	}
+
+	tcpChecks := icfg.TestHookDefaultTCPServiceChecklist()
+	seenTCP := map[string]icfg.TCPServiceChecklistCheck{}
+	for _, check := range tcpChecks {
+		seenTCP[strings.TrimSpace(check.Name)] = check
+	}
+	if seenTCP["mail-smtp"].Port != 25 || seenTCP["mail-smtp"].Service != "mailu-front" {
+		t.Fatalf("expected mail-smtp TCP default to target Mailu front")
+	}
+	if !seenTCP["mail-imaps"].TLS || seenTCP["mail-imaps"].Port != 993 {
+		t.Fatalf("expected mail-imaps TCP default with TLS on 993")
+	}
 
 	mergedChecks := icfg.TestHookMergeServiceChecklistDefaults(
 		[]icfg.ServiceChecklistCheck{
@@ -75,6 +93,20 @@ func TestHookServiceCatalogAndMergeContracts(t *testing.T) {
 		t.Fatalf("expected 3 merged checks with dedupe, got %d", len(mergedChecks))
 	}
 
+	mergedTCP := icfg.TestHookMergeTCPServiceChecklistDefaults(
+		[]icfg.TCPServiceChecklistCheck{
+			{Name: "custom-tcp", Host: "custom.bstein.dev", Port: 1234, TimeoutSeconds: 5},
+			{Name: "mail-smtp", Host: "override.invalid", Port: 25, TimeoutSeconds: 5},
+		},
+		[]icfg.TCPServiceChecklistCheck{
+			{Name: "mail-smtp", Host: "mail.bstein.dev", Port: 25, TimeoutSeconds: 5},
+			{Name: "mail-imaps", Host: "mail.bstein.dev", Port: 993, TimeoutSeconds: 5},
+		},
+	)
+	if len(mergedTCP) != 3 {
+		t.Fatalf("expected 3 merged TCP checks with dedupe, got %d", len(mergedTCP))
+	}
+
 	mergedStrings := icfg.TestHookMergeStringDefaults(
 		[]string{" one ", "one", "", "two"},
 		[]string{"two", "three", " "},
@@ -84,6 +116,33 @@ func TestHookServiceCatalogAndMergeContracts(t *testing.T) {
 	}
 }
 
+// TestTitanCoordinatorConfigInheritsMailHardeningDefaults runs one orchestration or CLI step.
+// Signature: TestTitanCoordinatorConfigInheritsMailHardeningDefaults(t *testing.T).
+// Why: the deployed coordinator config has a startup block, so new Mailu recovery
+// defaults must still be present after YAML decoding and default merging.
+func TestTitanCoordinatorConfigInheritsMailHardeningDefaults(t *testing.T) {
+	cfg, err := icfg.Load(filepath.Join("..", "..", "configs", "ananke.titan-db.yaml"))
+	if err != nil {
+		t.Fatalf("load titan coordinator config: %v", err)
+	}
+	if !cfg.Startup.CriticalServiceStartupProbeRepair {
+		t.Fatalf("expected titan config to inherit critical startup-probe repair")
+	}
+	seenMailAdmin := false
+	seenMailRspamd := false
+	for _, entry := range cfg.Startup.CriticalServiceEndpoints {
+		if entry == "mailu-mailserver/mailu-admin" {
+			seenMailAdmin = true
+		}
+		if entry == "mailu-mailserver/mailu-rspamd" {
+			seenMailRspamd = true
+		}
+	}
+	if !seenMailAdmin || !seenMailRspamd {
+		t.Fatalf("expected titan config to include Mailu critical endpoints, admin=%v rspamd=%v", seenMailAdmin, seenMailRspamd)
+	}
+}
+
 // TestValidateServiceChecklistAuthContracts runs one orchestration or CLI step.
 // Signature: TestValidateServiceChecklistAuthContracts(t *testing.T).
 // Why: covers service-checklist auth and final-url validation branches that are

testing/coverage/coverage_test.go

@@ -121,6 +121,7 @@ func TestPerFileCoverageReport(t *testing.T) {
 		"./metrics",
 		"./hygiene",
 		"./orchestrator",
+		"./service",
 		"./state",
 		"./sshutil",
 		"./ups",
@@ -167,17 +168,26 @@ func TestPerFileCoverageReport(t *testing.T) {
 	var report bytes.Buffer
 	report.WriteString("per-file coverage\n")
 	under := make([]string, 0)
+	minPercent := 100.0
 	for _, file := range keys {
 		fc := byFile[file]
 		pct := 100.0
 		if fc.total > 0 {
 			pct = (fc.covered / fc.total) * 100.0
 		}
+		if pct < minPercent {
+			minPercent = pct
+		}
 		report.WriteString(fmt.Sprintf("- %s: %.1f%%\n", file, pct))
 		if pct < target {
 			under = append(under, fmt.Sprintf("%s (%.1f%% < %.1f%%)", file, pct, target))
 		}
 	}
+	if outputPath := strings.TrimSpace(os.Getenv("ANANKE_PER_FILE_COVERAGE_PERCENT_FILE")); outputPath != "" {
+		if err := os.WriteFile(outputPath, []byte(fmt.Sprintf("%.1f\n", minPercent)), 0o644); err != nil {
+			t.Fatalf("write per-file coverage percent: %v", err)
+		}
+	}
 	t.Log(report.String())
 
 	if len(under) > 0 && enforce {