From a9ddc80e363ab2877b3a7c84ade16b4dd34c4dd5 Mon Sep 17 00:00:00 2001 From: jenkins Date: Thu, 18 Jun 2026 22:08:14 -0300 Subject: [PATCH] recovery(ananke): keep flux holds and place metrics on longhorn nodes --- scripts/cluster_power_recovery.sh | 132 +++++++++++++++++++++------ services/monitoring/helmrelease.yaml | 6 ++ 2 files changed, 109 insertions(+), 29 deletions(-) diff --git a/scripts/cluster_power_recovery.sh b/scripts/cluster_power_recovery.sh index 49afefb7..856cbcd0 100755 --- a/scripts/cluster_power_recovery.sh +++ b/scripts/cluster_power_recovery.sh @@ -16,7 +16,7 @@ fi usage() { cat < [options] + scripts/cluster_power_recovery.sh [options] Options: --execute Actually run commands (default is dry-run) @@ -81,6 +81,7 @@ Examples: scripts/cluster_power_recovery.sh bootstrap-seed --execute scripts/cluster_power_recovery.sh harbor-seed --execute scripts/cluster_power_recovery.sh longhorn-unlock --execute + scripts/cluster_power_recovery.sh flux-hold --execute scripts/cluster_power_recovery.sh status scripts/cluster_power_recovery.sh shutdown --execute scripts/cluster_power_recovery.sh startup --execute --force-flux-branch main @@ -95,7 +96,7 @@ fi shift || true case "${MODE}" in - prepare|status|bootstrap-seed|harbor-seed|longhorn-seed|longhorn-unlock|shutdown|startup) ;; + prepare|status|bootstrap-seed|harbor-seed|longhorn-seed|longhorn-unlock|flux-hold|shutdown|startup) ;; *) echo "Unknown mode: ${MODE}" >&2 usage @@ -147,6 +148,14 @@ RECOVERY_FLUX_ROOT_APPLY_TIMEOUT="${RECOVERY_FLUX_ROOT_APPLY_TIMEOUT:-15m}" RECOVERY_FLUX_SUSPEND_VERIFY_ATTEMPTS="${RECOVERY_FLUX_SUSPEND_VERIFY_ATTEMPTS:-6}" RECOVERY_FLUX_SUSPEND_VERIFY_SLEEP_SECONDS="${RECOVERY_FLUX_SUSPEND_VERIFY_SLEEP_SECONDS:-10}" RECOVERY_FLUX_FINAL_RESTART_KUSTOMIZE_CONTROLLER="${RECOVERY_FLUX_FINAL_RESTART_KUSTOMIZE_CONTROLLER:-0}" +RECOVERY_FLUX_PATCH_VERIFY_ATTEMPTS="${RECOVERY_FLUX_PATCH_VERIFY_ATTEMPTS:-3}" +RECOVERY_FLUX_PATCH_VERIFY_SLEEP_SECONDS="${RECOVERY_FLUX_PATCH_VERIFY_SLEEP_SECONDS:-1}" +RECOVERY_FLUX_FINAL_STABILITY_SECONDS="${RECOVERY_FLUX_FINAL_STABILITY_SECONDS:-45}" +RECOVERY_FLUX_FINAL_STABILITY_POLL_SECONDS="${RECOVERY_FLUX_FINAL_STABILITY_POLL_SECONDS:-5}" +RECOVERY_FLUX_FINAL_STABILITY_TIMEOUT_SECONDS="${RECOVERY_FLUX_FINAL_STABILITY_TIMEOUT_SECONDS:-300}" +RECOVERY_FLUX_FINAL_STOP_AUX_CONTROLLERS="${RECOVERY_FLUX_FINAL_STOP_AUX_CONTROLLERS:-1}" +RECOVERY_FLUX_AUX_CONTROLLERS="${RECOVERY_FLUX_AUX_CONTROLLERS:-image-automation-controller,image-reflector-controller,notification-controller}" +RECOVERY_KUBECTL_FIELD_MANAGER="${RECOVERY_KUBECTL_FIELD_MANAGER:-ananke-recovery-hold}" STARTUP_SERVICE_CHECK_TIMEOUT_SECONDS="${STARTUP_SERVICE_CHECK_TIMEOUT_SECONDS:-10}" STARTUP_SERVICE_CHECKLIST="${STARTUP_SERVICE_CHECKLIST:-}" STARTUP_INCLUDE_INGRESS_CHECKS="${STARTUP_INCLUDE_INGRESS_CHECKS:-1}" @@ -953,24 +962,44 @@ patch_flux_suspend_all() { while IFS= read -r k; do [[ -z "${k}" ]] && continue - run kubectl -n flux-system patch kustomization "${k}" --type=merge -p "${patch}" + patch_kustomization_suspend "${k}" "${value}" done <<< "${ks_list}" while IFS= read -r hr; do [[ -z "${hr}" ]] && continue local ns="${hr%%/*}" local name="${hr##*/}" - run kubectl -n "${ns}" patch helmrelease "${name}" --type=merge -p "${patch}" + run kubectl -n "${ns}" patch helmrelease "${name}" --field-manager="${RECOVERY_KUBECTL_FIELD_MANAGER}" --type=merge -p "${patch}" done <<< "${hr_list}" } +apply_kustomization_suspend_field() { + local name="$1" + local value="$2" + + if [[ "${EXECUTE}" -eq 1 ]]; then + log "EXEC: kubectl apply --server-side --force-conflicts --field-manager=${RECOVERY_KUBECTL_FIELD_MANAGER} kustomization/${name} suspend=${value}" + printf 'apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\nmetadata:\n name: %s\n namespace: flux-system\nspec:\n suspend: %s\n' "${name}" "${value}" \ + | kubectl apply --server-side --force-conflicts --field-manager="${RECOVERY_KUBECTL_FIELD_MANAGER}" -f - + else + log "DRY-RUN: kubectl apply --server-side --force-conflicts --field-manager=${RECOVERY_KUBECTL_FIELD_MANAGER} kustomization/${name} suspend=${value}" + fi +} + patch_kustomization_suspend() { local name="$1" local value="$2" - local patch - patch=$(printf '{"spec":{"suspend":%s}}' "${value}") if kubectl -n flux-system get kustomization "${name}" >/dev/null 2>&1; then - run kubectl -n flux-system patch kustomization "${name}" --type=merge -p "${patch}" + apply_kustomization_suspend_field "${name}" "${value}" + if [[ "${EXECUTE}" -eq 1 && "${value}" == "true" ]]; then + local attempt observed + for attempt in $(seq 1 "${RECOVERY_FLUX_PATCH_VERIFY_ATTEMPTS}"); do + observed="$(kubectl -n flux-system get kustomization "${name}" -o jsonpath='{.spec.suspend}' 2>/dev/null || true)" + [[ "${observed}" == "true" ]] && return 0 + sleep "${RECOVERY_FLUX_PATCH_VERIFY_SLEEP_SECONDS}" + done + warn "Flux Kustomization ${name} suspend=true did not verify after patch; observed=${observed:-missing}." + fi else warn "Flux Kustomization ${name} not found; skipping suspend=${value}." fi @@ -1062,6 +1091,61 @@ recovery_flux_unsuspended_list() { done } +reassert_recovery_flux_suspend_hold() { + if [[ "${RECOVERY_FLUX_SUSPEND_BOOTSTRAP_KUSTOMIZATION}" == "1" || "${RECOVERY_FLUX_SUSPEND_BOOTSTRAP_KUSTOMIZATION}" == "true" ]]; then + patch_kustomization_suspend flux-system true + fi + patch_recovery_optional_flux_suspend_without_snapshot true +} + +verify_recovery_flux_suspend_stable_window() { + [[ "${EXECUTE}" -eq 1 ]] || return 0 + + local deadline stable_since stable_for unsuspended + deadline=$((SECONDS + RECOVERY_FLUX_FINAL_STABILITY_TIMEOUT_SECONDS)) + stable_since=0 + + reassert_recovery_flux_suspend_hold + + while (( SECONDS < deadline )); do + unsuspended="$(recovery_flux_unsuspended_list | paste -sd, -)" + if [[ -n "${unsuspended}" ]]; then + warn "Flux suspend hold was overwritten; reasserting recovery hold: ${unsuspended}" + reassert_recovery_flux_suspend_hold + stable_since=0 + sleep "${RECOVERY_FLUX_FINAL_STABILITY_POLL_SECONDS}" + continue + fi + + if (( stable_since == 0 )); then + stable_since="${SECONDS}" + fi + stable_for=$((SECONDS - stable_since)) + if (( stable_for >= RECOVERY_FLUX_FINAL_STABILITY_SECONDS )); then + log "recovery-flux-suspend=stable seconds=${stable_for}" + return 0 + fi + sleep "${RECOVERY_FLUX_FINAL_STABILITY_POLL_SECONDS}" + done + + unsuspended="$(recovery_flux_unsuspended_list | paste -sd, -)" + warn "Timed out waiting for stable Flux suspend hold after ${RECOVERY_FLUX_FINAL_STABILITY_TIMEOUT_SECONDS}s: ${unsuspended:-none-unsuspended-now}" + return 1 +} + +stop_recovery_flux_aux_controllers() { + [[ "${EXECUTE}" -eq 1 ]] || return 0 + [[ "${RECOVERY_FLUX_FINAL_STOP_AUX_CONTROLLERS}" == "1" || "${RECOVERY_FLUX_FINAL_STOP_AUX_CONTROLLERS}" == "true" ]] || return 0 + + local controller + while IFS= read -r controller; do + if kubectl -n flux-system get deployment "${controller}" >/dev/null 2>&1; then + warn "Stopping Flux auxiliary controller ${controller} for recovery hold." + run kubectl -n flux-system scale deployment "${controller}" --replicas=0 + fi + done < <(csv_each "${RECOVERY_FLUX_AUX_CONTROLLERS}") +} + wait_for_kustomize_controller_scaled_down() { [[ "${EXECUTE}" -eq 1 ]] || return 0 @@ -1089,12 +1173,10 @@ force_recovery_flux_suspend_with_controller_stop() { warn "Stopping kustomize-controller for final Flux suspend reassertion." run kubectl -n flux-system scale deployment kustomize-controller --replicas=0 + stop_recovery_flux_aux_controllers wait_for_kustomize_controller_scaled_down || true - if [[ "${RECOVERY_FLUX_SUSPEND_BOOTSTRAP_KUSTOMIZATION}" == "1" || "${RECOVERY_FLUX_SUSPEND_BOOTSTRAP_KUSTOMIZATION}" == "true" ]]; then - patch_kustomization_suspend flux-system true - fi - patch_recovery_optional_flux_suspend_without_snapshot true + reassert_recovery_flux_suspend_hold if [[ "${RECOVERY_FLUX_FINAL_RESTART_KUSTOMIZE_CONTROLLER}" == "1" || "${RECOVERY_FLUX_FINAL_RESTART_KUSTOMIZE_CONTROLLER}" == "true" ]]; then run kubectl -n flux-system scale deployment kustomize-controller --replicas=1 @@ -1102,19 +1184,9 @@ force_recovery_flux_suspend_with_controller_stop() { sleep "${RECOVERY_FLUX_SUSPEND_VERIFY_SLEEP_SECONDS}" else warn "Leaving kustomize-controller stopped to preserve the recovery Flux hold." - sleep "${RECOVERY_FLUX_SUSPEND_VERIFY_SLEEP_SECONDS}" - if [[ "${RECOVERY_FLUX_SUSPEND_BOOTSTRAP_KUSTOMIZATION}" == "1" || "${RECOVERY_FLUX_SUSPEND_BOOTSTRAP_KUSTOMIZATION}" == "true" ]]; then - patch_kustomization_suspend flux-system true - fi - patch_recovery_optional_flux_suspend_without_snapshot true fi - local unsuspended - unsuspended="$(recovery_flux_unsuspended_list | paste -sd, -)" - if [[ -n "${unsuspended}" ]]; then - warn "Flux suspend state is still not stable after controller-stop finalization: ${unsuspended}" - return 1 - fi + verify_recovery_flux_suspend_stable_window || return 1 log "recovery-flux-suspend=verified-controller-stop" } @@ -1124,16 +1196,15 @@ stabilize_recovery_flux_suspend() { local attempt unsuspended for attempt in $(seq 1 "${RECOVERY_FLUX_SUSPEND_VERIFY_ATTEMPTS}"); do - if [[ "${RECOVERY_FLUX_SUSPEND_BOOTSTRAP_KUSTOMIZATION}" == "1" || "${RECOVERY_FLUX_SUSPEND_BOOTSTRAP_KUSTOMIZATION}" == "true" ]]; then - patch_kustomization_suspend flux-system true - fi - patch_recovery_optional_flux_suspend_without_snapshot true + reassert_recovery_flux_suspend_hold sleep "${RECOVERY_FLUX_SUSPEND_VERIFY_SLEEP_SECONDS}" unsuspended="$(recovery_flux_unsuspended_list | paste -sd, -)" if [[ -z "${unsuspended}" ]]; then - log "recovery-flux-suspend=verified attempts=${attempt}" - return 0 + verify_recovery_flux_suspend_stable_window && { + log "recovery-flux-suspend=verified attempts=${attempt}" + return 0 + } fi warn "Flux suspend state was overwritten during recovery thaw; reasserting attempt ${attempt}/${RECOVERY_FLUX_SUSPEND_VERIFY_ATTEMPTS}: ${unsuspended}" done @@ -1211,7 +1282,7 @@ patch_helmrelease_suspend() { local patch patch=$(printf '{"spec":{"suspend":%s}}' "${value}") if kubectl -n "${namespace}" get helmrelease "${name}" >/dev/null 2>&1; then - run kubectl -n "${namespace}" patch helmrelease "${name}" --type=merge -p "${patch}" + run kubectl -n "${namespace}" patch helmrelease "${name}" --field-manager="${RECOVERY_KUBECTL_FIELD_MANAGER}" --type=merge -p "${patch}" else warn "HelmRelease ${namespace}/${name} not found; skipping suspend=${value}." fi @@ -3318,6 +3389,9 @@ case "${MODE}" in longhorn-unlock) longhorn_unlock_flow ;; + flux-hold) + force_recovery_flux_suspend_with_controller_stop + ;; shutdown) planned_shutdown ;; diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml index e3ffdbc5..f9cf8db5 100644 --- a/services/monitoring/helmrelease.yaml +++ b/services/monitoring/helmrelease.yaml @@ -75,6 +75,8 @@ metadata: namespace: monitoring spec: interval: 15m + upgrade: + disableWait: true chart: spec: chart: victoria-metrics-single @@ -98,6 +100,10 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: + - key: longhorn-host + operator: In + values: + - "true" - key: kubernetes.io/hostname operator: NotIn values: