#!/usr/bin/env fish # Flux "hammer": disable KS/HR gates, force reconciling to latest Git/Helm artifacts, # and optionally delete a StatefulSet to unblock immutable changes. # Requires: fish 3.x+, kubectl, flux >= 2.x # -------- colors / helpers -------- set -l RED (set_color red); set -l GRN (set_color green) set -l YEL (set_color yellow); set -l CYA (set_color cyan) set -l RST (set_color normal) function _info; echo "$CYA$argv$RST"; end function _ok; echo "$GRN$argv$RST"; end function _warn; echo "$YEL$argv$RST"; end function _err; echo "$RED$argv$RST" >&2; end function _dbg if test "$VERBOSE" = "1" echo "$argv" end end # -------- check argparse exists -------- if not type -q argparse _err "fish >= 3.2 with 'argparse' builtin is required. You have: "(fish --version) exit 1 end # -------- small helper -------- function _split_ref -a ref default_ns if test -z "$ref" # Return two empty lines so callers can always read [1] and [2] printf "\n\n" return end if string match -rq '/' -- "$ref" # Split only once at the first slash set -l parts (string split -m 1 '/' -- "$ref") # IMPORTANT: print each piece on its own line so command substitution # returns two separate list items. printf "%s\n%s\n" $parts[1] $parts[2] else printf "%s\n%s\n" $default_ns $ref end end # ---- robust arg parse (short+long), then echo raw flags when --verbose ------- set -l KS ""; set -l HR ""; set -l REPO ""; set -l STS "" set -l VERBOSE 0; set -l NO_KS 0; set -l NO_HR 0; set -l DELETE_STS 0; set -l RESTORE 0 # Use both short and long names; fish creates _flag_ variables. argparse \ 'h/help' \ 'v/verbose' \ 'K/ks=' \ 'H/hr=' \ 'R/helmrepo=' \ 'S/sts=' \ 'D/delete-sts' \ 'r/restore' \ 'X/no-ks' \ 'Y/no-hr' -- $argv or begin _err "argparse failed (bad flags/values?)" exit 1 end # Copy flags into our own variables (arrays safe; take first element) set -q _flag_verbose; and set VERBOSE 1 set -q _flag_ks; and set KS "$_flag_ks[1]" set -q _flag_hr; and set HR "$_flag_hr[1]" set -q _flag_helmrepo; and set REPO "$_flag_helmrepo[1]" set -q _flag_sts; and set STS "$_flag_sts[1]" set -q _flag_delete_sts; and set DELETE_STS 1 set -q _flag_restore; and set RESTORE 1 set -q _flag_no_ks; and set NO_KS 1 set -q _flag_no_hr; and set NO_HR 1 if test $VERBOSE -eq 1 _info "raw flags:" _info " KS='$KS' HR='$HR' REPO='$REPO' STS='$STS'" _info " DELETE_STS=$DELETE_STS RESTORE=$RESTORE NO_KS=$NO_KS NO_HR=$NO_HR" end # ---- resolve refs ----------------------------------------------------------- set -l ks_ns ""; set -l ks_name "" if test -n "$KS" set -l t (_split_ref "$KS" flux-system); set ks_ns $t[1]; set ks_name $t[2] set ks_ns $t[1]; set ks_name $t[2] end set -l hr_ns ""; set -l hr_name "" if test -n "$HR" set -l t (_split_ref "$HR" default); set hr_ns $t[1]; set hr_name $t[2] set hr_ns $t[1]; set hr_name $t[2] end # convenience for common case (vault) if test "$hr_ns" = "default" -a "$hr_name" = "vault" set hr_ns vault end set -l repo_ns ""; set -l repo_name "" if test -n "$REPO" set -l t (_split_ref "$REPO" flux-system); set repo_ns $t[1]; set repo_name $t[2] set repo_ns $t[1]; set repo_name $t[2] end set -l sts_name "$STS" if test "$VERBOSE" = "1" _info "resolve: KS='$KS' → ks_ns='$ks_ns' ks_name='$ks_name'" _info "resolve: HR='$HR' → hr_ns='$hr_ns' hr_name='$hr_name'" _info "resolve: REPO='$REPO' → repo_ns='$repo_ns' repo_name='$repo_name'" end # ---- banner ----------------------------------------------------------------- set -l FLUX_VER (flux --version 2>/dev/null | string trim) set -l KCTX (kubectl config current-context) _info "► Flux: $FLUX_VER" _info "► Kube context: $KCTX" # Build nice single-line targets string (avoid odd spacing) set -l ks_disp "(none)" set -l hr_disp "(none)" set -l repo_disp "(none)" set -l sts_disp "(auto)" if test -n "$ks_name" set ks_disp "$ks_ns/$ks_name" end if test -n "$hr_name" set hr_disp "$hr_ns/$hr_name" end if test -n "$repo_name" set repo_disp "$repo_ns/$repo_name" end if test -n "$sts_name" if test -n "$hr_ns" set sts_disp "$hr_ns/$sts_name" else set sts_disp "$sts_name" end end _info "► Targets: KS=$ks_disp HR=$hr_disp REPO=$repo_disp STS=$sts_disp delete-sts=$DELETE_STS verbose=$VERBOSE" # Hard stop if nothing was requested to hammer if test -z "$ks_name" -a -z "$hr_name" _err "No --ks or --hr specified. Nothing to hammer." exit 1 end # -------- backup specs (best‑effort) -------- set -l BACK_DIR (mktemp -d /tmp/flux-hammer.XXXXXX) if test -n "$ks_name" _dbg "+ kubectl -n $ks_ns get kustomization $ks_name -o yaml > $BACK_DIR/ks-$ks_ns-$ks_name.yaml" kubectl -n "$ks_ns" get kustomization "$ks_name" -o yaml >"$BACK_DIR/ks-$ks_ns-$ks_name.yaml" 2>/dev/null end if test -n "$hr_name" _dbg "+ kubectl -n $hr_ns get helmrelease $hr_name -o yaml > $BACK_DIR/hr-$hr_ns-$hr_name.yaml" kubectl -n "$hr_ns" get helmrelease "$hr_name" -o yaml >"$BACK_DIR/hr-$hr_ns-$hr_name.yaml" 2>/dev/null end _ok "✔ Backed up current specs to $BACK_DIR" # -------- suspend (so gates don’t fight us) -------- if test $NO_KS -eq 0 -a -n "$ks_name" _dbg "+ flux suspend kustomization $ks_name -n $ks_ns" flux suspend kustomization $ks_name -n $ks_ns >/dev/null 2>&1 end if test $NO_HR -eq 0 -a -n "$hr_name" _dbg "+ flux suspend helmrelease $hr_name -n $hr_ns" flux suspend helmrelease $hr_name -n $hr_ns >/dev/null 2>&1 end _ok "✔ Suspended (best‑effort)" # -------- KS: disable ALL gates & demand reconcile -------- if test $NO_KS -eq 0 -a -n "$ks_name" _info "► KS: gate‑off patch (spec.force=true, spec.wait=false, spec.healthChecks=[])" _dbg "+ kubectl -n $ks_ns patch kustomization $ks_name --type=merge -p '{\"spec\":{\"force\":true,\"wait\":false,\"healthChecks\":[]}}'" kubectl -n $ks_ns patch kustomization $ks_name --type=merge -p \ '{"spec":{"force":true,"wait":false,"healthChecks":[]}}' >/dev/null 2>&1 set -l now (date --iso-8601=seconds) _dbg "+ annotate KS requestedAt=$now forceAt=$now" kubectl -n $ks_ns annotate kustomization $ks_name reconcile.fluxcd.io/requestedAt="$now" --overwrite >/dev/null 2>&1 kubectl -n $ks_ns annotate kustomization $ks_name reconcile.fluxcd.io/forceAt="$now" --overwrite >/dev/null 2>&1 end # -------- HR: force upgrade & disable waits -------- if test $NO_HR -eq 0 -a -n "$hr_name" _info "► HR: force patch (upgrade.force, install/upgrade.disableWait, retries=0)" _dbg "+ kubectl -n $hr_ns patch helmrelease $hr_name ... (disableWait/force)" kubectl -n $hr_ns patch helmrelease $hr_name --type=merge -p \ '{"spec":{"upgrade":{"force":true,"disableWait":true,"remediation":{"retries":0}},"install":{"disableWait":true}}}' >/dev/null 2>&1 set -l now (date --iso-8601=seconds) _dbg "+ annotate HR requestedAt=$now forceAt=$now" kubectl -n $hr_ns annotate helmrelease $hr_name reconcile.fluxcd.io/requestedAt="$now" --overwrite >/dev/null 2>&1 kubectl -n $hr_ns annotate helmrelease $hr_name reconcile.fluxcd.io/forceAt="$now" --overwrite >/dev/null 2>&1 end # -------- reconcile sources first -------- _info "► Reconcile GitRepository flux-system/flux-system" _dbg "+ flux reconcile source git flux-system -n flux-system --timeout=2m" flux reconcile source git flux-system -n flux-system --timeout=2m >/dev/null 2>&1 if test -n "$repo_name" _info "► Reconcile HelmRepository $repo_ns/$repo_name" _dbg "+ flux reconcile source helm $repo_name -n $repo_ns --timeout=2m" flux reconcile source helm $repo_name -n $repo_ns --timeout=2m >/dev/null 2>&1 end # -------- resume KS and force it to pick newest artifact -------- set -l latest_rev (kubectl -n flux-system get gitrepository flux-system -o jsonpath='{.status.artifact.revision}' 2>/dev/null) if test $NO_KS -eq 0 -a -n "$ks_name" _info "► KS: resume + reconcile (with source)" _dbg "+ flux resume kustomization $ks_name -n $ks_ns" flux resume kustomization $ks_name -n $ks_ns >/dev/null 2>&1 _dbg "+ flux reconcile kustomization $ks_name -n $ks_ns --with-source --timeout=5m" flux reconcile kustomization $ks_name -n $ks_ns --with-source --timeout=5m >/dev/null 2>&1 # Poll for newest revision (don’t block on health) set -l okrev 0 for i in (seq 1 30) set -l ksrev (kubectl -n $ks_ns get kustomization $ks_name -o jsonpath='{.status.lastAppliedRevision}' 2>/dev/null) _dbg " poll[$i]: latest_rev='$latest_rev' ksrev='$ksrev'" if test -n "$ksrev" if string match -rq -- "$latest_rev" "$ksrev" set okrev 1; break end end # extra nudge mid‑way if test $i -eq 12 -o $i -eq 24 set -l nonce (date +%s) _dbg " nudge: set annotation hammer.nonce=$nonce and reconcile again" kubectl -n $ks_ns patch kustomization $ks_name --type=merge -p \ '{"metadata":{"annotations":{"hammer.nonce":"'"$nonce"'"}}}' >/dev/null 2>&1 flux reconcile kustomization $ks_name -n $ks_ns --with-source --timeout=5m >/dev/null 2>&1 end sleep 2 end if test $okrev -eq 1 _ok "✔ KS now shows revision: $latest_rev" else _warn "⚠ KS has not reported $latest_rev; gates are OFF so the apply should still proceed." end end # -------- optionally delete the StatefulSet (PVCs retained) -------- if test $NO_HR -eq 0 -a -n "$hr_name" -a $DELETE_STS -eq 1 set -l sts_to_del "$sts_name" if test -z "$sts_to_del" _dbg "auto‑detect STS: by instance/name/app labels" set sts_to_del (kubectl -n $hr_ns get sts -l app.kubernetes.io/instance=$hr_name -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) if test -z "$sts_to_del" set sts_to_del (kubectl -n $hr_ns get sts $hr_name -o jsonpath='{.metadata.name}' 2>/dev/null) end if test -z "$sts_to_del" set sts_to_del (kubectl -n $hr_ns get sts -l app.kubernetes.io/name=$hr_name -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) end end if test -n "$sts_to_del" _warn "⚠ Deleting StatefulSet $hr_ns/$sts_to_del (PVCs retained)." _dbg "+ kubectl -n $hr_ns scale sts $sts_to_del --replicas=0 --timeout=2m" kubectl -n $hr_ns scale sts $sts_to_del --replicas=0 --timeout=2m >/dev/null 2>&1 _dbg "+ kubectl -n $hr_ns delete sts $sts_to_del --wait=true" kubectl -n $hr_ns delete sts $sts_to_del --wait=true >/dev/null 2>&1 else _warn "⚠ Could not detect a StatefulSet; use --sts if needed." end end # -------- resume HR and reconcile (try --force; fallback) -------- if test $NO_HR -eq 0 -a -n "$hr_name" _info "► HR: resume + reconcile (with source, try --force)" _dbg "+ flux resume helmrelease $hr_name -n $hr_ns" flux resume helmrelease $hr_name -n $hr_ns >/dev/null 2>&1 _dbg "+ flux reconcile helmrelease $hr_name -n $hr_ns --with-source --force --timeout=10m" flux reconcile helmrelease $hr_name -n $hr_ns --with-source --force --timeout=10m >/dev/null 2>&1 set -l rc $status if test $rc -ne 0 _warn "HR reconcile with --force failed (likely unsupported in this build); retrying without --force." _dbg "+ flux reconcile helmrelease $hr_name -n $hr_ns --with-source --timeout=10m" flux reconcile helmrelease $hr_name -n $hr_ns --with-source --timeout=10m >/dev/null 2>&1 end end # -------- status summary -------- _info "► Status:" if test $NO_KS -eq 0 -a -n "$ks_name" flux get kustomizations $ks_name -n $ks_ns end if test $NO_HR -eq 0 -a -n "$hr_name" flux get helmreleases -n $hr_ns | string match -r -- "NAME|$hr_name" end # -------- optional restore (turn gates back on from backup) -------- if test $RESTORE -eq 1 _info "► Restoring original specs" if test -n "$ks_name" -a -s "$BACK_DIR/ks-$ks_ns-$ks_name.yaml" _dbg "+ kubectl apply -f $BACK_DIR/ks-$ks_ns-$ks_name.yaml" kubectl apply -f "$BACK_DIR/ks-$ks_ns-$ks_name.yaml" >/dev/null 2>&1 end if test -n "$hr_name" -a -s "$BACK_DIR/hr-$hr_ns-$hr_name.yaml" _dbg "+ kubectl apply -f $BACK_DIR/hr-$hr_ns-$hr_name.yaml" kubectl apply -f "$BACK_DIR/hr-$hr_ns-$hr_name.yaml" >/dev/null 2>&1 end _ok "✔ Restored" else _warn "⚠ Not restoring KS/HR patches; gates remain OFF until you revert them in Git or re‑patch." end _ok "✔ Done."