diff --git a/scripts/hammer.fish b/scripts/hammer.fish new file mode 100755 index 0000000..31fb98c --- /dev/null +++ b/scripts/hammer.fish @@ -0,0 +1,104 @@ +#!/usr/bin/env fish +# Purpose: "Nuclear" reconcile for a target KS/HR by disabling gates, forcing update, then (optionally) restoring. + +# Defaults (edit if you want different names/namespaces) +set -gx KS_NAME "vault" +set -gx KS_NS "flux-system" +set -gx HR_NAME "vault" +set -gx HR_NS "vault" +set -gx GIT_SRC "flux-system" # GitRepository name +set -gx SRC_NS "flux-system" # Namespace for sources (Git/Helm) +set -gx HELM_REPO "hashicorp" # HelmRepository name + +# Behavior flags +set -gx RESTORE "no" +if test (count $argv) -gt 0 + if test "$argv[1]" = "--restore" + set RESTORE "yes" + end +end + +function info; printf "► %s\n" $argv; end +function ok; printf "✔ %s\n" $argv; end +function warn; printf "⚠ %s\n" $argv; end + +info "Flux: "(flux -v) +info "Kube context: "(kubectl config current-context) + +# --- 0) Preflight & backups --------------------------------------------------- +mkdir -p /tmp/flux-hammer +set KS_BAK "/tmp/flux-hammer/ks-$KS_NS-$KS_NAME.yaml" +set HR_BAK "/tmp/flux-hammer/hr-$HR_NS-$HR_NAME.yaml" + +kubectl -n $KS_NS get kustomization $KS_NAME -o yaml > $KS_BAK 2>/dev/null; or true +kubectl -n $HR_NS get helmrelease $HR_NAME -o yaml > $HR_BAK 2>/dev/null; or true +ok "Backed up current specs to $KS_BAK and $HR_BAK" + +# --- 1) Suspend --------------------------------------------------------------- +info "Suspending Kustomization $KS_NS/$KS_NAME and HelmRelease $HR_NS/$HR_NAME" +flux suspend kustomization $KS_NAME -n $KS_NS; or true +flux suspend helmrelease $HR_NAME -n $HR_NS; or true +ok "Suspended (best-effort)" + +# --- 2) Patch Kustomization: turn off waits/health & enable force ------------- +info "Patching Kustomization (force=true, wait=false)" +kubectl -n $KS_NS patch kustomization $KS_NAME --type merge \ + -p '{"spec":{"force":true,"wait":false}}'; or true + +# Remove healthChecks if present +set HAS_HC (kubectl -n $KS_NS get kustomization $KS_NAME -o json | jq -r '((.spec // {}) | has("healthChecks"))') +if test "$HAS_HC" = "true" + info "Removing .spec.healthChecks from Kustomization" + kubectl -n $KS_NS patch kustomization $KS_NAME --type json \ + -p='[{"op":"remove","path":"/spec/healthChecks"}]'; or true +else + ok "No .spec.healthChecks present" +end + +# --- 3) Patch HelmRelease: disable waits/tests & stop auto-remediation -------- +info "Patching HelmRelease (disable waits & retries)" +kubectl -n $HR_NS patch helmrelease $HR_NAME --type merge -p \ +'{ + "spec": { + "install": { "disableWait": true, "remediation": { "retries": 0 } }, + "upgrade": { "disableWait": true, "remediation": { "retries": 0 } }, + "rollback": { "disableWait": true }, + "test": { "enable": false } + } +}'; or true + +# --- 4) Reconcile sources (Git & Helm repo) ---------------------------------- +info "Reconciling sources" +flux reconcile source git $GIT_SRC -n $SRC_NS --timeout=2m; or true +flux reconcile source helm $HELM_REPO -n $SRC_NS --timeout=2m; or true +ok "Sources reconciled (best-effort)" + +# --- 5) Resume and force reconciles ------------------------------------------ +info "Resuming Kustomization and HelmRelease" +flux resume kustomization $KS_NAME -n $KS_NS; or true +flux resume helmrelease $HR_NAME -n $HR_NS; or true + +info "Reconciling Kustomization with source (no waits)" +flux reconcile kustomization $KS_NAME -n $KS_NS --with-source --timeout=10m; or true + +info "FORCING HelmRelease reconcile (no waits)" +flux reconcile helmrelease $HR_NAME -n $HR_NS --timeout=10m --force; or true + +# --- 6) Optional restore of original specs ----------------------------------- +if test "$RESTORE" = "yes" + info "Restoring original specs and reconciling" + if test -s $KS_BAK + kubectl apply -f $KS_BAK; and flux reconcile kustomization $KS_NAME -n $KS_NS --timeout=5m; or true + end + if test -s $HR_BAK + kubectl apply -f $HR_BAK; and flux reconcile helmrelease $HR_NAME -n $HR_NS --timeout=5m; or true + end + ok "Restored and reconciled" +else + warn "Not restoring original specs (you ran without --restore). Gates remain off until you change them back." +end + +# --- 7) Show status ----------------------------------------------------------- +info "Current filtered status:" +flux get kustomizations | grep -E "(NAME|^$KS_NAME\s)"; or true +flux get helmreleases -A | grep -E "(NAMESPACE|^$HR_NS\s+$HR_NAME\s)"; or true diff --git a/scripts/nuke.fish b/scripts/nuke.fish new file mode 100755 index 0000000..8cd1cea --- /dev/null +++ b/scripts/nuke.fish @@ -0,0 +1,92 @@ +#!/usr/bin/env fish +# Hard reset / "hammer" for a single Kustomization + HelmRelease pair. +# Default target is Vault (KS: flux-system/vault, HR: vault/vault). + +set -l KS_NS flux-system +set -l KS_NAME vault + +set -l HR_ns vault +set -l HR_name vault + +# Timeouts +set -l TIMEOUT "10m" +set -l SRC_TIMEOUT "5m" + +function step + echo (set_color --bold cyan)"$argv"(set_color normal) +end +function ok + echo (set_color --bold green)"✔ $argv"(set_color normal) +end +function warn + echo (set_color --bold yellow)"! $argv"(set_color normal) +end +function err + echo (set_color --bold red)"✗ $argv"(set_color normal) +end + +# Preflight +type -q flux; or begin err "flux CLI not found in PATH"; exit 1; end +type -q kubectl; or begin err "kubectl not found in PATH"; exit 1; end + +step "Flux: "(flux --version | string trim) +step "Kube context: "(kubectl config current-context) + +# 1) Suspend KS and HR (best effort) +step "Suspending Kustomization $KS_NS/$KS_NAME and HelmRelease $HR_ns/$HR_name" +flux suspend kustomization $KS_NAME -n $KS_NS >/dev/null; or warn "KS already suspended?" +flux suspend helmrelease $HR_name -n $HR_ns >/dev/null; or warn "HR already suspended?" +ok "Suspended" + +# 2) Ensure latest sources are fetched +step "Reconciling GitRepository flux-system and HelmRepository hashicorp" +flux reconcile source git flux-system -n flux-system --timeout=$SRC_TIMEOUT >/dev/null; or err "GitRepository reconcile failed" +flux reconcile source helm hashicorp -n flux-system --timeout=$SRC_TIMEOUT >/dev/null; or warn "HelmRepository reconcile failed (continuing)" +ok "Sources refreshed" + +# 3) Resume KS and push desired state (don’t block here) +step "Resuming Kustomization $KS_NS/$KS_NAME and reconciling (don’t wait)" +flux resume kustomization $KS_NAME -n $KS_NS >/dev/null; or err "Failed to resume KS" +# With Flux v2.6.x there is no --wait; use kubectl wait later. +flux reconcile kustomization $KS_NAME -n $KS_NS --with-source --timeout=$TIMEOUT >/dev/null; or warn "KS reconcile returned non-zero (continuing)" +# Give controller a moment to create/refresh the HelmRelease CR +sleep 3 + +# 4) Patch HelmRelease to BYPASS readiness waiting (critical for Vault) +# This prevents helm-controller from rolling back while Vault is sealed. +step "Patching HelmRelease $HR_ns/$HR_name to disable readiness waiting and extend timeouts" +# Wait until the HR object exists (up to ~2 minutes) +for i in (seq 1 60) + kubectl -n $HR_ns get helmrelease $HR_name >/dev/null 2>&1; and break + sleep 2 +end +kubectl -n $HR_ns patch helmrelease $HR_name --type merge -p '{ + "spec": { + "install": { "disableWait": true, "timeout": "30m", "remediation": { "retries": 0 } }, + "upgrade": { "disableWait": true, "timeout": "30m", "remediation": { "retries": 0 } } + } +}' >/dev/null; or warn "Patch failed (does HR exist yet?)" + +# 5) Resume HR and reconcile, then WAIT with kubectl +step "Resuming HelmRelease $HR_ns/$HR_name and reconciling" +flux resume helmrelease $HR_name -n $HR_ns >/dev/null; or err "Failed to resume HR" +flux reconcile helmrelease $HR_name -n $HR_ns --with-source --timeout=$TIMEOUT >/dev/null; or warn "HR reconcile returned non-zero (continuing)" + +# Wait for HelmRelease Ready condition (works with CRDs; no flux --wait needed) +step "Waiting for HelmRelease Ready (timeout $TIMEOUT)" +kubectl -n $HR_ns wait helmrelease/$HR_name --for=condition=Ready --timeout=$TIMEOUT >/dev/null +if test $status -ne 0 + warn "HelmRelease did not become Ready within $TIMEOUT (showing status)" + flux get helmreleases -n $HR_ns $HR_name +else + ok "HelmRelease is Ready" +end + +# 6) Show final status +step "Final Flux status (filtered)" +flux get kustomizations -n $KS_NS $KS_NAME +flux get helmreleases -n $HR_ns $HR_name + +# Optional: uncomment to wait for Kustomization Ready too +# step "Waiting for Kustomization Ready (timeout $TIMEOUT)" +# kubectl -n $KS_NS wait kustomization/$KS_NAME --for=condition=Ready --timeout=$TIMEOUT >/dev/null; and ok "Kustomization Ready" diff --git a/services/vault/kustomization.yaml b/services/vault/kustomization.yaml index ae64c73..75cf507 100644 --- a/services/vault/kustomization.yaml +++ b/services/vault/kustomization.yaml @@ -4,3 +4,5 @@ namespace: vault resources: - namespace.yaml - helmrelease.yaml + - certificate.yaml + - ingressroutetcp.yaml