backup: add native application database recovery bundles

scripts/ops/postgres_application_backup.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Create a root-only logical recovery bundle on a control-plane host.
+# Requires PostgreSQL client 16 and local root access to k3s kubectl.
+set -euo pipefail
+umask 077
+
+[[ $(id -u) == 0 ]] || { echo 'Run as root.' >&2; exit 1; }
+command -v pg_dump >/dev/null
+exec 9>/run/atlas-application-postgres-backup.lock
+flock -n 9 || exit 0
+
+backup_root=/var/backups/atlas-postgres
+install -d -m 700 "$backup_root"
+available=$(df -B1 --output=avail "$backup_root" | tail -1)
+(( available > 10737418240 )) || { echo 'Less than 10 GiB free.' >&2; exit 1; }
+bundle=$(mktemp -d "$backup_root/run-$(date -u +%Y%m%dT%H%M%SZ)-XXXXXX")
+passfile=$(mktemp /run/atlas-postgres-password.XXXXXX)
+trap 'rm -f "$passfile"' EXIT
+
+host=$(k3s kubectl -n postgres get pod postgres-0 -o jsonpath='{.status.podIP}')
+[[ $host =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || exit 1
+password=$(k3s kubectl -n postgres exec postgres-0 -c postgres -- cat /mnt/vault/postgres_password)
+password=${password//\\/\\\\}
+password=${password//:/\\:}
+printf '%s:5432:*:postgres:%s\n' "$host" "$password" > "$passfile"
+unset password
+export PGPASSFILE="$passfile" PGHOST="$host" PGUSER=postgres PGCONNECT_TIMEOUT=10
+
+# Native PostgreSQL traffic avoids sending entire dumps through the API server.
+psql -d postgres -Atc 'SELECT datname FROM pg_database WHERE NOT datistemplate ORDER BY datname' > "$bundle/databases.txt"
+pg_dumpall --globals-only > "$bundle/globals.sql"
+index=0
+while IFS= read -r database; do
+  [[ $database =~ ^[a-zA-Z0-9_]+$ ]] || { echo 'Unsupported database name.' >&2; exit 1; }
+  index=$((index + 1))
+  timeout --signal=TERM --kill-after=30s 3600 pg_dump -Fc -Z 1 -d "$database" > "$bundle/db-$index.dump.pending"
+  pg_restore --list "$bundle/db-$index.dump.pending" >/dev/null
+  mv "$bundle/db-$index.dump.pending" "$bundle/db-$index.dump"
+  printf 'Completed database %d.\n' "$index"
+done < "$bundle/databases.txt"
+
+(
+  cd "$bundle"
+  sha256sum globals.sql databases.txt ./*.dump > SHA256SUMS
+  sha256sum --check SHA256SUMS >/dev/null
+  date -u +%FT%TZ > COMPLETE
+)
+ln -sfn "$(basename "$bundle")" "$backup_root/latest.new"
+mv -Tf "$backup_root/latest.new" "$backup_root/latest"
+printf 'Verified recovery bundle: %s (%d databases).\n' "$bundle" "$index"