atlas-iac/scripts/ops/postgres_application_backup.sh

#!/usr/bin/env bash
# Create a root-only logical recovery bundle on a control-plane host.
# Requires PostgreSQL client 16 and local root access to k3s kubectl.
set -euo pipefail
umask 077

[[ $(id -u) == 0 ]] || { echo 'Run as root.' >&2; exit 1; }
command -v pg_dump >/dev/null
exec 9>/run/atlas-application-postgres-backup.lock
flock -n 9 || exit 0

backup_root=/var/backups/atlas-postgres
install -d -m 700 "$backup_root"
available=$(df -B1 --output=avail "$backup_root" | tail -1)
(( available > 10737418240 )) || { echo 'Less than 10 GiB free.' >&2; exit 1; }
bundle=$(mktemp -d "$backup_root/run-$(date -u +%Y%m%dT%H%M%SZ)-XXXXXX")
# Database-tool errors can include object names. Keep them with the protected copy.
exec 2>"$bundle/errors.log"
passfile=$(mktemp /run/atlas-postgres-password.XXXXXX)
trap 'rm -f "$passfile"' EXIT

host=$(k3s kubectl -n postgres get pod postgres-0 -o jsonpath='{.status.podIP}')
[[ $host =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || exit 1
password=$(k3s kubectl -n postgres exec postgres-0 -c postgres -- cat /mnt/vault/postgres_password)
password=${password//\\/\\\\}
password=${password//:/\\:}
printf '%s:5432:*:postgres:%s\n' "$host" "$password" > "$passfile"
unset password
export PGPASSFILE="$passfile" PGHOST="$host" PGUSER=postgres PGCONNECT_TIMEOUT=10

# Native PostgreSQL traffic avoids sending entire dumps through the API server.
psql -d postgres -Atc 'SELECT datname FROM pg_database WHERE NOT datistemplate ORDER BY datname' > "$bundle/databases.txt"
pg_dumpall --globals-only > "$bundle/globals.sql"
index=0
while IFS= read -r database; do
  [[ $database =~ ^[a-zA-Z0-9_]+$ ]] || { echo 'Unsupported database name.' >&2; exit 1; }
  index=$((index + 1))
  timeout --signal=TERM --kill-after=30s 3600 pg_dump -Fc -Z 1 -d "$database" > "$bundle/db-$index.dump.pending"
  pg_restore --list "$bundle/db-$index.dump.pending" >/dev/null
  mv "$bundle/db-$index.dump.pending" "$bundle/db-$index.dump"
  printf 'Completed database %d.\n' "$index"
done < "$bundle/databases.txt"

(
  cd "$bundle"
  sha256sum globals.sql databases.txt ./*.dump > SHA256SUMS
  sha256sum --check SHA256SUMS >/dev/null
  date -u +%FT%TZ > COMPLETE
)
ln -sfn "$(basename "$bundle")" "$backup_root/latest.new"
mv -Tf "$backup_root/latest.new" "$backup_root/latest"
metric_dir=/var/lib/node_exporter/textfile_collector
install -d -m 755 "$metric_dir"
metric_tmp=$(mktemp "$metric_dir/.application-backup.XXXXXX")
printf 'atlas_application_postgres_backup_last_success_timestamp_seconds %s\n' "$(date +%s)" > "$metric_tmp"
chmod 644 "$metric_tmp"
mv -f "$metric_tmp" "$metric_dir/atlas_application_postgres_backup.prom"
# A failed run never deletes the last good recovery set.
find "$backup_root" -mindepth 1 -maxdepth 1 -type d -name 'run-*' -mtime +7 -exec rm -rf -- {} +
printf 'Verified recovery bundle: %s (%d databases).\n' "$bundle" "$index"