61 lines
2.8 KiB
Bash
Executable File
61 lines
2.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Create a root-only logical recovery bundle on a control-plane host.
|
|
# Requires PostgreSQL client 16 and local root access to k3s kubectl.
|
|
set -euo pipefail
|
|
umask 077
|
|
|
|
[[ $(id -u) == 0 ]] || { echo 'Run as root.' >&2; exit 1; }
|
|
command -v pg_dump >/dev/null
|
|
exec 9>/run/atlas-application-postgres-backup.lock
|
|
flock -n 9 || exit 0
|
|
|
|
backup_root=/var/backups/atlas-postgres
|
|
install -d -m 700 "$backup_root"
|
|
available=$(df -B1 --output=avail "$backup_root" | tail -1)
|
|
(( available > 10737418240 )) || { echo 'Less than 10 GiB free.' >&2; exit 1; }
|
|
bundle=$(mktemp -d "$backup_root/run-$(date -u +%Y%m%dT%H%M%SZ)-XXXXXX")
|
|
# Database-tool errors can include object names. Keep them with the protected copy.
|
|
exec 2>"$bundle/errors.log"
|
|
passfile=$(mktemp /run/atlas-postgres-password.XXXXXX)
|
|
trap 'rm -f "$passfile"' EXIT
|
|
|
|
host=$(k3s kubectl -n postgres get pod postgres-0 -o jsonpath='{.status.podIP}')
|
|
[[ $host =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]] || exit 1
|
|
password=$(k3s kubectl -n postgres exec postgres-0 -c postgres -- cat /mnt/vault/postgres_password)
|
|
password=${password//\\/\\\\}
|
|
password=${password//:/\\:}
|
|
printf '%s:5432:*:postgres:%s\n' "$host" "$password" > "$passfile"
|
|
unset password
|
|
export PGPASSFILE="$passfile" PGHOST="$host" PGUSER=postgres PGCONNECT_TIMEOUT=10
|
|
|
|
# Native PostgreSQL traffic avoids sending entire dumps through the API server.
|
|
psql -d postgres -Atc 'SELECT datname FROM pg_database WHERE NOT datistemplate ORDER BY datname' > "$bundle/databases.txt"
|
|
pg_dumpall --globals-only > "$bundle/globals.sql"
|
|
index=0
|
|
while IFS= read -r database; do
|
|
[[ $database =~ ^[a-zA-Z0-9_]+$ ]] || { echo 'Unsupported database name.' >&2; exit 1; }
|
|
index=$((index + 1))
|
|
timeout --signal=TERM --kill-after=30s 3600 pg_dump -Fc -Z 1 -d "$database" > "$bundle/db-$index.dump.pending"
|
|
pg_restore --list "$bundle/db-$index.dump.pending" >/dev/null
|
|
mv "$bundle/db-$index.dump.pending" "$bundle/db-$index.dump"
|
|
printf 'Completed database %d.\n' "$index"
|
|
done < "$bundle/databases.txt"
|
|
|
|
(
|
|
cd "$bundle"
|
|
sha256sum globals.sql databases.txt ./*.dump > SHA256SUMS
|
|
sha256sum --check SHA256SUMS >/dev/null
|
|
date -u +%FT%TZ > COMPLETE
|
|
)
|
|
ln -sfn "$(basename "$bundle")" "$backup_root/latest.new"
|
|
mv -Tf "$backup_root/latest.new" "$backup_root/latest"
|
|
metric_dir=/var/lib/node_exporter/textfile_collector
|
|
install -d -m 755 "$metric_dir"
|
|
metric_tmp=$(mktemp "$metric_dir/.application-backup.XXXXXX")
|
|
printf 'atlas_application_postgres_backup_last_success_timestamp_seconds %s\n' "$(date +%s)" > "$metric_tmp"
|
|
chmod 644 "$metric_tmp"
|
|
mv -f "$metric_tmp" "$metric_dir/atlas_application_postgres_backup.prom"
|
|
# A failed run never deletes the last good recovery set.
|
|
find "$backup_root" -mindepth 1 -maxdepth 1 -type d -name 'run-*' -mtime +7 -exec rm -rf -- {} +
|
|
printf 'Verified recovery bundle: %s (%d databases).\n' "$bundle" "$index"
|