sync-icat-backups

#!/usr/bin/env bash
#
# Usage:
#  sync-icat-backups
#
# This program is responsible for maintaining copies of the ICAT DB backups.
# on the execution host and in the CyVerse Data Store. It makes the following
# assumptions.
#
# * The backups are generated by pg_dump.
# * The primary copies of the ICAT DB backups are stored in the folder
#   /home/db-backups/ on the primary node of the DBMS.
# * Each day's backup is stored in a folder named icat<YYYY><MM><DD>/.
# * The DBMS is accessible through the FQDN icat.cyverse.org.
# * It can connect as root to the DBMS host without having to provide a
#   password.
# * It is able to maintain copies of the backups locally in the folder
#   /nfs_exports/db_backups/ds-db3/.
# * GoCommands are installed in the folder /root/bin and are accessible by the
#   script.
# * It can connect to the CyVerse Data Store as ipc_admin without having to
#   provide a password.
# * It is able to maintain copies of the backups in the Data Store in the
#   collection /iplant/home/ipc_admin/ICAT-backups.

set -o errexit -o nounset -o pipefail

readonly DB_BASENAME=icat

readonly DBMS_DIR=/home/db-backups
readonly DBMS_HOST=icat.cyverse.org
readonly DBMS_SSH_PORT=1657
readonly DBMS_USER=root

readonly IRODS_COLL=/iplant/home/ipc_admin/ICAT-backups

readonly NFS_DIR=/nfs_exports/db_backups/ds-db3

readonly MAX_BACKUP_AGE=10

# NOTE: The DBMS primary can failover to the replica, making icat.cyverse.org
# point to the replica. The replica may not have 10 days worth backups. To
# prevent backups from being deleted early in this case, old backups are deleted
# by this script instead of relying on rsync to delete backups.

main() {
	local backupCutOffDate
	backupCutOffDate="$(/usr/bin/date --date="$MAX_BACKUP_AGE days ago" +%Y%m%d)"

	/usr/bin/printf 'Syncing backups from DBMS to NFS\n'
	sync_from_dbms || true
	delete_old_backups "$backupCutOffDate" || true
	/usr/bin/printf '\n'
	/usr/bin/printf 'Syncing backups from NFS to iRODS\n'
	sync_to_irods || true
}

delete_old_backups() {
	local cutOffDate="$1"

	local backups
	readarray -t backups < <(find "$NFS_DIR" -mindepth 1 -type d)

	local backup
	for backup in "${backups[@]}"; do
		local coll="${backup##*/}"

		if (( ${coll#"$DB_BASENAME"} <= cutOffDate )); then
			rm --recursive "$backup" || true
			printf '• Deleting old backup %s\n' "$coll"
		fi
	done
}

sync_from_dbms() {
	local rshCmd="/usr/bin/ssh -q -p $DBMS_SSH_PORT"
	local src="$DBMS_USER"@"$DBMS_HOST":"$DBMS_DIR"/

	/usr/bin/rsync --archive --verbose --rsh="$rshCmd" "$src" "$NFS_DIR" | filter_rsync_msgs
}

filter_rsync_msgs() {
	/usr/bin/awk --assign=DB_BASENAME="$DB_BASENAME" --file - <(/usr/bin/cat) <<'EOF'
$0 ~ DB_BASENAME "[0-9]+/$" {
	coll = gensub(".*(" DB_BASENAME "[0-9]+)/", "\\1", 1, $0)
	printf "• Downloading new backup %s\n", coll
}
EOF
}

sync_to_irods() {
	/root/bin/gocmd --yes \
			sync --debug --delete --no_root --verify_checksum "$NFS_DIR" i:"$IRODS_COLL" \
		| filter_gocmd_msgs
}

filter_gocmd_msgs() {
	/usr/bin/awk --assign=DB_BASENAME="$DB_BASENAME" --file - <(/usr/bin/cat) <<'EOF'
/msg="uploaded a file" .*\/toc.dat / {
	coll = gensub(".*/(" DB_BASENAME "[0-9]+)/.*", "\\1", 1, $0)
	printf "• Uploading new backup %s\n", coll
}

/msg="deleted an extra collection"/ {
	coll = gensub(".*/(" DB_BASENAME "[0-9]+).*", "\\1", 1, $0)
	printf "• Deleting old backup %s\n", coll
}
EOF
}

main "$@"