missing-repl-replacements

#!/usr/bin/env bash
#
# Usage:
#  missing-repl-replacements [AVOID-HOST]
#
# Parameters:
#  AVOID-HOST  if provided, avoid replicas on this host unless the only good
#              replica is on it
#
# Given a set of data objects that are missing their replicas, it looks for
# other data objects that have replicas with the same checksum. It reads the set
# of data objects from stdin, one per line. Each entry is expected to be an
# absolute, logical path. The candidate list is written to stdout, one candidate
# per line. A candidate is a tab-separated list having the following form.
#
# <data obj>	<bad resc>	<bad repl>	<candidate resc>	<candidate repl>
#
# Here <data obj> is the absolute, logical path to the data object read from
# stdin; <bad resc> is the host name of the resource server hosting the bad
# replica; <bad repl> is the absolute, physical path to the bad replica;
# <candidate resc> is the host name of the resource server hosting a candidate
# replica; and <candidate repl> is a replica that has the same checksum as the
# bad replica.
#
# © 2025 The Arizona Board of Regents on behalf of The University of Arizona.
# For license information, see https://cyverse.org/license.

set -o errexit -o nounset -o pipefail

EXEC_DIR="$(dirname "$(realpath --canonicalize-existing "$0")")"
readonly EXEC_DIR

readonly CHKSUM_CHUNK_LEN=500

declare -A HostOf ObjForChksum ReplOf
declare -a Candidates

main() {
	if (( $# > 0 )); then
		avoidHost="$1"
	fi

	printf 'Looking up checksums\n' >&2
	local obj
	while IFS= read -r obj; do
		map_checksum "$obj"
	done

	printf 'Finding copies\n' >&2
	local sameFiles
	sameFiles="$(find_copies)"

	printf 'Finding self candidates\n' >&2
	local selfCandidates
	selfCandidates="$(find_self_candidates)"

	printf 'Eliminating selves from candidates\n' >&2
	readarray -t Candidates <<< "$(comm -23 <(sort <<< "$sameFiles") <(sort <<< "$selfCandidates"))"

	printf 'Resolving replicas\n' >&2
	resolve_replicas "${avoidHost-}"
}

find_copies() {
	local chksums=("${!ObjForChksum[@]}")

	local chunkStart
	for chunkStart in $(seq 0 "$CHKSUM_CHUNK_LEN" ${#chksums[@]}); do
		local chksumList
		chksumList="$(mk_query_list "${chksums[@]:$chunkStart:$CHKSUM_CHUNK_LEN}")"

		local query
		printf -v query \
			'select DATA_CHECKSUM, RESC_LOC, DATA_PATH where DATA_CHECKSUM in (%s)\n' "$chksumList"

		"$EXEC_DIR"/quest $'%s\t%s\t%s' <<< "$query"
	done
}

mk_query_list() {
	local elmts=("$@")

	printf $'\'%s\'\n' "${elmts[@]}" | paste --serial --delimiter=,
}

find_self_candidates() {
	local chksum
	for chksum in "${!ObjForChksum[@]}"; do
		local obj="${ObjForChksum[$chksum]}"
		printf '%s\t%s\t%s\n' "$chksum" "${HostOf[$obj]}" "${ReplOf[$obj]}"
	done
}

map_checksum() {
	local objPath="$1"

	local coll objName
	coll="$(dirname "$objPath")"
	objName="$(basename "$objPath")"

	local chksum srv repl
	if IFS=$'\t' read -r chksum srv repl <<< "$(retrieve_chksum_and_loc "$coll" "$objName")"; then
		if [[ -n "$chksum" && "$chksum" != 0 && -n "$srv" && -n "$repl" ]]; then
			ObjForChksum[$chksum]="$objPath"
			HostOf[$objPath]="$srv"
			ReplOf[$objPath]="$repl"
		fi
	fi
}

retrieve_chksum_and_loc() {
	local collPath="$1"
	local dataName="$2"

	"$EXEC_DIR"/quest $'%s\t%s\t%s' <<EOF
select DATA_CHECKSUM, RESC_LOC, DATA_PATH where COLL_NAME = '$collPath' and DATA_NAME = '$dataName'
EOF
}

resolve_replicas() {
	local avoidHost="$1"

	if [[ -n "${Candidates[*]}" ]]; then
		local obj needyHost missingRepl
		local selectedHost=''
		local selectedRepl=''
		local lastObj=''

		local candidate
		for candidate in "${Candidates[@]}"; do
			local chksum srv repl
			IFS=$'\t' read -r chksum srv repl <<< "$candidate"

			obj="${ObjForChksum[$chksum]}"
			needyHost="${HostOf[$obj]}"

			if [[ "$obj" != "$lastObj" && -n "$lastObj" ]]; then
				show_selected_mapping \
					"$lastObj" \
					"${HostOf[$lastObj]}" \
					"${ReplOf[$lastObj]}" \
					"$selectedHost" \
					"$selectedRepl"

				selectedHost=''
				selectedRepl=''
			fi

			if select_copy "$needyHost" "$selectedHost" "$srv" "$avoidHost"; then
				selectedHost="$srv"
				selectedRepl="$repl"
			fi

			lastObj="$obj"
		done

		if [[ -n "$selectedHost" ]]; then
			show_selected_mapping \
				"$lastObj" "$needyHost" "${ReplOf[$obj]}" "$selectedHost" "$selectedRepl"
		fi
	fi
}

select_copy() {
	local needyHost="$1"
	local curCopyHost="$2"
	local candidateHost="$3"
	local avoidHost="$4"

	local avoidCandidate=false
	if [[ -n "$avoidHost" && "$candidateHost" == "$avoidHost" ]]; then
		avoidCandidate=true
	fi

	[[ "$candidateHost" == "$needyHost" ]] \
		|| [[ "$avoidCandidate" == false && "$curCopyHost" != "$needyHost" ]] \
		|| [[ "$avoidCandidate" == true && -z "$curCopyHost" ]]
}

show_selected_mapping() {
	local obj="$1"
	local needyHost="$2"
	local missingRepl="$3"
	local providingHost="$4"
	local selectedRepl="$5"

	printf '%s\t%s\t%s\t%s\t%s\n' \
		"$obj" "$needyHost" "$missingRepl" "$providingHost" "$selectedRepl"
}

main "$@"