68 lines
3.4 KiB
Bash
Executable File
68 lines
3.4 KiB
Bash
Executable File
#!/bin/sh
|
|
set -eu
|
|
|
|
destination="${BACKUP_DESTINATION:-/backups}"
|
|
verify_dir="${BACKUP_OFFSITE_VERIFY_DIR:-/offsite-verify}"
|
|
remote="${RCLONE_ONEDRIVE_REMOTE:-onedrive}"
|
|
remote_path="${RCLONE_ONEDRIVE_PATH:-FleetOps/backups}"
|
|
interval="${BACKUP_OFFSITE_INTERVAL_SECONDS:-900}"
|
|
restore_drill_interval="${BACKUP_RESTORE_DRILL_INTERVAL_SECONDS:-604800}"
|
|
retention_days="${BACKUP_RETENTION_DAYS:-30}"
|
|
|
|
case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
|
|
case "$verify_dir" in ""|"/"|".") echo "Unsafe off-site verify directory: $verify_dir" >&2; exit 1;; esac
|
|
case "$remote" in ""|*:*|*/*) echo "Rclone remote must be a configured remote name" >&2; exit 1;; esac
|
|
case "$remote_path" in ""|"/"|/*|*".."*) echo "Unsafe OneDrive backup path" >&2; exit 1;; esac
|
|
case "$interval:$restore_drill_interval:$retention_days" in *[!0-9:]*|:*|*:) echo "Off-site intervals and retention must be integers" >&2; exit 1;; esac
|
|
[ "$retention_days" -ge 1 ] || { echo "Off-site retention must be at least one day" >&2; exit 1; }
|
|
[ -f "${RCLONE_CONFIG:-/config/rclone/rclone.conf}" ] || {
|
|
echo "Rclone configuration is missing" >&2
|
|
exit 1
|
|
}
|
|
|
|
mkdir -p "$verify_dir"
|
|
|
|
while true; do
|
|
latest="$(find "$destination" -maxdepth 1 -type f -name 'mobilityops-*.dump' | sort -r | head -n 1)"
|
|
if [ -n "$latest" ] && [ -f "$latest.sha256" ]; then
|
|
name="$(basename "$latest")"
|
|
checksum_name="$name.sha256"
|
|
remote_root="$remote:$remote_path"
|
|
temporary_dump="$verify_dir/$name.partial"
|
|
temporary_checksum="$verify_dir/$checksum_name.partial"
|
|
verified_dump="$verify_dir/$name"
|
|
|
|
rm -f "$temporary_dump" "$temporary_checksum"
|
|
if rclone copyto "$latest" "$remote_root/$name" --checksum --retries 3 --low-level-retries 5 \
|
|
&& rclone copyto "$latest.sha256" "$remote_root/$checksum_name" --checksum --retries 3 --low-level-retries 5 \
|
|
&& rclone copyto "$remote_root/$name" "$temporary_dump" --retries 3 --low-level-retries 5 \
|
|
&& rclone copyto "$remote_root/$checksum_name" "$temporary_checksum" --retries 3 --low-level-retries 5 \
|
|
&& mv "$temporary_dump" "$verified_dump" \
|
|
&& mv "$temporary_checksum" "$verify_dir/$checksum_name" \
|
|
&& (cd "$verify_dir" && sha256sum -c "$checksum_name") \
|
|
&& pg_restore --list "$verified_dump" > /dev/null; then
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-offsite-success"
|
|
drill_minutes=$((restore_drill_interval / 60))
|
|
if [ ! -f "$destination/latest-offsite-restore-drill" ] \
|
|
|| ! find "$destination/latest-offsite-restore-drill" -mmin "-$drill_minutes" -print -quit | grep -q .; then
|
|
if /opt/mobilityops/restore-drill-postgres.sh "$verified_dump"; then
|
|
date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-offsite-restore-drill"
|
|
else
|
|
echo "Off-site restore drill failed for $name" >&2
|
|
fi
|
|
fi
|
|
rclone delete "$remote_root" --min-age "${retention_days}d" \
|
|
--include 'mobilityops-*.dump' --include 'mobilityops-*.dump.sha256' \
|
|
--retries 3 --low-level-retries 5
|
|
find "$verify_dir" -type f -mtime "+$retention_days" -delete
|
|
echo "Verified OneDrive round trip: $remote_root/$name"
|
|
else
|
|
rm -f "$temporary_dump" "$temporary_checksum"
|
|
echo "OneDrive backup synchronization failed for $name" >&2
|
|
fi
|
|
else
|
|
echo "No verified local backup is available for OneDrive synchronization" >&2
|
|
fi
|
|
sleep "$interval"
|
|
done
|