M44: harden release integrity and assurance

deploy/unraid/scheduled-backup.sh

@@ -6,9 +6,10 @@ secondary="${BACKUP_SECONDARY_DESTINATION:-}"
 interval="${BACKUP_INTERVAL_SECONDS:-86400}"
 retention_days="${BACKUP_RETENTION_DAYS:-30}"
 minimum_copies="${BACKUP_MINIMUM_COPIES:-7}"
+restore_drill_interval="${BACKUP_RESTORE_DRILL_INTERVAL_SECONDS:-604800}"
 
 case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
-case "$interval:$retention_days:$minimum_copies" in *[!0-9:]*|:*|*:) echo "Backup settings must be integers" >&2; exit 1;; esac
+case "$interval:$retention_days:$minimum_copies:$restore_drill_interval" in *[!0-9:]*|:*|*:) echo "Backup settings must be integers" >&2; exit 1;; esac
 mkdir -p "$destination"
 [ -z "$secondary" ] || mkdir -p "$secondary"
 
@@ -24,6 +25,7 @@ while true; do
     (cd "$destination" && sha256sum "$(basename "$target")" > "$(basename "$target").sha256")
     if [ -n "$secondary" ]; then
       cp "$target" "$target.sha256" "$secondary/"
+      (cd "$secondary" && sha256sum -c "$(basename "$target.sha256")")
     fi
     date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-success"
     BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \
@@ -32,6 +34,15 @@ while true; do
       BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \
         /opt/mobilityops/prune-postgres-backups.sh "$secondary"
     fi
+    drill_minutes=$((restore_drill_interval / 60))
+    if [ ! -f "$destination/latest-restore-drill" ] \
+        || ! find "$destination/latest-restore-drill" -mmin "-$drill_minutes" -print -quit | grep -q .; then
+      if /opt/mobilityops/restore-drill-postgres.sh "$target"; then
+        date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-restore-drill"
+      else
+        echo "Restore drill failed for $target" >&2
+      fi
+    fi
     echo "Verified database backup: $target"
   else
     rm -f "$temporary"