M23: automate verified database backups

deploy/unraid/README.md

@@ -25,7 +25,7 @@ cd /mnt/user/appdata/mobilityops
 ./deploy/unraid/configure-env.sh \
   http://192.168.10.150:1236 \
   http://192.168.10.150:5678/webhook/mobilityops-return
-docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml up --build -d db api web
+docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml up --build -d db api web backup
 docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml exec api \
   python -m app.cli seed --reset
 ```
@@ -55,14 +55,22 @@ docker logs --tail=200 n8n
 
 ## Backup and restore
 
-Create and structurally verify a timestamped PostgreSQL custom-format backup:
+The `backup` service creates a backup immediately and then every 24 hours. Every dump is
+validated with `pg_restore --list`, receives a SHA-256 sidecar and is retained for 30 days
+with at least seven copies protected from pruning. Its healthcheck becomes unhealthy when
+no successful backup has been recorded for 26 hours. Configure
+`BACKUP_SECONDARY_DESTINATION=/offsite` plus an independently mounted
+`MOBILITYOPS_BACKUP_SECONDARY_DIR` for a second copy.
+
+Create an additional on-demand backup or verify the newest scheduled backup:
 
 ```bash
 ./deploy/unraid/backup-postgres.sh
+./deploy/unraid/verify-postgres-backups.sh
 ```
 
-Copy backups off the server according to the host backup policy. A restore is deliberately
-guarded and creates an additional safety backup before replacing the database:
+A restore is deliberately guarded and creates an additional safety backup before
+replacing the database:
 
 ```bash
 ./deploy/unraid/restore-postgres.sh \

deploy/unraid/backup-postgres.sh

@@ -4,12 +4,15 @@ set -eu
 cd "$(dirname "$0")/../.."
 project="${COMPOSE_PROJECT_NAME:-mobilityops}"
 destination="${1:-backups/postgres}"
+retention_days="${BACKUP_RETENTION_DAYS:-30}"
+minimum_copies="${BACKUP_MINIMUM_COPIES:-7}"
 timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
 backup_file="${destination}/mobilityops-${timestamp}.dump"
 temporary_file="${backup_file}.partial"
 compose_files="-f compose.yaml -f compose.unraid.yaml"
 
 mkdir -p "$destination"
+case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
 trap 'rm -f "$temporary_file"' EXIT INT TERM
 
 docker compose -p "$project" $compose_files exec -T db sh -eu -c \
@@ -19,5 +22,8 @@ docker compose -p "$project" $compose_files exec -T db sh -eu -c \
 docker compose -p "$project" $compose_files exec -T db pg_restore --list \
   < "$temporary_file" > /dev/null
 mv "$temporary_file" "$backup_file"
+sha256sum "$backup_file" > "${backup_file}.sha256"
+BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \
+  ./deploy/unraid/prune-postgres-backups.sh "$destination"
 trap - EXIT INT TERM
 printf '%s\n' "$backup_file"

deploy/unraid/prune-postgres-backups.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+
+destination="${1:-backups/postgres}"
+retention_days="${BACKUP_RETENTION_DAYS:-30}"
+minimum_copies="${BACKUP_MINIMUM_COPIES:-7}"
+
+case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
+case "$retention_days:$minimum_copies" in *[!0-9:]*|:*|*:) echo "Retention values must be integers" >&2; exit 1;; esac
+[ -d "$destination" ] || exit 0
+
+count=0
+find "$destination" -maxdepth 1 -type f -name 'mobilityops-*.dump' | sort -r | while IFS= read -r file; do
+  count=$((count + 1))
+  if [ "$count" -gt "$minimum_copies" ] && [ -n "$(find "$file" -mtime "+$retention_days" -print)" ]; then
+    rm -f -- "$file" "${file}.sha256"
+  fi
+done

deploy/unraid/scheduled-backup.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+set -eu
+
+destination="${BACKUP_DESTINATION:-/backups}"
+secondary="${BACKUP_SECONDARY_DESTINATION:-}"
+interval="${BACKUP_INTERVAL_SECONDS:-86400}"
+retention_days="${BACKUP_RETENTION_DAYS:-30}"
+minimum_copies="${BACKUP_MINIMUM_COPIES:-7}"
+
+case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
+case "$interval:$retention_days:$minimum_copies" in *[!0-9:]*|:*|*:) echo "Backup settings must be integers" >&2; exit 1;; esac
+mkdir -p "$destination"
+[ -z "$secondary" ] || mkdir -p "$secondary"
+
+while true; do
+  timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+  target="$destination/mobilityops-$timestamp.dump"
+  temporary="$target.partial"
+  rm -f "$temporary"
+  if pg_dump --format=custom --no-owner --no-acl \
+      --host=db --username="$POSTGRES_USER" "$POSTGRES_DB" > "$temporary" \
+      && pg_restore --list "$temporary" > /dev/null; then
+    mv "$temporary" "$target"
+    sha256sum "$target" > "$target.sha256"
+    if [ -n "$secondary" ]; then
+      cp "$target" "$target.sha256" "$secondary/"
+    fi
+    date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-success"
+    BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \
+      /opt/mobilityops/prune-postgres-backups.sh "$destination"
+    if [ -n "$secondary" ]; then
+      BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \
+        /opt/mobilityops/prune-postgres-backups.sh "$secondary"
+    fi
+    echo "Verified database backup: $target"
+  else
+    rm -f "$temporary"
+    echo "Database backup failed at $timestamp" >&2
+  fi
+  sleep "$interval"
+done

deploy/unraid/verify-postgres-backups.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+
+destination="${1:-backups/postgres}"
+case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
+latest="$(find "$destination" -maxdepth 1 -type f -name 'mobilityops-*.dump' | sort -r | head -n 1)"
+[ -n "$latest" ] || { echo "No MobilityOps backup found" >&2; exit 1; }
+[ -f "$latest.sha256" ] || { echo "Checksum missing for $latest" >&2; exit 1; }
+(cd "$destination" && sha256sum -c "$(basename "$latest.sha256")")
+docker compose -p "${COMPOSE_PROJECT_NAME:-mobilityops}" \
+  -f compose.yaml -f compose.unraid.yaml exec -T db pg_restore --list < "$latest" > /dev/null
+printf 'Verified: %s\n' "$latest"