diff --git a/.env.example b/.env.example index 521d364..0976e14 100644 --- a/.env.example +++ b/.env.example @@ -32,6 +32,14 @@ METRICS_BEARER_TOKEN= GRAFANA_ADMIN_USER=admin GRAFANA_ADMIN_PASSWORD=change-me-before-start +# Verified scheduled PostgreSQL backups (Unraid override). +BACKUP_INTERVAL_SECONDS=86400 +BACKUP_RETENTION_DAYS=30 +BACKUP_MINIMUM_COPIES=7 +# Set both values to copy every verified backup to an independently mounted path. +BACKUP_SECONDARY_DESTINATION= +MOBILITYOPS_BACKUP_SECONDARY_DIR=./backups/offsite + # Demo presentation (fictional org identity, badge/manifest, reset safety valve). # DEMO_ALLOW_RESET=false permanently disables POST /api/v1/demo/reset (403), independent # of role -- a safety valve for any environment where the dataset must not be rebuildable. diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 64ccda9..def749e 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -2537,3 +2537,20 @@ evidence yet." Prometheus `promtool` accepted the scrape config and all six rules; merged Compose and Grafana dashboard JSON validate. Exact next action: automate verified backups, retention and restore-readiness checks. + +## M23 — scheduled and retained recovery points (2026-08-10) + +- Added a continuously running Unraid backup service that waits for healthy PostgreSQL, + creates an immediate custom-format dump and repeats at a configurable interval without + access to the Docker socket. The existing on-demand path remains supported. +- Every backup must pass `pg_restore --list` before publication and receives a SHA-256 + sidecar. Retention defaults to 30 days while always preserving at least seven newest + restore points. An optional independently mounted secondary destination receives the + same verified artifacts. +- Added a latest-success marker and container healthcheck that detects a missed 26-hour + recovery point, a standalone verification command, guarded destinations and bounded + backup/log storage. +- Evidence: all shell scripts pass Alpine `sh -n`; merged Unraid Compose validates; a + disposable PostgreSQL instance produced a real dump, checksum verification passed and + `pg_restore --list` accepted the artifact. Exact next action: implement privacy export, + anonymisation safeguards, retention reporting and governance documentation. diff --git a/compose.unraid.yaml b/compose.unraid.yaml index a156aa5..5a19f61 100644 --- a/compose.unraid.yaml +++ b/compose.unraid.yaml @@ -34,3 +34,31 @@ services: interval: 10s timeout: 5s retries: 20 + + backup: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB:-mobilityops} + POSTGRES_USER: ${POSTGRES_USER:-mobilityops} + PGPASSWORD: ${POSTGRES_PASSWORD:-mobilityops} + BACKUP_DESTINATION: /backups + BACKUP_SECONDARY_DESTINATION: ${BACKUP_SECONDARY_DESTINATION:-} + BACKUP_INTERVAL_SECONDS: ${BACKUP_INTERVAL_SECONDS:-86400} + BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-30} + BACKUP_MINIMUM_COPIES: ${BACKUP_MINIMUM_COPIES:-7} + command: ["/opt/mobilityops/scheduled-backup.sh"] + volumes: + - ./backups/postgres:/backups + - ${MOBILITYOPS_BACKUP_SECONDARY_DIR:-./backups/offsite}:/offsite + - ./deploy/unraid:/opt/mobilityops:ro + depends_on: + db: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "find /backups/latest-success -mmin -1560 -print -quit | grep -q ."] + interval: 30m + timeout: 5s + retries: 3 + start_period: 5m + networks: [mobilityops] diff --git a/deploy/unraid/README.md b/deploy/unraid/README.md index cddf81b..195e90b 100644 --- a/deploy/unraid/README.md +++ b/deploy/unraid/README.md @@ -25,7 +25,7 @@ cd /mnt/user/appdata/mobilityops ./deploy/unraid/configure-env.sh \ http://192.168.10.150:1236 \ http://192.168.10.150:5678/webhook/mobilityops-return -docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml up --build -d db api web +docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml up --build -d db api web backup docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml exec api \ python -m app.cli seed --reset ``` @@ -55,14 +55,22 @@ docker logs --tail=200 n8n ## Backup and restore -Create and structurally verify a timestamped PostgreSQL custom-format backup: +The `backup` service creates a backup immediately and then every 24 hours. Every dump is +validated with `pg_restore --list`, receives a SHA-256 sidecar and is retained for 30 days +with at least seven copies protected from pruning. Its healthcheck becomes unhealthy when +no successful backup has been recorded for 26 hours. Configure +`BACKUP_SECONDARY_DESTINATION=/offsite` plus an independently mounted +`MOBILITYOPS_BACKUP_SECONDARY_DIR` for a second copy. + +Create an additional on-demand backup or verify the newest scheduled backup: ```bash ./deploy/unraid/backup-postgres.sh +./deploy/unraid/verify-postgres-backups.sh ``` -Copy backups off the server according to the host backup policy. A restore is deliberately -guarded and creates an additional safety backup before replacing the database: +A restore is deliberately guarded and creates an additional safety backup before +replacing the database: ```bash ./deploy/unraid/restore-postgres.sh \ diff --git a/deploy/unraid/backup-postgres.sh b/deploy/unraid/backup-postgres.sh index 32f51a3..cc3f5ca 100755 --- a/deploy/unraid/backup-postgres.sh +++ b/deploy/unraid/backup-postgres.sh @@ -4,12 +4,15 @@ set -eu cd "$(dirname "$0")/../.." project="${COMPOSE_PROJECT_NAME:-mobilityops}" destination="${1:-backups/postgres}" +retention_days="${BACKUP_RETENTION_DAYS:-30}" +minimum_copies="${BACKUP_MINIMUM_COPIES:-7}" timestamp="$(date -u +%Y%m%dT%H%M%SZ)" backup_file="${destination}/mobilityops-${timestamp}.dump" temporary_file="${backup_file}.partial" compose_files="-f compose.yaml -f compose.unraid.yaml" mkdir -p "$destination" +case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac trap 'rm -f "$temporary_file"' EXIT INT TERM docker compose -p "$project" $compose_files exec -T db sh -eu -c \ @@ -19,5 +22,8 @@ docker compose -p "$project" $compose_files exec -T db sh -eu -c \ docker compose -p "$project" $compose_files exec -T db pg_restore --list \ < "$temporary_file" > /dev/null mv "$temporary_file" "$backup_file" +sha256sum "$backup_file" > "${backup_file}.sha256" +BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \ + ./deploy/unraid/prune-postgres-backups.sh "$destination" trap - EXIT INT TERM printf '%s\n' "$backup_file" diff --git a/deploy/unraid/prune-postgres-backups.sh b/deploy/unraid/prune-postgres-backups.sh new file mode 100755 index 0000000..020adda --- /dev/null +++ b/deploy/unraid/prune-postgres-backups.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu + +destination="${1:-backups/postgres}" +retention_days="${BACKUP_RETENTION_DAYS:-30}" +minimum_copies="${BACKUP_MINIMUM_COPIES:-7}" + +case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac +case "$retention_days:$minimum_copies" in *[!0-9:]*|:*|*:) echo "Retention values must be integers" >&2; exit 1;; esac +[ -d "$destination" ] || exit 0 + +count=0 +find "$destination" -maxdepth 1 -type f -name 'mobilityops-*.dump' | sort -r | while IFS= read -r file; do + count=$((count + 1)) + if [ "$count" -gt "$minimum_copies" ] && [ -n "$(find "$file" -mtime "+$retention_days" -print)" ]; then + rm -f -- "$file" "${file}.sha256" + fi +done diff --git a/deploy/unraid/scheduled-backup.sh b/deploy/unraid/scheduled-backup.sh new file mode 100755 index 0000000..00b5a03 --- /dev/null +++ b/deploy/unraid/scheduled-backup.sh @@ -0,0 +1,41 @@ +#!/bin/sh +set -eu + +destination="${BACKUP_DESTINATION:-/backups}" +secondary="${BACKUP_SECONDARY_DESTINATION:-}" +interval="${BACKUP_INTERVAL_SECONDS:-86400}" +retention_days="${BACKUP_RETENTION_DAYS:-30}" +minimum_copies="${BACKUP_MINIMUM_COPIES:-7}" + +case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac +case "$interval:$retention_days:$minimum_copies" in *[!0-9:]*|:*|*:) echo "Backup settings must be integers" >&2; exit 1;; esac +mkdir -p "$destination" +[ -z "$secondary" ] || mkdir -p "$secondary" + +while true; do + timestamp="$(date -u +%Y%m%dT%H%M%SZ)" + target="$destination/mobilityops-$timestamp.dump" + temporary="$target.partial" + rm -f "$temporary" + if pg_dump --format=custom --no-owner --no-acl \ + --host=db --username="$POSTGRES_USER" "$POSTGRES_DB" > "$temporary" \ + && pg_restore --list "$temporary" > /dev/null; then + mv "$temporary" "$target" + sha256sum "$target" > "$target.sha256" + if [ -n "$secondary" ]; then + cp "$target" "$target.sha256" "$secondary/" + fi + date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-success" + BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \ + /opt/mobilityops/prune-postgres-backups.sh "$destination" + if [ -n "$secondary" ]; then + BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \ + /opt/mobilityops/prune-postgres-backups.sh "$secondary" + fi + echo "Verified database backup: $target" + else + rm -f "$temporary" + echo "Database backup failed at $timestamp" >&2 + fi + sleep "$interval" +done diff --git a/deploy/unraid/verify-postgres-backups.sh b/deploy/unraid/verify-postgres-backups.sh new file mode 100755 index 0000000..38d9032 --- /dev/null +++ b/deploy/unraid/verify-postgres-backups.sh @@ -0,0 +1,12 @@ +#!/bin/sh +set -eu + +destination="${1:-backups/postgres}" +case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac +latest="$(find "$destination" -maxdepth 1 -type f -name 'mobilityops-*.dump' | sort -r | head -n 1)" +[ -n "$latest" ] || { echo "No MobilityOps backup found" >&2; exit 1; } +[ -f "$latest.sha256" ] || { echo "Checksum missing for $latest" >&2; exit 1; } +(cd "$destination" && sha256sum -c "$(basename "$latest.sha256")") +docker compose -p "${COMPOSE_PROJECT_NAME:-mobilityops}" \ + -f compose.yaml -f compose.unraid.yaml exec -T db pg_restore --list < "$latest" > /dev/null +printf 'Verified: %s\n' "$latest"