M23: automate verified database backups

2026-08-10 15:46:06 +02:00
parent 689e499634
commit f0f1be83ae
8 changed files with 142 additions and 4 deletions
@@ -32,6 +32,14 @@ METRICS_BEARER_TOKEN=
 GRAFANA_ADMIN_USER=admin
 GRAFANA_ADMIN_PASSWORD=change-me-before-start

+# Verified scheduled PostgreSQL backups (Unraid override).
+BACKUP_INTERVAL_SECONDS=86400
+BACKUP_RETENTION_DAYS=30
+BACKUP_MINIMUM_COPIES=7
+# Set both values to copy every verified backup to an independently mounted path.
+BACKUP_SECONDARY_DESTINATION=
+MOBILITYOPS_BACKUP_SECONDARY_DIR=./backups/offsite
+
 # Demo presentation (fictional org identity, badge/manifest, reset safety valve).
 # DEMO_ALLOW_RESET=false permanently disables POST /api/v1/demo/reset (403), independent
 # of role -- a safety valve for any environment where the dataset must not be rebuildable.
@@ -2537,3 +2537,20 @@ evidence yet."
  Prometheus `promtool` accepted the scrape config and all six rules; merged Compose and
  Grafana dashboard JSON validate. Exact next action: automate verified backups,
  retention and restore-readiness checks.
+
+## M23 — scheduled and retained recovery points (2026-08-10)
+
+- Added a continuously running Unraid backup service that waits for healthy PostgreSQL,
+  creates an immediate custom-format dump and repeats at a configurable interval without
+  access to the Docker socket. The existing on-demand path remains supported.
+- Every backup must pass `pg_restore --list` before publication and receives a SHA-256
+  sidecar. Retention defaults to 30 days while always preserving at least seven newest
+  restore points. An optional independently mounted secondary destination receives the
+  same verified artifacts.
+- Added a latest-success marker and container healthcheck that detects a missed 26-hour
+  recovery point, a standalone verification command, guarded destinations and bounded
+  backup/log storage.
+- Evidence: all shell scripts pass Alpine `sh -n`; merged Unraid Compose validates; a
+  disposable PostgreSQL instance produced a real dump, checksum verification passed and
+  `pg_restore --list` accepted the artifact. Exact next action: implement privacy export,
+  anonymisation safeguards, retention reporting and governance documentation.
@@ -34,3 +34,31 @@ services:
      interval: 10s
      timeout: 5s
      retries: 20
+
+  backup:
+    image: postgres:16-alpine
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-mobilityops}
+      POSTGRES_USER: ${POSTGRES_USER:-mobilityops}
+      PGPASSWORD: ${POSTGRES_PASSWORD:-mobilityops}
+      BACKUP_DESTINATION: /backups
+      BACKUP_SECONDARY_DESTINATION: ${BACKUP_SECONDARY_DESTINATION:-}
+      BACKUP_INTERVAL_SECONDS: ${BACKUP_INTERVAL_SECONDS:-86400}
+      BACKUP_RETENTION_DAYS: ${BACKUP_RETENTION_DAYS:-30}
+      BACKUP_MINIMUM_COPIES: ${BACKUP_MINIMUM_COPIES:-7}
+    command: ["/opt/mobilityops/scheduled-backup.sh"]
+    volumes:
+      - ./backups/postgres:/backups
+      - ${MOBILITYOPS_BACKUP_SECONDARY_DIR:-./backups/offsite}:/offsite
+      - ./deploy/unraid:/opt/mobilityops:ro
+    depends_on:
+      db:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD-SHELL", "find /backups/latest-success -mmin -1560 -print -quit | grep -q ."]
+      interval: 30m
+      timeout: 5s
+      retries: 3
+      start_period: 5m
+    networks: [mobilityops]
@@ -25,7 +25,7 @@ cd /mnt/user/appdata/mobilityops
 ./deploy/unraid/configure-env.sh \
  http://192.168.10.150:1236 \
  http://192.168.10.150:5678/webhook/mobilityops-return
-docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml up --build -d db api web
+docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml up --build -d db api web backup
 docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml exec api \
  python -m app.cli seed --reset
 ```
@@ -55,14 +55,22 @@ docker logs --tail=200 n8n

 ## Backup and restore

-Create and structurally verify a timestamped PostgreSQL custom-format backup:
+The `backup` service creates a backup immediately and then every 24 hours. Every dump is
+validated with `pg_restore --list`, receives a SHA-256 sidecar and is retained for 30 days
+with at least seven copies protected from pruning. Its healthcheck becomes unhealthy when
+no successful backup has been recorded for 26 hours. Configure
+`BACKUP_SECONDARY_DESTINATION=/offsite` plus an independently mounted
+`MOBILITYOPS_BACKUP_SECONDARY_DIR` for a second copy.
+
+Create an additional on-demand backup or verify the newest scheduled backup:

 ```bash
 ./deploy/unraid/backup-postgres.sh
+./deploy/unraid/verify-postgres-backups.sh
 ```

-Copy backups off the server according to the host backup policy. A restore is deliberately
-guarded and creates an additional safety backup before replacing the database:
+A restore is deliberately guarded and creates an additional safety backup before
+replacing the database:

 ```bash
 ./deploy/unraid/restore-postgres.sh \
@@ -4,12 +4,15 @@ set -eu
 cd "$(dirname "$0")/../.."
 project="${COMPOSE_PROJECT_NAME:-mobilityops}"
 destination="${1:-backups/postgres}"
+retention_days="${BACKUP_RETENTION_DAYS:-30}"
+minimum_copies="${BACKUP_MINIMUM_COPIES:-7}"
 timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
 backup_file="${destination}/mobilityops-${timestamp}.dump"
 temporary_file="${backup_file}.partial"
 compose_files="-f compose.yaml -f compose.unraid.yaml"

 mkdir -p "$destination"
+case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
 trap 'rm -f "$temporary_file"' EXIT INT TERM

 docker compose -p "$project" $compose_files exec -T db sh -eu -c \
@@ -19,5 +22,8 @@ docker compose -p "$project" $compose_files exec -T db sh -eu -c \
 docker compose -p "$project" $compose_files exec -T db pg_restore --list \
  < "$temporary_file" > /dev/null
 mv "$temporary_file" "$backup_file"
+sha256sum "$backup_file" > "${backup_file}.sha256"
+BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \
+  ./deploy/unraid/prune-postgres-backups.sh "$destination"
 trap - EXIT INT TERM
 printf '%s\n' "$backup_file"
@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+
+destination="${1:-backups/postgres}"
+retention_days="${BACKUP_RETENTION_DAYS:-30}"
+minimum_copies="${BACKUP_MINIMUM_COPIES:-7}"
+
+case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
+case "$retention_days:$minimum_copies" in *[!0-9:]*|:*|*:) echo "Retention values must be integers" >&2; exit 1;; esac
+[ -d "$destination" ] || exit 0
+
+count=0
+find "$destination" -maxdepth 1 -type f -name 'mobilityops-*.dump' | sort -r | while IFS= read -r file; do
+  count=$((count + 1))
+  if [ "$count" -gt "$minimum_copies" ] && [ -n "$(find "$file" -mtime "+$retention_days" -print)" ]; then
+    rm -f -- "$file" "${file}.sha256"
+  fi
+done
@@ -0,0 +1,41 @@
+#!/bin/sh
+set -eu
+
+destination="${BACKUP_DESTINATION:-/backups}"
+secondary="${BACKUP_SECONDARY_DESTINATION:-}"
+interval="${BACKUP_INTERVAL_SECONDS:-86400}"
+retention_days="${BACKUP_RETENTION_DAYS:-30}"
+minimum_copies="${BACKUP_MINIMUM_COPIES:-7}"
+
+case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
+case "$interval:$retention_days:$minimum_copies" in *[!0-9:]*|:*|*:) echo "Backup settings must be integers" >&2; exit 1;; esac
+mkdir -p "$destination"
+[ -z "$secondary" ] || mkdir -p "$secondary"
+
+while true; do
+  timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+  target="$destination/mobilityops-$timestamp.dump"
+  temporary="$target.partial"
+  rm -f "$temporary"
+  if pg_dump --format=custom --no-owner --no-acl \
+      --host=db --username="$POSTGRES_USER" "$POSTGRES_DB" > "$temporary" \
+      && pg_restore --list "$temporary" > /dev/null; then
+    mv "$temporary" "$target"
+    sha256sum "$target" > "$target.sha256"
+    if [ -n "$secondary" ]; then
+      cp "$target" "$target.sha256" "$secondary/"
+    fi
+    date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-success"
+    BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \
+      /opt/mobilityops/prune-postgres-backups.sh "$destination"
+    if [ -n "$secondary" ]; then
+      BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \
+        /opt/mobilityops/prune-postgres-backups.sh "$secondary"
+    fi
+    echo "Verified database backup: $target"
+  else
+    rm -f "$temporary"
+    echo "Database backup failed at $timestamp" >&2
+  fi
+  sleep "$interval"
+done
@@ -0,0 +1,12 @@
+#!/bin/sh
+set -eu
+
+destination="${1:-backups/postgres}"
+case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
+latest="$(find "$destination" -maxdepth 1 -type f -name 'mobilityops-*.dump' | sort -r | head -n 1)"
+[ -n "$latest" ] || { echo "No MobilityOps backup found" >&2; exit 1; }
+[ -f "$latest.sha256" ] || { echo "Checksum missing for $latest" >&2; exit 1; }
+(cd "$destination" && sha256sum -c "$(basename "$latest.sha256")")
+docker compose -p "${COMPOSE_PROJECT_NAME:-mobilityops}" \
+  -f compose.yaml -f compose.unraid.yaml exec -T db pg_restore --list < "$latest" > /dev/null
+printf 'Verified: %s\n' "$latest"