M48: harden demo operations and offsite recovery

deploy/unraid/Dockerfile.backup-tools

@@ -0,0 +1,17 @@
+FROM golang:1.26.6-alpine@sha256:3889b425f035be855a72fb4755265311293b6d414521f0a519d819df32222d83 AS rclone
+RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
+    CGO_ENABLED=0 go install -trimpath \
+    -ldflags '-s -X github.com/rclone/rclone/fs.Version=v1.75.0' \
+    github.com/rclone/rclone@v1.75.0
+
+FROM postgres:16-alpine@sha256:cf78e76683b9ca8c5733cbbdce6c9262b45b6767934dd0a95e671f9a0fc20685
+ARG VCS_REF=development
+ARG BUILD_DATE=unknown
+LABEL org.opencontainers.image.title="MobilityOps backup tools" \
+      org.opencontainers.image.source="https://github.com/ITWorxBE/MobilityOps" \
+      org.opencontainers.image.revision="$VCS_REF" \
+      org.opencontainers.image.created="$BUILD_DATE"
+COPY --from=rclone /go/bin/rclone /usr/local/bin/rclone
+# The PostgreSQL image's entrypoint uses gosu only when starting the database server;
+# this tools-only image always overrides that entrypoint with the sync worker.
+RUN rm -f /usr/local/bin/gosu

deploy/unraid/README.md

@@ -33,10 +33,19 @@ sha256sum /tmp/mobilityops-source.tar.gz
 ```
 
 `deploy-release.sh` stages a clean, commit-named release, builds OCI-labelled immutable
-API/web images, promotes without a seed/reset, and verifies migrations, readiness,
-backups and observability. Run `python -m app.cli seed --reset` only for initial setup or
+API/web images, starts two candidate replicas per stateless service behind a stable gateway,
+and removes the previous replicas only after public readiness passes. It promotes without a
+seed/reset and leaves the database, backups and monitoring untouched during routine app
+releases. Run `python -m app.cli seed --reset` only for initial setup or
 an explicit synthetic-demo reset; it is never part of a routine deployment.
 
+Refresh the pinned stateful and monitoring containers explicitly after reviewing their
+version/configuration changes:
+
+```bash
+./deploy/unraid/refresh-infrastructure.sh
+```
+
 Migrations run automatically in the API entrypoint. Import and publish the MobilityOps
 workflow into the existing n8n container:
 
@@ -66,9 +75,22 @@ The `backup` service creates a backup immediately and then every 24 hours. Every
 checked by SHA-256 and `pg_restore --list`; at least weekly the newest dump is also restored
 into a disposable database and its migration revision and core table counts are verified.
 Backups are retained for 30 days with at least seven copies protected from pruning. Its
-healthcheck becomes unhealthy when the daily backup or eight-day restore-drill SLA is missed. Configure
-`BACKUP_SECONDARY_DESTINATION=/offsite` plus an independently mounted
-`MOBILITYOPS_BACKUP_SECONDARY_DIR` for a second copy.
+healthcheck becomes unhealthy when the daily backup or eight-day restore-drill SLA is missed.
+
+For this synthetic demo, OneDrive is the supported real off-site target. The optional worker
+uploads the newest locally verified dump and checksum, downloads both again, verifies the
+round trip, and performs a weekly restore into a disposable database. Its OAuth token remains
+in the untracked mode-0600 rclone configuration directory. Configure it once from an
+interactive server terminal (Microsoft browser authorization is required):
+
+```bash
+./deploy/unraid/configure-onedrive-backup.sh onedrive
+docker inspect --format '{{.State.Health.Status}}' mobilityops-offsite-backup-1
+```
+
+Set `RCLONE_ONEDRIVE_PATH` to change the default `FleetOps/backups` folder. Do not set
+`BACKUP_SECONDARY_DESTINATION` to another path on the same server and call that off-site;
+that legacy mount remains available only for an independently mounted filesystem.
 
 Create an additional on-demand backup, verify the newest backup, or execute the isolated
 restore drill:

deploy/unraid/configure-onedrive-backup.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+set -eu
+
+root="${MOBILITYOPS_DEPLOY_ROOT:-/mnt/user/appdata/mobilityops}"
+config_dir="${MOBILITYOPS_RCLONE_CONFIG_DIR:-$root/.secrets/rclone}"
+remote="${1:-onedrive}"
+
+case "$config_dir" in ""|"/"|".") echo "Unsafe rclone configuration directory" >&2; exit 1;; esac
+case "$remote" in ""|*:*|*/*) echo "Remote must be a simple rclone remote name" >&2; exit 1;; esac
+mkdir -p "$config_dir"
+chmod 700 "$config_dir"
+
+revision="$(cat "$root/.deploy/source-revision")"
+release_dir="$root/.deploy/releases/$revision"
+short_revision="$(printf '%s' "$revision" | cut -c1-12)"
+image="mobilityops-backup-tools:$short_revision"
+docker build --build-arg "VCS_REF=$revision" --tag "$image" \
+  --file "$release_dir/deploy/unraid/Dockerfile.backup-tools" "$release_dir"
+
+echo "Configure Microsoft OneDrive as remote '$remote'; OAuth opens in your browser."
+docker run --rm -it --entrypoint rclone -v "$config_dir:/config/rclone" "$image" config
+chmod 600 "$config_dir/rclone.conf"
+docker run --rm --entrypoint rclone -v "$config_dir:/config/rclone" "$image" lsd "$remote:"
+
+history="$(tail -n 1 "$root/.deploy/release-history.log")"
+# shellcheck disable=SC2086
+set -- $history
+[ "$1" = "$revision" ] || { echo "Release history does not match active revision" >&2; exit 1; }
+export MOBILITYOPS_API_IMAGE="$2" MOBILITYOPS_WEB_IMAGE="$3"
+export MOBILITYOPS_SOURCE_REVISION="$revision"
+export MOBILITYOPS_BACKUP_TOOLS_IMAGE="$image"
+export MOBILITYOPS_RCLONE_CONFIG_DIR="$config_dir"
+export MOBILITYOPS_BACKUP_DIR="${MOBILITYOPS_BACKUP_DIR:-$root/backups/postgres}"
+export MOBILITYOPS_OFFSITE_VERIFY_DIR="${MOBILITYOPS_OFFSITE_VERIFY_DIR:-$root/backups/offsite-verify}"
+export RCLONE_ONEDRIVE_REMOTE="$remote"
+compose="docker compose --env-file $root/.env -p ${COMPOSE_PROJECT_NAME:-mobilityops} -f $release_dir/compose.yaml -f $release_dir/compose.unraid.yaml -f $release_dir/compose.observability.yaml -f $release_dir/compose.release.yaml --profile offsite"
+$compose up --no-build -d offsite-backup
+echo "OneDrive remote '$remote' is reachable and the off-site backup worker is starting."

deploy/unraid/deploy-release.sh

@@ -50,29 +50,191 @@ for image in "$api_image" "$web_image"; do
 done
 
 compose="docker compose --env-file $root/.env -p $project -f $release_dir/compose.yaml -f $release_dir/compose.unraid.yaml -f $release_dir/compose.observability.yaml -f $release_dir/compose.release.yaml --profile observability"
-old_api_id="$(docker inspect --format '{{.Image}}' "$project-api-1" 2>/dev/null || true)"
-old_web_id="$(docker inspect --format '{{.Image}}' "$project-web-1" 2>/dev/null || true)"
 export MOBILITYOPS_API_IMAGE="$api_image" MOBILITYOPS_WEB_IMAGE="$web_image"
-$compose up --no-build -d api web backup prometheus alertmanager grafana
+gateway_config="$root/.deploy/gateway.conf"
+candidate_gateway="$(mktemp "$root/.deploy/gateway-candidate.XXXXXX")"
+previous_gateway="$(mktemp "$root/.deploy/gateway-previous.XXXXXX")"
+sed -e "s/server web:80/server web-$short_revision:80/" \
+  -e "s/server api:8000/server api-$short_revision:8000/" \
+  "$release_dir/deploy/unraid/gateway.conf" > "$candidate_gateway"
+if [ -f "$gateway_config" ]; then
+  cp "$gateway_config" "$previous_gateway"
+else
+  cp "$candidate_gateway" "$gateway_config"
+fi
+export MOBILITYOPS_GATEWAY_CONFIG="$gateway_config"
+
+# Bootstrap infrastructure only when it is absent. Existing stateful/monitoring
+# containers are deliberately not reconciled against every commit-named source path.
+missing_infrastructure=""
+for service in db backup prometheus alertmanager grafana; do
+  [ -n "$($compose ps -q "$service")" ] || missing_infrastructure="$missing_infrastructure $service"
+done
+if [ -n "$missing_infrastructure" ]; then
+  # shellcheck disable=SC2086
+  $compose up --no-build -d $missing_infrastructure
+fi
+
+# Apply schema changes once before starting the new stateless replicas. Migrations in a
+# release must remain backwards compatible with the still-serving previous API.
+$compose run --rm --no-deps --entrypoint alembic api upgrade head
+
+network="$(docker inspect --format '{{range $name, $network := .NetworkSettings.Networks}}{{$name}}{{end}}' "$project-db-1")"
+[ -n "$network" ] || { echo "MobilityOps network was not found" >&2; exit 1; }
+
+old_api_ids="$(docker ps -q --filter label=com.mobilityops.role=api --filter "label=com.mobilityops.project=$project")"
+old_web_ids="$(docker ps -q --filter label=com.mobilityops.role=web --filter "label=com.mobilityops.project=$project")"
+if [ -z "$old_api_ids" ]; then
+  old_api_ids="$(docker ps -q --filter "name=^/${project}-api-")"
+fi
+if [ -z "$old_web_ids" ]; then
+  old_web_ids="$(docker ps -q --filter "name=^/${project}-web-")"
+fi
+
+# Reuse the effective, already-secret-resolved API environment without printing it.
+api_environment="$(mktemp "$root/.deploy/api-environment.XXXXXX")"
+chmod 600 "$api_environment"
+cleanup() {
+  rm -f "$api_environment" "$candidate_gateway" "$previous_gateway"
+}
+trap cleanup EXIT INT TERM
+if [ -n "$old_api_ids" ]; then
+  # shellcheck disable=SC2086
+  first_old_api="$(printf '%s\n' $old_api_ids | head -n 1)"
+  docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$first_old_api" > "$api_environment"
+else
+  # A clean bootstrap first lets Compose resolve the complete API environment. This
+  # container is retained as the previous slot until the candidates pass.
+  $compose up --no-build --no-deps -d api web
+  old_api_ids="$($compose ps -q api)"
+  old_web_ids="$($compose ps -q web)"
+  docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$old_api_ids" > "$api_environment"
+fi
+
+new_api_ids=""
+new_web_ids=""
+old_web_image=""
+# shellcheck disable=SC2086
+[ -z "$old_web_ids" ] || old_web_image="$(docker inspect --format '{{.Config.Image}}' "$(printf '%s\n' $old_web_ids | head -n 1)")"
+for replica in 1 2; do
+  name="$project-api-$short_revision-$replica"
+  id="$(docker run -d --name "$name" --restart unless-stopped \
+    --label com.mobilityops.role=api --label "com.mobilityops.project=$project" \
+    --label "com.mobilityops.revision=$revision" \
+    --network "$network" --network-alias "api-$short_revision" --env-file "$api_environment" \
+    --env RUN_MIGRATIONS=false "$api_image")"
+  new_api_ids="$new_api_ids $id"
+done
+
+wait_for_ids() {
+  role="$1"
+  shift
+  attempt=0
+  while true; do
+    healthy=0
+    count=0
+    for id in "$@"; do
+      count=$((count + 1))
+      if [ "$role" = "api" ]; then
+        docker exec "$id" python -c \
+          "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/ready')" \
+          > /dev/null 2>&1 && healthy=$((healthy + 1))
+      else
+        docker exec "$id" wget -q -O /dev/null http://127.0.0.1/health \
+          > /dev/null 2>&1 && healthy=$((healthy + 1))
+      fi
+    done
+    [ "$count" -eq 2 ] && [ "$healthy" -eq 2 ] && return 0
+    attempt=$((attempt + 1))
+    [ "$attempt" -lt 60 ] || { echo "$role candidates did not become healthy" >&2; return 1; }
+    sleep 2
+  done
+}
+
+# shellcheck disable=SC2086
+if ! wait_for_ids api $new_api_ids; then
+  docker rm -f $new_api_ids > /dev/null 2>&1 || true
+  exit 1
+fi
+
+for replica in 1 2; do
+  name="$project-web-$short_revision-$replica"
+  id="$(docker run -d --name "$name" --restart unless-stopped \
+    --label com.mobilityops.role=web --label "com.mobilityops.project=$project" \
+    --label "com.mobilityops.revision=$revision" \
+    --network "$network" --network-alias "web-$short_revision" "$web_image")"
+  new_web_ids="$new_web_ids $id"
+done
+# shellcheck disable=SC2086
+if ! wait_for_ids web $new_web_ids; then
+  docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
+  exit 1
+fi
+
+# Install the stable gateway once. Future releases leave it running while Docker DNS
+# exposes both old and new web aliases. The first migration from direct port ownership
+# necessarily has a brief hand-off while port 1236 moves to the gateway.
+gateway_id="$($compose ps -q gateway)"
+cp "$candidate_gateway" "$gateway_config"
+if [ -z "$gateway_id" ]; then
+  if [ -n "$old_web_ids" ]; then
+    # shellcheck disable=SC2086
+    docker rm -f $old_web_ids > /dev/null
+    old_web_ids=""
+  fi
+  $compose up --no-build --no-deps -d gateway
+else
+  if ! docker exec "$gateway_id" nginx -t; then
+    [ ! -s "$previous_gateway" ] || cp "$previous_gateway" "$gateway_config"
+    # shellcheck disable=SC2086
+    docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
+    echo "Candidate gateway configuration was rejected; previous replicas remain active" >&2
+    exit 1
+  fi
+  docker exec "$gateway_id" nginx -s reload
+fi
 
 attempt=0
 until curl -fsS http://127.0.0.1:1236/health/ready > /dev/null; do
   attempt=$((attempt + 1))
   if [ "$attempt" -ge 30 ]; then
-    if [ -n "$old_api_id" ] && [ -n "$old_web_id" ]; then
-      rollback_api="mobilityops-api:rollback-$short_revision"
-      rollback_web="mobilityops-web:rollback-$short_revision"
-      docker tag "$old_api_id" "$rollback_api"
-      docker tag "$old_web_id" "$rollback_web"
-      export MOBILITYOPS_API_IMAGE="$rollback_api" MOBILITYOPS_WEB_IMAGE="$rollback_web"
-      $compose up --no-build -d api web || true
+    if [ -n "$gateway_id" ] && [ -s "$previous_gateway" ]; then
+      cp "$previous_gateway" "$gateway_config"
+      docker exec "$gateway_id" nginx -t && docker exec "$gateway_id" nginx -s reload
+      # shellcheck disable=SC2086
+      docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
+    elif [ -n "$old_web_image" ]; then
+      # The first gateway migration keeps the already healthy candidate APIs because
+      # the rendered gateway points at their versioned alias.
+      # shellcheck disable=SC2086
+      docker rm -f $new_web_ids > /dev/null 2>&1 || true
+      for replica in 1 2; do
+        docker run -d --name "$project-web-rollback-$short_revision-$replica" --restart unless-stopped \
+          --label com.mobilityops.role=web --label "com.mobilityops.project=$project" \
+          --label com.mobilityops.revision=rollback \
+          --network "$network" --network-alias "web-$short_revision" "$old_web_image" > /dev/null
+      done
+    else
+      # shellcheck disable=SC2086
+      docker rm -f $new_api_ids $new_web_ids > /dev/null 2>&1 || true
     fi
-    echo "Release failed readiness; source revision was not promoted" >&2
+    echo "Release failed readiness; previous API and restored web replicas remain active" >&2
     exit 1
   fi
   sleep 2
 done
-$compose exec -T api alembic current
+
+# Promotion is start-first: only now remove the previous containers. A short drain lets
+# workers using the pre-reload configuration complete their in-flight requests.
+sleep 3
+# shellcheck disable=SC2086
+[ -z "$old_api_ids" ] || docker rm -f $old_api_ids > /dev/null
+# shellcheck disable=SC2086
+[ -z "$old_web_ids" ] || docker rm -f $old_web_ids > /dev/null
+curl -fsS http://127.0.0.1:1236/health/ready > /dev/null
+# shellcheck disable=SC2086
+first_new_api="$(printf '%s\n' $new_api_ids | head -n 1)"
+docker exec "$first_new_api" alembic current
 for service in backup prometheus alertmanager grafana; do
   attempt=0
   until status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$project-$service-1" 2>/dev/null)" \

deploy/unraid/gateway.conf

@@ -0,0 +1,58 @@
+resolver 127.0.0.11 valid=2s ipv6=off;
+limit_req_zone $binary_remote_addr zone=demo_login:10m rate=600r/m;
+
+upstream fleet_ops_web {
+  zone fleet_ops_web 64k;
+  server web:80 resolve max_fails=1 fail_timeout=2s;
+  keepalive 32;
+}
+
+upstream fleet_ops_api {
+  zone fleet_ops_api 64k;
+  server api:8000 resolve max_fails=1 fail_timeout=2s;
+  keepalive 32;
+}
+
+server {
+  listen 80;
+  server_name _;
+  server_tokens off;
+
+  location = /api/v1/demo/login {
+    limit_req zone=demo_login burst=100 nodelay;
+    proxy_pass http://fleet_ops_api;
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto $scheme;
+  }
+
+  location /api/ {
+    proxy_pass http://fleet_ops_api;
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto $scheme;
+  }
+
+  location /health {
+    proxy_pass http://fleet_ops_api/health;
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+  }
+
+  location / {
+    proxy_pass http://fleet_ops_web;
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_next_upstream error timeout http_502 http_503 http_504;
+    proxy_next_upstream_tries 3;
+    proxy_next_upstream_timeout 3s;
+    proxy_connect_timeout 1s;
+  }
+}

deploy/unraid/refresh-infrastructure.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+set -eu
+
+root="${MOBILITYOPS_DEPLOY_ROOT:-/mnt/user/appdata/mobilityops}"
+project="${COMPOSE_PROJECT_NAME:-mobilityops}"
+revision="$(cat "$root/.deploy/source-revision")"
+release_dir="$root/.deploy/releases/$revision"
+[ -d "$release_dir" ] || { echo "Active release directory is missing" >&2; exit 1; }
+
+history="$(tail -n 1 "$root/.deploy/release-history.log")"
+# shellcheck disable=SC2086
+set -- $history
+[ "$1" = "$revision" ] || { echo "Release history does not match active revision" >&2; exit 1; }
+export MOBILITYOPS_API_IMAGE="$2" MOBILITYOPS_WEB_IMAGE="$3"
+compose="docker compose --env-file $root/.env -p $project -f $release_dir/compose.yaml -f $release_dir/compose.unraid.yaml -f $release_dir/compose.observability.yaml -f $release_dir/compose.release.yaml --profile observability"
+
+$compose up --no-build -d db backup prometheus alertmanager grafana
+for service in db backup prometheus alertmanager grafana; do
+  attempt=0
+  until status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$project-$service-1" 2>/dev/null)" \
+      && [ "$status" = "healthy" ]; do
+    attempt=$((attempt + 1))
+    [ "$attempt" -lt 60 ] || { echo "$service did not become healthy" >&2; exit 1; }
+    sleep 5
+  done
+done
+echo "Refreshed Fleet Ops infrastructure for $revision"

deploy/unraid/sync-onedrive-backups.sh

@@ -0,0 +1,67 @@
+#!/bin/sh
+set -eu
+
+destination="${BACKUP_DESTINATION:-/backups}"
+verify_dir="${BACKUP_OFFSITE_VERIFY_DIR:-/offsite-verify}"
+remote="${RCLONE_ONEDRIVE_REMOTE:-onedrive}"
+remote_path="${RCLONE_ONEDRIVE_PATH:-FleetOps/backups}"
+interval="${BACKUP_OFFSITE_INTERVAL_SECONDS:-900}"
+restore_drill_interval="${BACKUP_RESTORE_DRILL_INTERVAL_SECONDS:-604800}"
+retention_days="${BACKUP_RETENTION_DAYS:-30}"
+
+case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
+case "$verify_dir" in ""|"/"|".") echo "Unsafe off-site verify directory: $verify_dir" >&2; exit 1;; esac
+case "$remote" in ""|*:*|*/*) echo "Rclone remote must be a configured remote name" >&2; exit 1;; esac
+case "$remote_path" in ""|"/"|/*|*".."*) echo "Unsafe OneDrive backup path" >&2; exit 1;; esac
+case "$interval:$restore_drill_interval:$retention_days" in *[!0-9:]*|:*|*:) echo "Off-site intervals and retention must be integers" >&2; exit 1;; esac
+[ "$retention_days" -ge 1 ] || { echo "Off-site retention must be at least one day" >&2; exit 1; }
+[ -f "${RCLONE_CONFIG:-/config/rclone/rclone.conf}" ] || {
+  echo "Rclone configuration is missing" >&2
+  exit 1
+}
+
+mkdir -p "$verify_dir"
+
+while true; do
+  latest="$(find "$destination" -maxdepth 1 -type f -name 'mobilityops-*.dump' | sort -r | head -n 1)"
+  if [ -n "$latest" ] && [ -f "$latest.sha256" ]; then
+    name="$(basename "$latest")"
+    checksum_name="$name.sha256"
+    remote_root="$remote:$remote_path"
+    temporary_dump="$verify_dir/$name.partial"
+    temporary_checksum="$verify_dir/$checksum_name.partial"
+    verified_dump="$verify_dir/$name"
+
+    rm -f "$temporary_dump" "$temporary_checksum"
+    if rclone copyto "$latest" "$remote_root/$name" --checksum --retries 3 --low-level-retries 5 \
+        && rclone copyto "$latest.sha256" "$remote_root/$checksum_name" --checksum --retries 3 --low-level-retries 5 \
+        && rclone copyto "$remote_root/$name" "$temporary_dump" --retries 3 --low-level-retries 5 \
+        && rclone copyto "$remote_root/$checksum_name" "$temporary_checksum" --retries 3 --low-level-retries 5 \
+        && mv "$temporary_dump" "$verified_dump" \
+        && mv "$temporary_checksum" "$verify_dir/$checksum_name" \
+        && (cd "$verify_dir" && sha256sum -c "$checksum_name") \
+        && pg_restore --list "$verified_dump" > /dev/null; then
+      date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-offsite-success"
+      drill_minutes=$((restore_drill_interval / 60))
+      if [ ! -f "$destination/latest-offsite-restore-drill" ] \
+          || ! find "$destination/latest-offsite-restore-drill" -mmin "-$drill_minutes" -print -quit | grep -q .; then
+        if /opt/mobilityops/restore-drill-postgres.sh "$verified_dump"; then
+          date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-offsite-restore-drill"
+        else
+          echo "Off-site restore drill failed for $name" >&2
+        fi
+      fi
+      rclone delete "$remote_root" --min-age "${retention_days}d" \
+        --include 'mobilityops-*.dump' --include 'mobilityops-*.dump.sha256' \
+        --retries 3 --low-level-retries 5
+      find "$verify_dir" -type f -mtime "+$retention_days" -delete
+      echo "Verified OneDrive round trip: $remote_root/$name"
+    else
+      rm -f "$temporary_dump" "$temporary_checksum"
+      echo "OneDrive backup synchronization failed for $name" >&2
+    fi
+  else
+    echo "No verified local backup is available for OneDrive synchronization" >&2
+  fi
+  sleep "$interval"
+done