M44: harden release integrity and assurance

deploy/observability/alertmanager.yml

@@ -0,0 +1,21 @@
+route:
+  receiver: fleetops-owner
+  group_by: [alertname, severity]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+
+receivers:
+  - name: fleetops-owner
+    webhook_configs:
+      - url: __WEBHOOK_URL__
+        send_resolved: true
+        http_config:
+          http_headers:
+            X-Fleet-Ops-Trigger-Token:
+              secrets: [__WEBHOOK_TOKEN__]
+
+inhibit_rules:
+  - source_matchers: ['alertname="MobilityOpsApiDown"']
+    target_matchers: ['severity="warning"']
+    equal: [alertname]

deploy/observability/alerts.yml

@@ -1,6 +1,12 @@
 groups:
   - name: mobilityops
     rules:
+      - alert: FleetOpsWatchdog
+        expr: vector(1)
+        labels: {severity: none}
+        annotations:
+          summary: Fleet Ops alert delivery watchdog
+          description: This alert must always be visible at the configured receiver.
       - alert: MobilityOpsApiDown
         expr: up{job="mobilityops-api"} == 0
         for: 2m

deploy/observability/prometheus.yml

@@ -5,6 +5,11 @@ global:
 rule_files:
   - /etc/prometheus/alerts.yml
 
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets: ["alertmanager:9093"]
+
 scrape_configs:
   - job_name: mobilityops-api
     metrics_path: /metrics

deploy/unraid/README.md

@@ -17,8 +17,9 @@ existing shared n8n remains available on its established port 5678.
 
 Create `.env` from `.env.example`, replace every placeholder secret, set
 `MOBILITYOPS_ENV=production`, set both public URLs to
-`https://fleetops.itworx.tech`, set `SESSION_COOKIE_SECURE=true`, and retain
-`KNOWLEDGE_PROVIDER=demo` while RAGcore is not available. The internal `1236` listener is
+`https://fleetops.itworx.tech`, set `SESSION_COOKIE_SECURE=true`, and configure
+`KNOWLEDGE_PROVIDER=ragcore` only after the RAGcore health and source inventory checks pass.
+The internal `1236` listener is
 an upstream for the TLS proxy, not a user-facing URL.
 
 ```bash
@@ -26,11 +27,16 @@ cd /mnt/user/appdata/mobilityops
 ./deploy/unraid/configure-env.sh \
   https://fleetops.itworx.tech \
   https://n8n.itworx.tech/webhook/mobilityops-return
-docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml up --build -d db api web backup
-docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml exec api \
-  python -m app.cli seed --reset
+git archive --format=tar.gz -o /tmp/mobilityops-source.tar.gz HEAD
+sha256sum /tmp/mobilityops-source.tar.gz
+# Copy the archive and run deploy-release.sh with its SHA-256 and full Git SHA.
 ```
 
+`deploy-release.sh` stages a clean, commit-named release, builds OCI-labelled immutable
+API/web images, promotes without a seed/reset, and verifies migrations, readiness,
+backups and observability. Run `python -m app.cli seed --reset` only for initial setup or
+an explicit synthetic-demo reset; it is never part of a routine deployment.
+
 Migrations run automatically in the API entrypoint. Import and publish the MobilityOps
 workflow into the existing n8n container:
 
@@ -57,17 +63,21 @@ docker logs --tail=200 n8n
 ## Backup and restore
 
 The `backup` service creates a backup immediately and then every 24 hours. Every dump is
-validated with `pg_restore --list`, receives a SHA-256 sidecar and is retained for 30 days
-with at least seven copies protected from pruning. Its healthcheck becomes unhealthy when
-no successful backup has been recorded for 26 hours. Configure
+checked by SHA-256 and `pg_restore --list`; at least weekly the newest dump is also restored
+into a disposable database and its migration revision and core table counts are verified.
+Backups are retained for 30 days with at least seven copies protected from pruning. Its
+healthcheck becomes unhealthy when the daily backup or eight-day restore-drill SLA is missed. Configure
 `BACKUP_SECONDARY_DESTINATION=/offsite` plus an independently mounted
 `MOBILITYOPS_BACKUP_SECONDARY_DIR` for a second copy.
 
-Create an additional on-demand backup or verify the newest scheduled backup:
+Create an additional on-demand backup, verify the newest backup, or execute the isolated
+restore drill:
 
 ```bash
 ./deploy/unraid/backup-postgres.sh
 ./deploy/unraid/verify-postgres-backups.sh
+docker compose -p mobilityops -f compose.yaml -f compose.unraid.yaml exec -T backup \
+  /opt/mobilityops/restore-drill-postgres.sh /backups/<backup>.dump
 ```
 
 A restore is deliberately guarded and creates an additional safety backup before

deploy/unraid/deploy-release.sh

@@ -0,0 +1,88 @@
+#!/bin/sh
+set -eu
+
+# Stage a committed source archive, build commit-labelled immutable images and promote
+# them without overlaying source files or reseeding persisted data.
+archive="${1:-}"
+expected_checksum="${2:-}"
+revision="${3:-}"
+root="${MOBILITYOPS_DEPLOY_ROOT:-/mnt/user/appdata/mobilityops}"
+project="${COMPOSE_PROJECT_NAME:-mobilityops}"
+
+[ "${#revision}" -eq 40 ] || { echo "Revision must be a full Git SHA" >&2; exit 1; }
+case "$revision" in *[!0-9a-f]*) echo "Revision must be lowercase hexadecimal" >&2; exit 1;; esac
+[ -f "$archive" ] || { echo "Archive not found: $archive" >&2; exit 1; }
+[ -n "$expected_checksum" ] || { echo "Expected SHA-256 is required" >&2; exit 1; }
+[ -f "$root/.env" ] || { echo "Production .env is missing" >&2; exit 1; }
+
+actual_checksum="$(sha256sum "$archive" | awk '{print $1}')"
+[ "$actual_checksum" = "$expected_checksum" ] || {
+  echo "Archive checksum mismatch" >&2
+  exit 1
+}
+
+release_root="$root/.deploy/releases"
+release_dir="$release_root/$revision"
+mkdir -p "$release_root"
+if [ -e "$release_dir" ]; then
+  echo "Release directory already exists; refusing to overwrite: $release_dir" >&2
+  exit 1
+fi
+mkdir "$release_dir"
+tar -xzf "$archive" -C "$release_dir"
+
+short_revision="$(printf '%s' "$revision" | cut -c1-12)"
+build_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+api_image="mobilityops-api:$short_revision"
+web_image="mobilityops-web:$short_revision"
+
+docker build --target runtime --build-arg "VCS_REF=$revision" --build-arg "BUILD_DATE=$build_date" \
+  --tag "$api_image" --file "$release_dir/backend/Dockerfile" "$release_dir"
+docker build --build-arg "VCS_REF=$revision" --build-arg "BUILD_DATE=$build_date" \
+  --tag "$web_image" "$release_dir/frontend"
+
+for image in "$api_image" "$web_image"; do
+  labelled_revision="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image")"
+  [ "$labelled_revision" = "$revision" ] || {
+    echo "Image revision label mismatch for $image" >&2
+    exit 1
+  }
+done
+
+compose="docker compose --env-file $root/.env -p $project -f $release_dir/compose.yaml -f $release_dir/compose.unraid.yaml -f $release_dir/compose.observability.yaml -f $release_dir/compose.release.yaml --profile observability"
+old_api_id="$(docker inspect --format '{{.Image}}' "$project-api-1" 2>/dev/null || true)"
+old_web_id="$(docker inspect --format '{{.Image}}' "$project-web-1" 2>/dev/null || true)"
+export MOBILITYOPS_API_IMAGE="$api_image" MOBILITYOPS_WEB_IMAGE="$web_image"
+$compose up --no-build -d api web backup prometheus alertmanager grafana
+
+attempt=0
+until curl -fsS http://127.0.0.1:1236/health/ready > /dev/null; do
+  attempt=$((attempt + 1))
+  if [ "$attempt" -ge 30 ]; then
+    if [ -n "$old_api_id" ] && [ -n "$old_web_id" ]; then
+      rollback_api="mobilityops-api:rollback-$short_revision"
+      rollback_web="mobilityops-web:rollback-$short_revision"
+      docker tag "$old_api_id" "$rollback_api"
+      docker tag "$old_web_id" "$rollback_web"
+      export MOBILITYOPS_API_IMAGE="$rollback_api" MOBILITYOPS_WEB_IMAGE="$rollback_web"
+      $compose up --no-build -d api web || true
+    fi
+    echo "Release failed readiness; source revision was not promoted" >&2
+    exit 1
+  fi
+  sleep 2
+done
+$compose exec -T api alembic current
+for service in backup prometheus alertmanager grafana; do
+  attempt=0
+  until status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$project-$service-1" 2>/dev/null)" \
+    && { [ "$status" = "healthy" ] || [ "$status" = "running" ]; }; do
+    attempt=$((attempt + 1))
+    [ "$attempt" -lt 60 ] || { echo "$service did not become healthy" >&2; exit 1; }
+    sleep 5
+  done
+done
+printf '%s\n' "$revision" > "$root/.deploy/source-revision"
+printf '%s %s %s %s\n' "$revision" "$api_image" "$web_image" "$expected_checksum" \
+  >> "$root/.deploy/release-history.log"
+echo "Promoted Fleet Ops release $revision"

deploy/unraid/restore-drill-postgres.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+set -eu
+
+backup_file="${1:-}"
+[ -f "$backup_file" ] || { echo "Restore-drill backup not found: $backup_file" >&2; exit 1; }
+
+timestamp="$(date -u +%Y%m%d%H%M%S)"
+drill_database="mobilityops_restore_drill_$timestamp"
+source_database="${POSTGRES_DB:-mobilityops}"
+database_host="${POSTGRES_HOST:-db}"
+
+cleanup() {
+  dropdb --if-exists --force --host="$database_host" --username="$POSTGRES_USER" \
+    "$drill_database" > /dev/null 2>&1 || true
+}
+trap cleanup EXIT INT TERM
+
+source_revision="$(psql --host="$database_host" --username="$POSTGRES_USER" \
+  --dbname="$source_database" --tuples-only --no-align \
+  --command='SELECT version_num FROM alembic_version')"
+[ -n "$source_revision" ] || { echo "Source Alembic revision is unavailable" >&2; exit 1; }
+
+createdb --host="$database_host" --username="$POSTGRES_USER" "$drill_database"
+pg_restore --no-owner --no-acl --host="$database_host" --username="$POSTGRES_USER" \
+  --dbname="$drill_database" "$backup_file"
+
+restored_revision="$(psql --host="$database_host" --username="$POSTGRES_USER" \
+  --dbname="$drill_database" --tuples-only --no-align \
+  --command='SELECT version_num FROM alembic_version')"
+[ "$restored_revision" = "$source_revision" ] || {
+  echo "Restored Alembic revision does not match production" >&2
+  exit 1
+}
+
+counts="$(psql --host="$database_host" --username="$POSTGRES_USER" \
+  --dbname="$drill_database" --tuples-only --no-align --field-separator=, \
+  --command='SELECT (SELECT count(*) FROM users), (SELECT count(*) FROM vehicles), (SELECT count(*) FROM bookings), (SELECT count(*) FROM audit_events)')"
+case "$counts" in
+  0,*|*,0,*|*,*,0,*|*,*,*,0) echo "Restored database is missing canonical records: $counts" >&2; exit 1 ;;
+esac
+
+cleanup
+trap - EXIT INT TERM
+echo "Restore drill passed: revision=$restored_revision counts=$counts"

deploy/unraid/scheduled-backup.sh

@@ -6,9 +6,10 @@ secondary="${BACKUP_SECONDARY_DESTINATION:-}"
 interval="${BACKUP_INTERVAL_SECONDS:-86400}"
 retention_days="${BACKUP_RETENTION_DAYS:-30}"
 minimum_copies="${BACKUP_MINIMUM_COPIES:-7}"
+restore_drill_interval="${BACKUP_RESTORE_DRILL_INTERVAL_SECONDS:-604800}"
 
 case "$destination" in ""|"/"|".") echo "Unsafe backup destination: $destination" >&2; exit 1;; esac
-case "$interval:$retention_days:$minimum_copies" in *[!0-9:]*|:*|*:) echo "Backup settings must be integers" >&2; exit 1;; esac
+case "$interval:$retention_days:$minimum_copies:$restore_drill_interval" in *[!0-9:]*|:*|*:) echo "Backup settings must be integers" >&2; exit 1;; esac
 mkdir -p "$destination"
 [ -z "$secondary" ] || mkdir -p "$secondary"
 
@@ -24,6 +25,7 @@ while true; do
     (cd "$destination" && sha256sum "$(basename "$target")" > "$(basename "$target").sha256")
     if [ -n "$secondary" ]; then
       cp "$target" "$target.sha256" "$secondary/"
+      (cd "$secondary" && sha256sum -c "$(basename "$target.sha256")")
     fi
     date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-success"
     BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \
@@ -32,6 +34,15 @@ while true; do
       BACKUP_RETENTION_DAYS="$retention_days" BACKUP_MINIMUM_COPIES="$minimum_copies" \
         /opt/mobilityops/prune-postgres-backups.sh "$secondary"
     fi
+    drill_minutes=$((restore_drill_interval / 60))
+    if [ ! -f "$destination/latest-restore-drill" ] \
+        || ! find "$destination/latest-restore-drill" -mmin "-$drill_minutes" -print -quit | grep -q .; then
+      if /opt/mobilityops/restore-drill-postgres.sh "$target"; then
+        date -u +%Y-%m-%dT%H:%M:%SZ > "$destination/latest-restore-drill"
+      else
+        echo "Restore drill failed for $target" >&2
+      fi
+    fi
     echo "Verified database backup: $target"
   else
     rm -f "$temporary"

deploy/unraid/setup-alert-receiver.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+
+container_name="${1:-n8n}"
+source_workflow="${2:-n8n/workflows/fleet-ops-alert-receiver.json}"
+[ -f "$source_workflow" ] || { echo "Missing workflow export: $source_workflow" >&2; exit 1; }
+docker inspect "$container_name" >/dev/null 2>&1 || { echo "n8n container not found" >&2; exit 1; }
+
+container_workflow="/tmp/fleet-ops-alert-receiver.json"
+trap 'docker exec "$container_name" rm -f "$container_workflow" >/dev/null 2>&1 || true' EXIT
+docker cp "$source_workflow" "$container_name:$container_workflow" >/dev/null
+docker exec "$container_name" n8n import:workflow --input="$container_workflow"
+echo "Imported Fleet Ops alert receiver. Resolve the named Header Auth credential and publish the workflow."

deploy/unraid/verify-postgres-backups.sh

@@ -9,4 +9,7 @@ latest="$(find "$destination" -maxdepth 1 -type f -name 'mobilityops-*.dump' | s
 (cd "$destination" && sha256sum -c "$(basename "$latest.sha256")")
 docker compose -p "${COMPOSE_PROJECT_NAME:-mobilityops}" \
   -f compose.yaml -f compose.unraid.yaml exec -T db pg_restore --list < "$latest" > /dev/null
-printf 'Verified: %s\n' "$latest"
+docker compose -p "${COMPOSE_PROJECT_NAME:-mobilityops}" \
+  -f compose.yaml -f compose.unraid.yaml exec -T backup \
+  /opt/mobilityops/restore-drill-postgres.sh "/backups/$(basename "$latest")"
+printf 'Verified and restore-drilled: %s\n' "$latest"