Initial public release

deploy/unraid/deploy-release.sh

@@ -0,0 +1,630 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT"
+
+GEOINTEL_DEPLOY_LOCK_FILE="${GEOINTEL_DEPLOY_LOCK_FILE:-/tmp/geointel-release-deploy.lock}"
+if ! command -v flock >/dev/null 2>&1; then
+  echo "GeoIntel release deployment requires flock to prevent concurrent container replacement." >&2
+  exit 2
+fi
+exec 9>"$GEOINTEL_DEPLOY_LOCK_FILE"
+if ! flock -n 9; then
+  echo "Another GeoIntel release deployment is already running." >&2
+  exit 3
+fi
+
+if [ -f .env ]; then
+  set -a
+  # shellcheck disable=SC1091
+  . ./.env
+  set +a
+fi
+
+if [ -n "${DEPLOY_GEOINTEL_INSTALL_AI:-}" ]; then
+  GEOINTEL_INSTALL_AI="$DEPLOY_GEOINTEL_INSTALL_AI"
+fi
+
+GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"
+if [ "$GEOINTEL_INSTALL_AI" != "true" ]; then
+  echo "Production release deployment requires the gated AI image (GEOINTEL_INSTALL_AI=true)." >&2
+  exit 2
+fi
+GEOINTEL_APP_VERSION="$(tr -d '[:space:]' < VERSION)"
+if ! [[ "$GEOINTEL_APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then
+  echo "Invalid semantic version in VERSION: ${GEOINTEL_APP_VERSION}" >&2
+  exit 2
+fi
+# Hash of everything that actually lands in the image. This is the honest
+# answer to "does this image need rebuilding?" — unlike a git SHA, it changes
+# when working-tree files change without a commit.
+source_tree_hash() {
+  local hash=""
+  command -v sha1sum >/dev/null 2>&1 || return 1
+  hash="$(
+    find backend frontend deploy scripts fixtures VERSION \
+      -type f \
+      ! -path '*/node_modules/*' \
+      ! -path '*/dist/*' \
+      ! -path '*/__pycache__/*' \
+      ! -path '*/.pytest_cache/*' \
+      ! -name '*.pyc' \
+      -print0 2>/dev/null \
+      | sort -z \
+      | xargs -0 sha1sum 2>/dev/null \
+      | sha1sum \
+      | cut -c1-40
+  )" || return 1
+  [ -n "$hash" ] || return 1
+  printf '%s' "$hash"
+}
+
+resolve_build_sha() {
+  local head="" content="" controller_sha="" controller_source=""
+  local git_top="" marker_sha="" marker_path="$ROOT/.gitea-deploy/revision"
+
+  if [ -n "${GITEA_COMMIT_SHA:-}" ]; then
+    controller_sha="$GITEA_COMMIT_SHA"
+    controller_source="GITEA_COMMIT_SHA"
+  fi
+  if [ -n "${GITHUB_SHA:-}" ]; then
+    if ! [[ "$GITHUB_SHA" =~ ^[0-9A-Fa-f]{40}$ ]]; then
+      echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2
+      return 2
+    fi
+    if [ -n "$controller_sha" ] && [ "${controller_sha,,}" != "${GITHUB_SHA,,}" ]; then
+      echo "Controller commit variables disagree." >&2
+      return 2
+    fi
+    controller_sha="$GITHUB_SHA"
+    controller_source="${controller_source:-GITHUB_SHA}"
+  fi
+  if [ -n "$controller_sha" ]; then
+    if ! [[ "$controller_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
+      echo "${controller_source} must contain one full 40-character Git commit SHA." >&2
+      return 2
+    fi
+    controller_sha="${controller_sha,,}"
+    if [ -n "${GEOINTEL_BUILD_SHA:-}" ] && [ "${GEOINTEL_BUILD_SHA,,}" != "$controller_sha" ]; then
+      echo "Explicit build revision differs from the controller revision." >&2
+      return 2
+    fi
+    if [ -f "$marker_path" ]; then
+      marker_sha="$(tr -d '[:space:]' < "$marker_path")"
+      if ! [[ "$marker_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
+        echo "Prepared source revision marker is invalid." >&2
+        return 2
+      fi
+      if [ "${marker_sha,,}" != "$controller_sha" ]; then
+        echo "Prepared source revision marker does not match the controller revision." >&2
+        return 2
+      fi
+    fi
+    if command -v git >/dev/null 2>&1 && git rev-parse --git-dir >/dev/null 2>&1; then
+      git_top="$(git rev-parse --show-toplevel 2>/dev/null || true)"
+    fi
+    if [ -n "$git_top" ] && [ "$(cd "$git_top" && pwd -P)" = "$(pwd -P)" ]; then
+      head="$(git rev-parse HEAD 2>/dev/null || true)"
+      if [ "${head,,}" != "$controller_sha" ]; then
+        echo "Prepared Git checkout does not match the controller revision." >&2
+        return 2
+      fi
+      if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
+        echo "Prepared Git checkout contains changes outside the controller revision." >&2
+        return 2
+      fi
+    elif [ -z "$marker_sha" ]; then
+      echo "Prepared source is neither an exact Git checkout nor bound by a controller revision marker." >&2
+      return 2
+    fi
+    printf '%s' "$controller_sha"
+    return 0
+  fi
+  if [ -n "${GITEA_REPOSITORY:-}" ] || [ -n "${GITHUB_REPOSITORY:-}" ]; then
+    echo "Automated deployment context is missing GITEA_COMMIT_SHA/GITHUB_SHA." >&2
+    return 2
+  fi
+
+  # 1. Explicit override wins.
+  if [ -n "${GEOINTEL_BUILD_SHA:-}" ]; then
+    printf '%s' "$GEOINTEL_BUILD_SHA"
+    return 0
+  fi
+
+  # 2. Git checkout, but only when the working tree matches the commit.
+  #    A manually copied tree often carries .git along while the files on disk
+  #    have moved on. Trusting HEAD there produces an unchanged image tag, and
+  #    the deploy silently reuses the previous image instead of rebuilding.
+  if command -v git >/dev/null 2>&1 && git rev-parse --git-dir >/dev/null 2>&1; then
+    head="$(git rev-parse HEAD 2>/dev/null || true)"
+    if [ -n "$head" ]; then
+      if [ -z "$(git status --porcelain 2>/dev/null)" ]; then
+        printf '%s' "$head"
+        return 0
+      fi
+      echo "Working tree differs from HEAD; tagging this build by content." >&2
+      content="$(source_tree_hash || true)"
+      if [ -n "$content" ]; then
+        printf '%s-wip%s' "${head:0:12}" "${content:0:12}"
+        return 0
+      fi
+    fi
+  fi
+
+  # 3. Manually copied tree with a RELEASE_SHA marker file.
+  if [ -f RELEASE_SHA ]; then
+    tr -d '[:space:]' < RELEASE_SHA
+    return 0
+  fi
+
+  # 4. No git: content hash, so an unchanged redeploy still reuses its image.
+  content="$(source_tree_hash || true)"
+  if [ -n "$content" ]; then
+    printf '%s' "$content"
+    return 0
+  fi
+
+  # 5. Last resort: unique per deploy.
+  printf 'manual%s' "$(date -u +%Y%m%d%H%M%S)"
+}
+
+GEOINTEL_BUILD_SHA="$(resolve_build_sha)"
+if [ -z "$GEOINTEL_BUILD_SHA" ]; then
+  echo "Could not determine a build revision for this deployment." >&2
+  exit 2
+fi
+export GEOINTEL_BUILD_SHA
+echo "Build revision: ${GEOINTEL_BUILD_SHA}"
+GEOINTEL_RELEASE_TOKEN="$(printf '%s' "$GEOINTEL_BUILD_SHA" | tr -c 'A-Za-z0-9._-' '_' | cut -c1-48)"
+if [ -z "$GEOINTEL_RELEASE_TOKEN" ]; then
+  echo "Could not derive a safe release evidence identifier." >&2
+  exit 2
+fi
+GEOINTEL_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+GEOINTEL_IMAGE_REPOSITORY="${GEOINTEL_IMAGE_REPOSITORY:-geointel-all-in-one}"
+if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then
+  GEOINTEL_RELEASE_VARIANT="ai"
+else
+  GEOINTEL_RELEASE_VARIANT="gis"
+fi
+GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"
+FRONTEND_URL="${FRONTEND_URL:-http://127.0.0.1:${GEOINTEL_FRONTEND_PORT:-1202}}"
+GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
+GEOINTEL_STORAGE_PATH="${GEOINTEL_STORAGE_PATH:-/mnt/user/appdata/geointel/storage}"
+GEOINTEL_MODELS_PATH="${GEOINTEL_MODELS_PATH:-/mnt/user/appdata/geointel/models}"
+GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}"
+GEOINTEL_DEPLOY_EVIDENCE_DIR="${GEOINTEL_DEPLOY_EVIDENCE_DIR:-artifacts/release-evidence/deploy/${GEOINTEL_RELEASE_TOKEN}-ai}"
+GEOINTEL_PREDEPLOY_BACKUP_DIR=""
+GEOINTEL_RELEASE_IMAGE_ID=""
+GEOINTEL_BACKUP_LINK_DEST=""
+GEOINTEL_PREDEPLOY_ROLLBACK_TAG=""
+
+case "$GEOINTEL_DEPLOY_EVIDENCE_DIR" in
+  /*|*..*)
+    echo "Deployment evidence directory must be repository-relative and must not contain '..'." >&2
+    exit 2
+    ;;
+esac
+
+wait_for_geointel_health() {
+  local status=""
+  for attempt in $(seq 1 480); do
+    status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' geointel 2>/dev/null || true)"
+    if [ "$status" = "healthy" ]; then
+      echo "GeoIntel container is healthy after attempt ${attempt}."
+      return 0
+    fi
+    if [ "$status" = "unhealthy" ] || [ "$status" = "exited" ] || [ "$status" = "dead" ]; then
+      echo "GeoIntel container entered terminal state: ${status}" >&2
+      docker logs --tail 120 geointel >&2 || true
+      return 1
+    fi
+    sleep 2
+  done
+  echo "GeoIntel container did not become healthy (last state: ${status:-missing})." >&2
+  docker logs --tail 120 geointel >&2 || true
+  return 1
+}
+
+start_image() {
+  local image="$1"
+  local running_image_id=""
+  local running_revision=""
+  local running_ai=""
+  GEOINTEL_IMAGE="$image" bash deploy/unraid/run-dockerman-container.sh
+  wait_for_geointel_health
+  running_image_id="$(docker inspect --format '{{.Image}}' geointel)"
+  if [ "$running_image_id" != "$image" ]; then
+    echo "Running container image ${running_image_id} differs from attested image ${image}." >&2
+    return 1
+  fi
+  running_revision="$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' geointel)"
+  running_ai="$(docker inspect --format '{{index .Config.Labels "io.geointel.ai.enabled"}}' geointel)"
+  if [ "$running_revision" != "$GEOINTEL_BUILD_SHA" ] || [ "$running_ai" != "true" ]; then
+    echo "Running container labels do not match the attested AI revision." >&2
+    return 1
+  fi
+  echo "Running container matches attested image: ${running_image_id}"
+}
+
+scan_release_image() {
+  local scanned_image_id=""
+  local current_image_id=""
+  local inspect_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/image-inspect.json"
+  local sbom_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-sbom.spdx.json"
+  local vulnerability_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-container-vulnerabilities.json"
+  local attestation_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/deployment-attestation.json"
+
+  scanned_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
+  test -n "$scanned_image_id"
+  mkdir -p "$ROOT/$GEOINTEL_DEPLOY_EVIDENCE_DIR"
+  docker image inspect "$GEOINTEL_RELEASE_IMAGE" > "$ROOT/$inspect_output"
+  (
+    export GEOINTEL_IMAGE_ARCHIVE="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-image.tar"
+    export GEOINTEL_KEEP_IMAGE_ARCHIVE=true
+    export SYFT_PARALLELISM=1
+    trap 'rm -f -- \
+      "$ROOT/$GEOINTEL_IMAGE_ARCHIVE" \
+      "$ROOT/$GEOINTEL_IMAGE_ARCHIVE.image-id" \
+      "$ROOT/$GEOINTEL_IMAGE_ARCHIVE".partial.*' EXIT
+    bash scripts/generate_container_sbom.sh "$scanned_image_id" "$sbom_output"
+    bash scripts/scan_container_image.sh "$scanned_image_id" "$vulnerability_output"
+  )
+  current_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
+  if [ "$current_image_id" != "$scanned_image_id" ]; then
+    echo "Release image tag changed while SBOM/scan evidence was being generated." >&2
+    return 1
+  fi
+  test -s "$ROOT/$inspect_output"
+  test -s "$ROOT/$sbom_output"
+  test -s "$ROOT/$vulnerability_output"
+  GEOINTEL_RELEASE_IMAGE_ID="$scanned_image_id"
+  python3 - \
+    "$ROOT/$attestation_output" \
+    "$GEOINTEL_RELEASE_IMAGE" \
+    "$GEOINTEL_RELEASE_IMAGE_ID" \
+    "$GEOINTEL_BUILD_SHA" \
+    "$inspect_output" \
+    "$sbom_output" \
+    "$vulnerability_output" <<'PY'
+import datetime
+import json
+import pathlib
+import sys
+
+output, image_tag, image_id, revision, inspect_path, sbom_path, vulnerability_path = sys.argv[1:]
+payload = {
+    "schema_version": 1,
+    "attested_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+    "image_tag": image_tag,
+    "image_id": image_id,
+    "image_config_digest": image_id,
+    "revision": revision,
+    "variant": "ai",
+    "evidence": {
+        "image_inspect": inspect_path,
+        "sbom": sbom_path,
+        "vulnerabilities": vulnerability_path,
+    },
+}
+path = pathlib.Path(output)
+temporary = path.with_suffix(".json.partial")
+temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+temporary.replace(path)
+PY
+  test -s "$ROOT/$attestation_output"
+  echo "Exact deployment image scanned: ${GEOINTEL_RELEASE_IMAGE_ID}"
+}
+
+preflight_backup_capacity() {
+  local database_name=""
+  local database_user=""
+  local database_size_bytes=""
+
+  database_name="$(docker exec geointel sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"')"
+  database_user="$(docker exec geointel sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"')"
+  database_size_bytes="$(docker exec geointel psql -X -v ON_ERROR_STOP=1 \
+    -U "$database_user" -d "$database_name" -Atqc \
+    'SELECT pg_database_size(current_database());')"
+  mkdir -p "$GEOINTEL_BACKUPS_PATH"
+  python3 - \
+    "$GEOINTEL_BACKUPS_PATH" \
+    "$GEOINTEL_STORAGE_PATH" \
+    "$GEOINTEL_MODELS_PATH" \
+    "$database_size_bytes" <<'PY'
+import os
+import pathlib
+import shutil
+import stat
+import sys
+
+backup_root = pathlib.Path(sys.argv[1]).expanduser().resolve()
+sources = [pathlib.Path(value).expanduser().resolve() for value in sys.argv[2:4]]
+database_bytes = int(sys.argv[4])
+
+def retained_bytes(root: pathlib.Path) -> int:
+    if not root.is_dir():
+        raise SystemExit(f"Mandatory snapshot source is not a directory: {root}")
+    total = 0
+    for current, directories, files in os.walk(root, topdown=True, followlinks=False):
+        current_path = pathlib.Path(current)
+        for name in [*directories, *files]:
+            path = current_path / name
+            details = path.lstat()
+            if stat.S_ISLNK(details.st_mode):
+                raise SystemExit(f"Mandatory snapshot refuses symlinked content: {path}")
+            if name in directories and not stat.S_ISDIR(details.st_mode):
+                raise SystemExit(f"Snapshot directory changed during capacity preflight: {path}")
+            if name in files:
+                if not stat.S_ISREG(details.st_mode):
+                    raise SystemExit(f"Mandatory snapshot refuses non-regular content: {path}")
+                total += details.st_size
+    return total
+
+source_bytes = sum(retained_bytes(source) for source in sources)
+# Reflink clones are used when the backing filesystem supports them. Budget for
+# a complete copy plus two uncompressed database sizes (dump and isolated
+# restore/cutover recovery) so fallback still fails before the live backend is
+# quiesced rather than midway through the snapshot.
+required = source_bytes + (2 * database_bytes)
+headroom = max(5 * 1024**3, required // 10)
+free = shutil.disk_usage(backup_root).free
+if free < required + headroom:
+    raise SystemExit(
+        "Insufficient free space for a fail-safe predeploy snapshot: "
+        f"required={required + headroom} free={free} source={source_bytes} database={database_bytes}"
+    )
+print(
+    "Predeploy snapshot capacity: "
+    f"source_bytes={source_bytes} database_bytes={database_bytes} free_bytes={free}"
+)
+PY
+}
+
+run_low_impact() {
+  local priority_command=()
+
+  # Backups are mandatory, but their first byte-complete copy and SHA-256
+  # verification must not starve the live Unraid services. BusyBox hosts do
+  # not always provide both tools, so use every available scheduler without
+  # weakening the backup when one is absent.
+  if command -v ionice >/dev/null 2>&1; then
+    priority_command+=(ionice -c 2 -n 7)
+  fi
+  if command -v nice >/dev/null 2>&1; then
+    priority_command+=(nice -n 10)
+  fi
+  if [ "${#priority_command[@]}" -eq 0 ]; then
+    "$@"
+    return
+  fi
+  "${priority_command[@]}" "$@"
+}
+
+select_verified_link_dest() {
+  local candidate=""
+  GEOINTEL_BACKUP_LINK_DEST=""
+  while IFS= read -r candidate; do
+    if (
+      cd "$candidate" \
+        && run_low_impact sha256sum -c CHECKSUMS.sha256 >/dev/null \
+        && run_low_impact python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate"
+    ); then
+      GEOINTEL_BACKUP_LINK_DEST="$candidate"
+      echo "Using verified prior byte snapshot as link-dest: ${candidate}"
+      return 0
+    fi
+    echo "Skipping unusable prior backup link-dest: ${candidate}" >&2
+  done < <(
+    python3 - "$GEOINTEL_BACKUPS_PATH" <<'PY'
+import pathlib
+import sys
+
+root = pathlib.Path(sys.argv[1]).expanduser().resolve()
+candidates = sorted(
+    (
+        path
+        for path in root.iterdir()
+        if path.is_dir() and not path.name.startswith(".") and (path / "manifest.json").is_file()
+    ),
+    key=lambda path: path.stat().st_mtime_ns,
+    reverse=True,
+)
+for candidate in candidates:
+    print(candidate)
+PY
+  )
+  echo "No verified prior byte snapshot found; this deployment will create a full first snapshot."
+}
+
+create_predeploy_backup() {
+  local container_exists="false"
+  local container_running="false"
+  local release_id=""
+  local backup_link_args=()
+  local current_image_id=""
+  local existing_rollback_id=""
+
+  if docker ps -a --format '{{.Names}}' | grep -Fxq geointel; then
+    container_exists="true"
+  fi
+  if [ "$(docker inspect -f '{{.State.Running}}' geointel 2>/dev/null || true)" = "true" ]; then
+    container_running="true"
+  fi
+
+  if [ "$container_exists" = "false" ]; then
+    if [ -f "$GEOINTEL_POSTGIS_DATA_PATH/PG_VERSION" ]; then
+      echo "PostGIS data exists without a running GeoIntel container; refusing an unbacked migration." >&2
+      return 1
+    fi
+    echo "No existing GeoIntel state found; pre-deploy backup is not required for this fresh install."
+    return 0
+  fi
+  if [ "$container_running" != "true" ]; then
+    echo "Existing GeoIntel container is not running; refusing deployment because a consistent backup cannot be created." >&2
+    return 1
+  fi
+
+  current_image_id="$(docker inspect --format '{{.Image}}' geointel)"
+  if ! [[ "$current_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+    echo "Running release does not expose one immutable Docker image ID." >&2
+    return 1
+  fi
+  release_id="predeploy-${GEOINTEL_RELEASE_TOKEN:0:24}-$(date -u +%Y%m%dT%H%M%SZ)-$$"
+  GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"
+  existing_rollback_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" 2>/dev/null || true)"
+  if [ -n "$existing_rollback_id" ] && [ "$existing_rollback_id" != "$current_image_id" ]; then
+    echo "Backup-specific rollback tag already identifies different image bytes." >&2
+    return 1
+  fi
+  docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"
+
+  # This conservative full-copy fallback estimate runs while the existing
+  # release is still healthy. Verified backup-to-backup hardlinks normally
+  # avoid recopying unchanged bytes, but are never assumed for this fail-closed
+  # capacity decision.
+  preflight_backup_capacity
+  select_verified_link_dest
+  if [ -n "$GEOINTEL_BACKUP_LINK_DEST" ]; then
+    backup_link_args=(--link-dest-backup "$GEOINTEL_BACKUP_LINK_DEST")
+  fi
+
+  echo "Quiescing the current backend so the rollback point cannot miss concurrent writes..."
+  if ! docker exec -i geointel python - <<'PY'
+import os
+import pathlib
+import signal
+import time
+
+matches = []
+for item in pathlib.Path("/proc").iterdir():
+    if not item.name.isdigit() or int(item.name) in {os.getpid(), os.getppid()}:
+        continue
+    try:
+        command = (item / "cmdline").read_bytes().replace(b"\0", b" ")
+    except (OSError, PermissionError):
+        continue
+    if b"uvicorn" in command and b"app.main:app" in command:
+        matches.append(int(item.name))
+if not matches:
+    raise SystemExit("Could not identify the running GeoIntel backend")
+for process_id in matches:
+    os.kill(process_id, signal.SIGTERM)
+deadline = time.monotonic() + 60
+remaining = matches
+while remaining and time.monotonic() < deadline:
+    time.sleep(0.25)
+    remaining = [process_id for process_id in remaining if pathlib.Path(f"/proc/{process_id}").exists()]
+if remaining:
+    raise SystemExit(f"Backend did not stop cleanly: {remaining}")
+print(f"Stopped {len(matches)} backend process(es)")
+PY
+  then
+    echo "Could not quiesce the current backend; refusing a potentially inconsistent backup." >&2
+    docker restart geointel >/dev/null || true
+    wait_for_geointel_health || true
+    return 1
+  fi
+
+  GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}"
+  echo "Creating mandatory pre-deploy backup ${release_id}..."
+  if ! run_low_impact bash scripts/backup_release_state.sh \
+      --container geointel \
+      --output-root "$GEOINTEL_BACKUPS_PATH" \
+      --release-id "$release_id" \
+      --storage-path "$GEOINTEL_STORAGE_PATH" \
+      --models-path "$GEOINTEL_MODELS_PATH" \
+      --inventory-mode sha256 \
+      --rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \
+      "${backup_link_args[@]}" \
+    || ! run_low_impact bash scripts/verify_release_backup.sh \
+      --container geointel \
+      --backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then
+    echo "Pre-deploy backup failed; restarting the unchanged current release." >&2
+    docker restart geointel >/dev/null || true
+    wait_for_geointel_health || true
+    GEOINTEL_PREDEPLOY_BACKUP_DIR=""
+    return 1
+  fi
+  echo "Pre-deploy backup verified: ${GEOINTEL_PREDEPLOY_BACKUP_DIR}"
+}
+
+rollback_previous() {
+  if [ -z "$GEOINTEL_PREDEPLOY_BACKUP_DIR" ]; then
+    echo "Automatic rollback unavailable: no verified pre-deploy database backup was created." >&2
+    return 1
+  fi
+  echo "Rolling back database and image to the verified pre-deploy state..."
+  GEOINTEL_DEPLOY_LOCK_HELD=true \
+    bash deploy/unraid/rollback-dockerman-container.sh \
+      --backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR" \
+      --confirm-production-database-restore
+}
+
+docker compose -f docker-compose.unraid.yml config >/dev/null
+
+if docker image inspect "$GEOINTEL_RELEASE_IMAGE" >/dev/null 2>&1; then
+  stored_revision="$(
+    docker image inspect \
+      --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \
+      "$GEOINTEL_RELEASE_IMAGE"
+  )"
+  stored_ai="$(
+    docker image inspect \
+      --format '{{index .Config.Labels "io.geointel.ai.enabled"}}' \
+      "$GEOINTEL_RELEASE_IMAGE"
+  )"
+  stored_version="$(
+    docker image inspect \
+      --format '{{index .Config.Labels "org.opencontainers.image.version"}}' \
+      "$GEOINTEL_RELEASE_IMAGE"
+  )"
+  if (
+    [ "$stored_revision" != "$GEOINTEL_BUILD_SHA" ] ||
+    [ "$stored_ai" != "$GEOINTEL_INSTALL_AI" ] ||
+    [ "$stored_version" != "$GEOINTEL_APP_VERSION" ]
+  ); then
+    echo "Immutable release tag has conflicting metadata: ${GEOINTEL_RELEASE_IMAGE}" >&2
+    exit 2
+  fi
+  echo "Reusing existing immutable image ${GEOINTEL_RELEASE_IMAGE}."
+  docker tag "$GEOINTEL_RELEASE_IMAGE" "${GEOINTEL_IMAGE_REPOSITORY}:latest"
+else
+  docker build \
+    --build-arg GEOINTEL_INSTALL_AI="$GEOINTEL_INSTALL_AI" \
+    --build-arg GEOINTEL_BUILD_SHA="$GEOINTEL_BUILD_SHA" \
+    --build-arg GEOINTEL_BUILD_TIME="$GEOINTEL_BUILD_TIME" \
+    --build-arg GEOINTEL_APP_VERSION="$GEOINTEL_APP_VERSION" \
+    -f deploy/unraid/Dockerfile.all-in-one \
+    -t "$GEOINTEL_RELEASE_IMAGE" \
+    -t "${GEOINTEL_IMAGE_REPOSITORY}:latest" \
+    .
+fi
+
+scan_release_image
+create_predeploy_backup
+
+if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"; then
+  rollback_previous || true
+  exit 1
+fi
+
+if [ -f scripts/live_migration_smoke.sh ]; then
+  if ! LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh; then
+    rollback_previous || true
+    exit 1
+  fi
+fi
+
+if [ -f scripts/verify_browser_runtime.sh ]; then
+  if ! bash scripts/verify_browser_runtime.sh "$FRONTEND_URL"; then
+    rollback_previous || true
+    exit 1
+  fi
+fi
+
+echo "Deployed immutable image ${GEOINTEL_RELEASE_IMAGE}."
+docker image inspect \
+  --format 'version={{index .Config.Labels "org.opencontainers.image.version"}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} created={{index .Config.Labels "org.opencontainers.image.created"}}' \
+  "$GEOINTEL_RELEASE_IMAGE"