geointel/deploy/unraid/deploy-release.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$ROOT"

GEOINTEL_DEPLOY_LOCK_FILE="${GEOINTEL_DEPLOY_LOCK_FILE:-/tmp/geointel-release-deploy.lock}"
if ! command -v flock >/dev/null 2>&1; then
  echo "GeoIntel release deployment requires flock to prevent concurrent container replacement." >&2
  exit 2
fi
exec 9>"$GEOINTEL_DEPLOY_LOCK_FILE"
if ! flock -n 9; then
  echo "Another GeoIntel release deployment is already running." >&2
  exit 3
fi

if [ -f .env ]; then
  set -a
  # shellcheck disable=SC1091
  . ./.env
  set +a
fi

if [ -n "${DEPLOY_GEOINTEL_INSTALL_AI:-}" ]; then
  GEOINTEL_INSTALL_AI="$DEPLOY_GEOINTEL_INSTALL_AI"
fi

GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"
if [ "$GEOINTEL_INSTALL_AI" != "true" ]; then
  echo "Production release deployment requires the gated AI image (GEOINTEL_INSTALL_AI=true)." >&2
  exit 2
fi
GEOINTEL_APP_VERSION="$(tr -d '[:space:]' < VERSION)"
if ! [[ "$GEOINTEL_APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then
  echo "Invalid semantic version in VERSION: ${GEOINTEL_APP_VERSION}" >&2
  exit 2
fi
# Hash of everything that actually lands in the image. This is the honest
# answer to "does this image need rebuilding?" — unlike a git SHA, it changes
# when working-tree files change without a commit.
source_tree_hash() {
  local hash=""
  command -v sha1sum >/dev/null 2>&1 || return 1
  hash="$(
    find backend frontend deploy scripts fixtures VERSION \
      -type f \
      ! -path '*/node_modules/*' \
      ! -path '*/dist/*' \
      ! -path '*/__pycache__/*' \
      ! -path '*/.pytest_cache/*' \
      ! -name '*.pyc' \
      -print0 2>/dev/null \
      | sort -z \
      | xargs -0 sha1sum 2>/dev/null \
      | sha1sum \
      | cut -c1-40
  )" || return 1
  [ -n "$hash" ] || return 1
  printf '%s' "$hash"
}

resolve_build_sha() {
  local head="" content="" controller_sha="" controller_source=""
  local git_top="" marker_sha="" marker_path="$ROOT/.gitea-deploy/revision"

  if [ -n "${GITEA_COMMIT_SHA:-}" ]; then
    controller_sha="$GITEA_COMMIT_SHA"
    controller_source="GITEA_COMMIT_SHA"
  fi
  if [ -n "${GITHUB_SHA:-}" ]; then
    if ! [[ "$GITHUB_SHA" =~ ^[0-9A-Fa-f]{40}$ ]]; then
      echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2
      return 2
    fi
    if [ -n "$controller_sha" ] && [ "${controller_sha,,}" != "${GITHUB_SHA,,}" ]; then
      echo "Controller commit variables disagree." >&2
      return 2
    fi
    controller_sha="$GITHUB_SHA"
    controller_source="${controller_source:-GITHUB_SHA}"
  fi
  if [ -n "$controller_sha" ]; then
    if ! [[ "$controller_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
      echo "${controller_source} must contain one full 40-character Git commit SHA." >&2
      return 2
    fi
    controller_sha="${controller_sha,,}"
    if [ -n "${GEOINTEL_BUILD_SHA:-}" ] && [ "${GEOINTEL_BUILD_SHA,,}" != "$controller_sha" ]; then
      echo "Explicit build revision differs from the controller revision." >&2
      return 2
    fi
    if [ -f "$marker_path" ]; then
      marker_sha="$(tr -d '[:space:]' < "$marker_path")"
      if ! [[ "$marker_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
        echo "Prepared source revision marker is invalid." >&2
        return 2
      fi
      if [ "${marker_sha,,}" != "$controller_sha" ]; then
        echo "Prepared source revision marker does not match the controller revision." >&2
        return 2
      fi
    fi
    if command -v git >/dev/null 2>&1 && git rev-parse --git-dir >/dev/null 2>&1; then
      git_top="$(git rev-parse --show-toplevel 2>/dev/null || true)"
    fi
    if [ -n "$git_top" ] && [ "$(cd "$git_top" && pwd -P)" = "$(pwd -P)" ]; then
      head="$(git rev-parse HEAD 2>/dev/null || true)"
      if [ "${head,,}" != "$controller_sha" ]; then
        echo "Prepared Git checkout does not match the controller revision." >&2
        return 2
      fi
      if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
        echo "Prepared Git checkout contains changes outside the controller revision." >&2
        return 2
      fi
    elif [ -z "$marker_sha" ]; then
      echo "Prepared source is neither an exact Git checkout nor bound by a controller revision marker." >&2
      return 2
    fi
    printf '%s' "$controller_sha"
    return 0
  fi
  if [ -n "${GITEA_REPOSITORY:-}" ] || [ -n "${GITHUB_REPOSITORY:-}" ]; then
    echo "Automated deployment context is missing GITEA_COMMIT_SHA/GITHUB_SHA." >&2
    return 2
  fi

  # 1. Explicit override wins.
  if [ -n "${GEOINTEL_BUILD_SHA:-}" ]; then
    printf '%s' "$GEOINTEL_BUILD_SHA"
    return 0
  fi

  # 2. Git checkout, but only when the working tree matches the commit.
  #    A manually copied tree often carries .git along while the files on disk
  #    have moved on. Trusting HEAD there produces an unchanged image tag, and
  #    the deploy silently reuses the previous image instead of rebuilding.
  if command -v git >/dev/null 2>&1 && git rev-parse --git-dir >/dev/null 2>&1; then
    head="$(git rev-parse HEAD 2>/dev/null || true)"
    if [ -n "$head" ]; then
      if [ -z "$(git status --porcelain 2>/dev/null)" ]; then
        printf '%s' "$head"
        return 0
      fi
      echo "Working tree differs from HEAD; tagging this build by content." >&2
      content="$(source_tree_hash || true)"
      if [ -n "$content" ]; then
        printf '%s-wip%s' "${head:0:12}" "${content:0:12}"
        return 0
      fi
    fi
  fi

  # 3. Manually copied tree with a RELEASE_SHA marker file.
  if [ -f RELEASE_SHA ]; then
    tr -d '[:space:]' < RELEASE_SHA
    return 0
  fi

  # 4. No git: content hash, so an unchanged redeploy still reuses its image.
  content="$(source_tree_hash || true)"
  if [ -n "$content" ]; then
    printf '%s' "$content"
    return 0
  fi

  # 5. Last resort: unique per deploy.
  printf 'manual%s' "$(date -u +%Y%m%d%H%M%S)"
}

GEOINTEL_BUILD_SHA="$(resolve_build_sha)"
if [ -z "$GEOINTEL_BUILD_SHA" ]; then
  echo "Could not determine a build revision for this deployment." >&2
  exit 2
fi
export GEOINTEL_BUILD_SHA
echo "Build revision: ${GEOINTEL_BUILD_SHA}"
GEOINTEL_RELEASE_TOKEN="$(printf '%s' "$GEOINTEL_BUILD_SHA" | tr -c 'A-Za-z0-9._-' '_' | cut -c1-48)"
if [ -z "$GEOINTEL_RELEASE_TOKEN" ]; then
  echo "Could not derive a safe release evidence identifier." >&2
  exit 2
fi
GEOINTEL_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
GEOINTEL_IMAGE_REPOSITORY="${GEOINTEL_IMAGE_REPOSITORY:-geointel-all-in-one}"
if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then
  GEOINTEL_RELEASE_VARIANT="ai"
else
  GEOINTEL_RELEASE_VARIANT="gis"
fi
GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"
FRONTEND_URL="${FRONTEND_URL:-http://127.0.0.1:${GEOINTEL_FRONTEND_PORT:-1202}}"
GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
GEOINTEL_STORAGE_PATH="${GEOINTEL_STORAGE_PATH:-/mnt/user/appdata/geointel/storage}"
GEOINTEL_MODELS_PATH="${GEOINTEL_MODELS_PATH:-/mnt/user/appdata/geointel/models}"
GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}"
GEOINTEL_DEPLOY_EVIDENCE_DIR="${GEOINTEL_DEPLOY_EVIDENCE_DIR:-artifacts/release-evidence/deploy/${GEOINTEL_RELEASE_TOKEN}-ai}"
GEOINTEL_PREDEPLOY_BACKUP_DIR=""
GEOINTEL_RELEASE_IMAGE_ID=""
GEOINTEL_BACKUP_LINK_DEST=""
GEOINTEL_PREDEPLOY_ROLLBACK_TAG=""

case "$GEOINTEL_DEPLOY_EVIDENCE_DIR" in
  /*|*..*)
    echo "Deployment evidence directory must be repository-relative and must not contain '..'." >&2
    exit 2
    ;;
esac

wait_for_geointel_health() {
  local status=""
  for attempt in $(seq 1 480); do
    status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' geointel 2>/dev/null || true)"
    if [ "$status" = "healthy" ]; then
      echo "GeoIntel container is healthy after attempt ${attempt}."
      return 0
    fi
    if [ "$status" = "unhealthy" ] || [ "$status" = "exited" ] || [ "$status" = "dead" ]; then
      echo "GeoIntel container entered terminal state: ${status}" >&2
      docker logs --tail 120 geointel >&2 || true
      return 1
    fi
    sleep 2
  done
  echo "GeoIntel container did not become healthy (last state: ${status:-missing})." >&2
  docker logs --tail 120 geointel >&2 || true
  return 1
}

start_image() {
  local image="$1"
  local running_image_id=""
  local running_revision=""
  local running_ai=""
  GEOINTEL_IMAGE="$image" bash deploy/unraid/run-dockerman-container.sh
  wait_for_geointel_health
  running_image_id="$(docker inspect --format '{{.Image}}' geointel)"
  if [ "$running_image_id" != "$image" ]; then
    echo "Running container image ${running_image_id} differs from attested image ${image}." >&2
    return 1
  fi
  running_revision="$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' geointel)"
  running_ai="$(docker inspect --format '{{index .Config.Labels "io.geointel.ai.enabled"}}' geointel)"
  if [ "$running_revision" != "$GEOINTEL_BUILD_SHA" ] || [ "$running_ai" != "true" ]; then
    echo "Running container labels do not match the attested AI revision." >&2
    return 1
  fi
  echo "Running container matches attested image: ${running_image_id}"
}

scan_release_image() {
  local scanned_image_id=""
  local current_image_id=""
  local inspect_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/image-inspect.json"
  local sbom_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-sbom.spdx.json"
  local vulnerability_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-container-vulnerabilities.json"
  local attestation_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/deployment-attestation.json"

  scanned_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
  test -n "$scanned_image_id"
  mkdir -p "$ROOT/$GEOINTEL_DEPLOY_EVIDENCE_DIR"
  docker image inspect "$GEOINTEL_RELEASE_IMAGE" > "$ROOT/$inspect_output"
  (
    export GEOINTEL_IMAGE_ARCHIVE="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-image.tar"
    export GEOINTEL_KEEP_IMAGE_ARCHIVE=true
    export SYFT_PARALLELISM=1
    trap 'rm -f -- \
      "$ROOT/$GEOINTEL_IMAGE_ARCHIVE" \
      "$ROOT/$GEOINTEL_IMAGE_ARCHIVE.image-id" \
      "$ROOT/$GEOINTEL_IMAGE_ARCHIVE".partial.*' EXIT
    bash scripts/generate_container_sbom.sh "$scanned_image_id" "$sbom_output"
    bash scripts/scan_container_image.sh "$scanned_image_id" "$vulnerability_output"
  )
  current_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
  if [ "$current_image_id" != "$scanned_image_id" ]; then
    echo "Release image tag changed while SBOM/scan evidence was being generated." >&2
    return 1
  fi
  test -s "$ROOT/$inspect_output"
  test -s "$ROOT/$sbom_output"
  test -s "$ROOT/$vulnerability_output"
  GEOINTEL_RELEASE_IMAGE_ID="$scanned_image_id"
  python3 - \
    "$ROOT/$attestation_output" \
    "$GEOINTEL_RELEASE_IMAGE" \
    "$GEOINTEL_RELEASE_IMAGE_ID" \
    "$GEOINTEL_BUILD_SHA" \
    "$inspect_output" \
    "$sbom_output" \
    "$vulnerability_output" <<'PY'
import datetime
import json
import pathlib
import sys

output, image_tag, image_id, revision, inspect_path, sbom_path, vulnerability_path = sys.argv[1:]
payload = {
    "schema_version": 1,
    "attested_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
    "image_tag": image_tag,
    "image_id": image_id,
    "image_config_digest": image_id,
    "revision": revision,
    "variant": "ai",
    "evidence": {
        "image_inspect": inspect_path,
        "sbom": sbom_path,
        "vulnerabilities": vulnerability_path,
    },
}
path = pathlib.Path(output)
temporary = path.with_suffix(".json.partial")
temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
temporary.replace(path)
PY
  test -s "$ROOT/$attestation_output"
  echo "Exact deployment image scanned: ${GEOINTEL_RELEASE_IMAGE_ID}"
}

preflight_backup_capacity() {
  local database_name=""
  local database_user=""
  local database_size_bytes=""

  database_name="$(docker exec geointel sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"')"
  database_user="$(docker exec geointel sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"')"
  database_size_bytes="$(docker exec geointel psql -X -v ON_ERROR_STOP=1 \
    -U "$database_user" -d "$database_name" -Atqc \
    'SELECT pg_database_size(current_database());')"
  mkdir -p "$GEOINTEL_BACKUPS_PATH"
  python3 - \
    "$GEOINTEL_BACKUPS_PATH" \
    "$GEOINTEL_STORAGE_PATH" \
    "$GEOINTEL_MODELS_PATH" \
    "$database_size_bytes" <<'PY'
import os
import pathlib
import shutil
import stat
import sys

backup_root = pathlib.Path(sys.argv[1]).expanduser().resolve()
sources = [pathlib.Path(value).expanduser().resolve() for value in sys.argv[2:4]]
database_bytes = int(sys.argv[4])

def retained_bytes(root: pathlib.Path) -> int:
    if not root.is_dir():
        raise SystemExit(f"Mandatory snapshot source is not a directory: {root}")
    total = 0
    for current, directories, files in os.walk(root, topdown=True, followlinks=False):
        current_path = pathlib.Path(current)
        for name in [*directories, *files]:
            path = current_path / name
            details = path.lstat()
            if stat.S_ISLNK(details.st_mode):
                raise SystemExit(f"Mandatory snapshot refuses symlinked content: {path}")
            if name in directories and not stat.S_ISDIR(details.st_mode):
                raise SystemExit(f"Snapshot directory changed during capacity preflight: {path}")
            if name in files:
                if not stat.S_ISREG(details.st_mode):
                    raise SystemExit(f"Mandatory snapshot refuses non-regular content: {path}")
                total += details.st_size
    return total

source_bytes = sum(retained_bytes(source) for source in sources)
# Reflink clones are used when the backing filesystem supports them. Budget for
# a complete copy plus two uncompressed database sizes (dump and isolated
# restore/cutover recovery) so fallback still fails before the live backend is
# quiesced rather than midway through the snapshot.
required = source_bytes + (2 * database_bytes)
headroom = max(5 * 1024**3, required // 10)
free = shutil.disk_usage(backup_root).free
if free < required + headroom:
    raise SystemExit(
        "Insufficient free space for a fail-safe predeploy snapshot: "
        f"required={required + headroom} free={free} source={source_bytes} database={database_bytes}"
    )
print(
    "Predeploy snapshot capacity: "
    f"source_bytes={source_bytes} database_bytes={database_bytes} free_bytes={free}"
)
PY
}

run_low_impact() {
  local priority_command=()

  # Backups are mandatory, but their first byte-complete copy and SHA-256
  # verification must not starve the live Unraid services. BusyBox hosts do
  # not always provide both tools, so use every available scheduler without
  # weakening the backup when one is absent.
  if command -v ionice >/dev/null 2>&1; then
    priority_command+=(ionice -c 2 -n 7)
  fi
  if command -v nice >/dev/null 2>&1; then
    priority_command+=(nice -n 10)
  fi
  if [ "${#priority_command[@]}" -eq 0 ]; then
    "$@"
    return
  fi
  "${priority_command[@]}" "$@"
}

select_verified_link_dest() {
  local candidate=""
  GEOINTEL_BACKUP_LINK_DEST=""
  while IFS= read -r candidate; do
    if (
      cd "$candidate" \
        && run_low_impact sha256sum -c CHECKSUMS.sha256 >/dev/null \
        && run_low_impact python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate"
    ); then
      GEOINTEL_BACKUP_LINK_DEST="$candidate"
      echo "Using verified prior byte snapshot as link-dest: ${candidate}"
      return 0
    fi
    echo "Skipping unusable prior backup link-dest: ${candidate}" >&2
  done < <(
    python3 - "$GEOINTEL_BACKUPS_PATH" <<'PY'
import pathlib
import sys

root = pathlib.Path(sys.argv[1]).expanduser().resolve()
candidates = sorted(
    (
        path
        for path in root.iterdir()
        if path.is_dir() and not path.name.startswith(".") and (path / "manifest.json").is_file()
    ),
    key=lambda path: path.stat().st_mtime_ns,
    reverse=True,
)
for candidate in candidates:
    print(candidate)
PY
  )
  echo "No verified prior byte snapshot found; this deployment will create a full first snapshot."
}

create_predeploy_backup() {
  local container_exists="false"
  local container_running="false"
  local release_id=""
  local backup_link_args=()
  local current_image_id=""
  local existing_rollback_id=""

  if docker ps -a --format '{{.Names}}' | grep -Fxq geointel; then
    container_exists="true"
  fi
  if [ "$(docker inspect -f '{{.State.Running}}' geointel 2>/dev/null || true)" = "true" ]; then
    container_running="true"
  fi

  if [ "$container_exists" = "false" ]; then
    if [ -f "$GEOINTEL_POSTGIS_DATA_PATH/PG_VERSION" ]; then
      echo "PostGIS data exists without a running GeoIntel container; refusing an unbacked migration." >&2
      return 1
    fi
    echo "No existing GeoIntel state found; pre-deploy backup is not required for this fresh install."
    return 0
  fi
  if [ "$container_running" != "true" ]; then
    echo "Existing GeoIntel container is not running; refusing deployment because a consistent backup cannot be created." >&2
    return 1
  fi

  current_image_id="$(docker inspect --format '{{.Image}}' geointel)"
  if ! [[ "$current_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
    echo "Running release does not expose one immutable Docker image ID." >&2
    return 1
  fi
  release_id="predeploy-${GEOINTEL_RELEASE_TOKEN:0:24}-$(date -u +%Y%m%dT%H%M%SZ)-$$"
  GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"
  existing_rollback_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" 2>/dev/null || true)"
  if [ -n "$existing_rollback_id" ] && [ "$existing_rollback_id" != "$current_image_id" ]; then
    echo "Backup-specific rollback tag already identifies different image bytes." >&2
    return 1
  fi
  docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"

  # This conservative full-copy fallback estimate runs while the existing
  # release is still healthy. Verified backup-to-backup hardlinks normally
  # avoid recopying unchanged bytes, but are never assumed for this fail-closed
  # capacity decision.
  preflight_backup_capacity
  select_verified_link_dest
  if [ -n "$GEOINTEL_BACKUP_LINK_DEST" ]; then
    backup_link_args=(--link-dest-backup "$GEOINTEL_BACKUP_LINK_DEST")
  fi

  echo "Quiescing the current backend so the rollback point cannot miss concurrent writes..."
  if ! docker exec -i geointel python - <<'PY'
import os
import pathlib
import signal
import time

matches = []
for item in pathlib.Path("/proc").iterdir():
    if not item.name.isdigit() or int(item.name) in {os.getpid(), os.getppid()}:
        continue
    try:
        command = (item / "cmdline").read_bytes().replace(b"\0", b" ")
    except (OSError, PermissionError):
        continue
    if b"uvicorn" in command and b"app.main:app" in command:
        matches.append(int(item.name))
if not matches:
    raise SystemExit("Could not identify the running GeoIntel backend")
for process_id in matches:
    os.kill(process_id, signal.SIGTERM)
deadline = time.monotonic() + 60
remaining = matches
while remaining and time.monotonic() < deadline:
    time.sleep(0.25)
    remaining = [process_id for process_id in remaining if pathlib.Path(f"/proc/{process_id}").exists()]
if remaining:
    raise SystemExit(f"Backend did not stop cleanly: {remaining}")
print(f"Stopped {len(matches)} backend process(es)")
PY
  then
    echo "Could not quiesce the current backend; refusing a potentially inconsistent backup." >&2
    docker restart geointel >/dev/null || true
    wait_for_geointel_health || true
    return 1
  fi

  GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}"
  echo "Creating mandatory pre-deploy backup ${release_id}..."
  if ! run_low_impact bash scripts/backup_release_state.sh \
      --container geointel \
      --output-root "$GEOINTEL_BACKUPS_PATH" \
      --release-id "$release_id" \
      --storage-path "$GEOINTEL_STORAGE_PATH" \
      --models-path "$GEOINTEL_MODELS_PATH" \
      --inventory-mode sha256 \
      --rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \
      "${backup_link_args[@]}" \
    || ! run_low_impact bash scripts/verify_release_backup.sh \
      --container geointel \
      --backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then
    echo "Pre-deploy backup failed; restarting the unchanged current release." >&2
    docker restart geointel >/dev/null || true
    wait_for_geointel_health || true
    GEOINTEL_PREDEPLOY_BACKUP_DIR=""
    return 1
  fi
  echo "Pre-deploy backup verified: ${GEOINTEL_PREDEPLOY_BACKUP_DIR}"
}

rollback_previous() {
  if [ -z "$GEOINTEL_PREDEPLOY_BACKUP_DIR" ]; then
    echo "Automatic rollback unavailable: no verified pre-deploy database backup was created." >&2
    return 1
  fi
  echo "Rolling back database and image to the verified pre-deploy state..."
  GEOINTEL_DEPLOY_LOCK_HELD=true \
    bash deploy/unraid/rollback-dockerman-container.sh \
      --backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR" \
      --confirm-production-database-restore
}

docker compose -f docker-compose.unraid.yml config >/dev/null

if docker image inspect "$GEOINTEL_RELEASE_IMAGE" >/dev/null 2>&1; then
  stored_revision="$(
    docker image inspect \
      --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \
      "$GEOINTEL_RELEASE_IMAGE"
  )"
  stored_ai="$(
    docker image inspect \
      --format '{{index .Config.Labels "io.geointel.ai.enabled"}}' \
      "$GEOINTEL_RELEASE_IMAGE"
  )"
  stored_version="$(
    docker image inspect \
      --format '{{index .Config.Labels "org.opencontainers.image.version"}}' \
      "$GEOINTEL_RELEASE_IMAGE"
  )"
  if (
    [ "$stored_revision" != "$GEOINTEL_BUILD_SHA" ] ||
    [ "$stored_ai" != "$GEOINTEL_INSTALL_AI" ] ||
    [ "$stored_version" != "$GEOINTEL_APP_VERSION" ]
  ); then
    echo "Immutable release tag has conflicting metadata: ${GEOINTEL_RELEASE_IMAGE}" >&2
    exit 2
  fi
  echo "Reusing existing immutable image ${GEOINTEL_RELEASE_IMAGE}."
  docker tag "$GEOINTEL_RELEASE_IMAGE" "${GEOINTEL_IMAGE_REPOSITORY}:latest"
else
  docker build \
    --build-arg GEOINTEL_INSTALL_AI="$GEOINTEL_INSTALL_AI" \
    --build-arg GEOINTEL_BUILD_SHA="$GEOINTEL_BUILD_SHA" \
    --build-arg GEOINTEL_BUILD_TIME="$GEOINTEL_BUILD_TIME" \
    --build-arg GEOINTEL_APP_VERSION="$GEOINTEL_APP_VERSION" \
    -f deploy/unraid/Dockerfile.all-in-one \
    -t "$GEOINTEL_RELEASE_IMAGE" \
    -t "${GEOINTEL_IMAGE_REPOSITORY}:latest" \
    .
fi

scan_release_image
create_predeploy_backup

if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"; then
  rollback_previous || true
  exit 1
fi

if [ -f scripts/live_migration_smoke.sh ]; then
  if ! LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh; then
    rollback_previous || true
    exit 1
  fi
fi

if [ -f scripts/verify_browser_runtime.sh ]; then
  if ! bash scripts/verify_browser_runtime.sh "$FRONTEND_URL"; then
    rollback_previous || true
    exit 1
  fi
fi

echo "Deployed immutable image ${GEOINTEL_RELEASE_IMAGE}."
docker image inspect \
  --format 'version={{index .Config.Labels "org.opencontainers.image.version"}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} created={{index .Config.Labels "org.opencontainers.image.created"}}' \
  "$GEOINTEL_RELEASE_IMAGE"