fix(release): make deployment backup and rollback immutable

deploy/unraid/deploy-release.sh

@@ -26,7 +26,11 @@ if [ -n "${DEPLOY_GEOINTEL_INSTALL_AI:-}" ]; then
   GEOINTEL_INSTALL_AI="$DEPLOY_GEOINTEL_INSTALL_AI"
 fi
 
-GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-false}"
+GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"
+if [ "$GEOINTEL_INSTALL_AI" != "true" ]; then
+  echo "Production release deployment requires the gated AI image (GEOINTEL_INSTALL_AI=true)." >&2
+  exit 2
+fi
 GEOINTEL_APP_VERSION="$(tr -d '[:space:]' < VERSION)"
 if ! [[ "$GEOINTEL_APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then
   echo "Invalid semantic version in VERSION: ${GEOINTEL_APP_VERSION}" >&2
@@ -57,7 +61,52 @@ source_tree_hash() {
 }
 
 resolve_build_sha() {
-  local head="" content=""
+  local head="" content="" controller_sha="" controller_source=""
+
+  if [ -n "${GITEA_COMMIT_SHA:-}" ]; then
+    controller_sha="$GITEA_COMMIT_SHA"
+    controller_source="GITEA_COMMIT_SHA"
+  fi
+  if [ -n "${GITHUB_SHA:-}" ]; then
+    if ! [[ "$GITHUB_SHA" =~ ^[0-9A-Fa-f]{40}$ ]]; then
+      echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2
+      return 2
+    fi
+    if [ -n "$controller_sha" ] && [ "${controller_sha,,}" != "${GITHUB_SHA,,}" ]; then
+      echo "Controller commit variables disagree." >&2
+      return 2
+    fi
+    controller_sha="$GITHUB_SHA"
+    controller_source="${controller_source:-GITHUB_SHA}"
+  fi
+  if [ -n "$controller_sha" ]; then
+    if ! [[ "$controller_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
+      echo "${controller_source} must contain one full 40-character Git commit SHA." >&2
+      return 2
+    fi
+    controller_sha="${controller_sha,,}"
+    if [ -n "${GEOINTEL_BUILD_SHA:-}" ] && [ "${GEOINTEL_BUILD_SHA,,}" != "$controller_sha" ]; then
+      echo "Explicit build revision differs from the controller revision." >&2
+      return 2
+    fi
+    if command -v git >/dev/null 2>&1 && git rev-parse --git-dir >/dev/null 2>&1; then
+      head="$(git rev-parse HEAD 2>/dev/null || true)"
+      if [ "${head,,}" != "$controller_sha" ]; then
+        echo "Prepared Git checkout does not match the controller revision." >&2
+        return 2
+      fi
+      if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
+        echo "Prepared Git checkout contains changes outside the controller revision." >&2
+        return 2
+      fi
+    fi
+    printf '%s' "$controller_sha"
+    return 0
+  fi
+  if [ -n "${GITEA_REPOSITORY:-}" ] || [ -n "${GITHUB_REPOSITORY:-}" ]; then
+    echo "Automated deployment context is missing GITEA_COMMIT_SHA/GITHUB_SHA." >&2
+    return 2
+  fi
 
   # 1. Explicit override wins.
   if [ -n "${GEOINTEL_BUILD_SHA:-}" ]; then
@@ -107,7 +156,13 @@ if [ -z "$GEOINTEL_BUILD_SHA" ]; then
   echo "Could not determine a build revision for this deployment." >&2
   exit 2
 fi
+export GEOINTEL_BUILD_SHA
 echo "Build revision: ${GEOINTEL_BUILD_SHA}"
+GEOINTEL_RELEASE_TOKEN="$(printf '%s' "$GEOINTEL_BUILD_SHA" | tr -c 'A-Za-z0-9._-' '_' | cut -c1-48)"
+if [ -z "$GEOINTEL_RELEASE_TOKEN" ]; then
+  echo "Could not derive a safe release evidence identifier." >&2
+  exit 2
+fi
 GEOINTEL_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
 GEOINTEL_IMAGE_REPOSITORY="${GEOINTEL_IMAGE_REPOSITORY:-geointel-all-in-one}"
 if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then
@@ -116,8 +171,23 @@ else
   GEOINTEL_RELEASE_VARIANT="gis"
 fi
 GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"
-GEOINTEL_PREVIOUS_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:previous"
 FRONTEND_URL="${FRONTEND_URL:-http://127.0.0.1:${GEOINTEL_FRONTEND_PORT:-1202}}"
+GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
+GEOINTEL_STORAGE_PATH="${GEOINTEL_STORAGE_PATH:-/mnt/user/appdata/geointel/storage}"
+GEOINTEL_MODELS_PATH="${GEOINTEL_MODELS_PATH:-/mnt/user/appdata/geointel/models}"
+GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}"
+GEOINTEL_DEPLOY_EVIDENCE_DIR="${GEOINTEL_DEPLOY_EVIDENCE_DIR:-artifacts/release-evidence/deploy/${GEOINTEL_RELEASE_TOKEN}-ai}"
+GEOINTEL_PREDEPLOY_BACKUP_DIR=""
+GEOINTEL_RELEASE_IMAGE_ID=""
+GEOINTEL_BACKUP_LINK_DEST=""
+GEOINTEL_PREDEPLOY_ROLLBACK_TAG=""
+
+case "$GEOINTEL_DEPLOY_EVIDENCE_DIR" in
+  /*|*..*)
+    echo "Deployment evidence directory must be repository-relative and must not contain '..'." >&2
+    exit 2
+    ;;
+esac
 
 wait_for_geointel_health() {
   local status=""
@@ -141,35 +211,312 @@ wait_for_geointel_health() {
 
 start_image() {
   local image="$1"
+  local running_image_id=""
+  local running_revision=""
+  local running_ai=""
   GEOINTEL_IMAGE="$image" bash deploy/unraid/run-dockerman-container.sh
   wait_for_geointel_health
+  running_image_id="$(docker inspect --format '{{.Image}}' geointel)"
+  if [ "$running_image_id" != "$image" ]; then
+    echo "Running container image ${running_image_id} differs from attested image ${image}." >&2
+    return 1
+  fi
+  running_revision="$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' geointel)"
+  running_ai="$(docker inspect --format '{{index .Config.Labels "io.geointel.ai.enabled"}}' geointel)"
+  if [ "$running_revision" != "$GEOINTEL_BUILD_SHA" ] || [ "$running_ai" != "true" ]; then
+    echo "Running container labels do not match the attested AI revision." >&2
+    return 1
+  fi
+  echo "Running container matches attested image: ${running_image_id}"
+}
+
+scan_release_image() {
+  local scanned_image_id=""
+  local current_image_id=""
+  local inspect_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/image-inspect.json"
+  local sbom_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-sbom.spdx.json"
+  local vulnerability_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-container-vulnerabilities.json"
+  local attestation_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/deployment-attestation.json"
+
+  scanned_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
+  test -n "$scanned_image_id"
+  mkdir -p "$ROOT/$GEOINTEL_DEPLOY_EVIDENCE_DIR"
+  docker image inspect "$GEOINTEL_RELEASE_IMAGE" > "$ROOT/$inspect_output"
+  bash scripts/generate_container_sbom.sh "$GEOINTEL_RELEASE_IMAGE" "$sbom_output"
+  bash scripts/scan_container_image.sh "$GEOINTEL_RELEASE_IMAGE" "$vulnerability_output"
+  current_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
+  if [ "$current_image_id" != "$scanned_image_id" ]; then
+    echo "Release image tag changed while SBOM/scan evidence was being generated." >&2
+    return 1
+  fi
+  test -s "$ROOT/$inspect_output"
+  test -s "$ROOT/$sbom_output"
+  test -s "$ROOT/$vulnerability_output"
+  GEOINTEL_RELEASE_IMAGE_ID="$scanned_image_id"
+  python3 - \
+    "$ROOT/$attestation_output" \
+    "$GEOINTEL_RELEASE_IMAGE" \
+    "$GEOINTEL_RELEASE_IMAGE_ID" \
+    "$GEOINTEL_BUILD_SHA" \
+    "$inspect_output" \
+    "$sbom_output" \
+    "$vulnerability_output" <<'PY'
+import datetime
+import json
+import pathlib
+import sys
+
+output, image_tag, image_id, revision, inspect_path, sbom_path, vulnerability_path = sys.argv[1:]
+payload = {
+    "schema_version": 1,
+    "attested_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+    "image_tag": image_tag,
+    "image_id": image_id,
+    "image_config_digest": image_id,
+    "revision": revision,
+    "variant": "ai",
+    "evidence": {
+        "image_inspect": inspect_path,
+        "sbom": sbom_path,
+        "vulnerabilities": vulnerability_path,
+    },
+}
+path = pathlib.Path(output)
+temporary = path.with_suffix(".json.partial")
+temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+temporary.replace(path)
+PY
+  test -s "$ROOT/$attestation_output"
+  echo "Exact deployment image scanned: ${GEOINTEL_RELEASE_IMAGE_ID}"
+}
+
+preflight_backup_capacity() {
+  local database_name=""
+  local database_user=""
+  local database_size_bytes=""
+
+  database_name="$(docker exec geointel sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"')"
+  database_user="$(docker exec geointel sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"')"
+  database_size_bytes="$(docker exec geointel psql -X -v ON_ERROR_STOP=1 \
+    -U "$database_user" -d "$database_name" -Atqc \
+    'SELECT pg_database_size(current_database());')"
+  mkdir -p "$GEOINTEL_BACKUPS_PATH"
+  python3 - \
+    "$GEOINTEL_BACKUPS_PATH" \
+    "$GEOINTEL_STORAGE_PATH" \
+    "$GEOINTEL_MODELS_PATH" \
+    "$database_size_bytes" <<'PY'
+import os
+import pathlib
+import shutil
+import stat
+import sys
+
+backup_root = pathlib.Path(sys.argv[1]).expanduser().resolve()
+sources = [pathlib.Path(value).expanduser().resolve() for value in sys.argv[2:4]]
+database_bytes = int(sys.argv[4])
+
+def retained_bytes(root: pathlib.Path) -> int:
+    if not root.is_dir():
+        raise SystemExit(f"Mandatory snapshot source is not a directory: {root}")
+    total = 0
+    for current, directories, files in os.walk(root, topdown=True, followlinks=False):
+        current_path = pathlib.Path(current)
+        for name in [*directories, *files]:
+            path = current_path / name
+            details = path.lstat()
+            if stat.S_ISLNK(details.st_mode):
+                raise SystemExit(f"Mandatory snapshot refuses symlinked content: {path}")
+            if name in directories and not stat.S_ISDIR(details.st_mode):
+                raise SystemExit(f"Snapshot directory changed during capacity preflight: {path}")
+            if name in files:
+                if not stat.S_ISREG(details.st_mode):
+                    raise SystemExit(f"Mandatory snapshot refuses non-regular content: {path}")
+                total += details.st_size
+    return total
+
+source_bytes = sum(retained_bytes(source) for source in sources)
+# Reflink clones are used when the backing filesystem supports them. Budget for
+# a complete copy plus two uncompressed database sizes (dump and isolated
+# restore/cutover recovery) so fallback still fails before the live backend is
+# quiesced rather than midway through the snapshot.
+required = source_bytes + (2 * database_bytes)
+headroom = max(5 * 1024**3, required // 10)
+free = shutil.disk_usage(backup_root).free
+if free < required + headroom:
+    raise SystemExit(
+        "Insufficient free space for a fail-safe predeploy snapshot: "
+        f"required={required + headroom} free={free} source={source_bytes} database={database_bytes}"
+    )
+print(
+    "Predeploy snapshot capacity: "
+    f"source_bytes={source_bytes} database_bytes={database_bytes} free_bytes={free}"
+)
+PY
+}
+
+select_verified_link_dest() {
+  local candidate=""
+  GEOINTEL_BACKUP_LINK_DEST=""
+  while IFS= read -r candidate; do
+    if (
+      cd "$candidate" \
+        && sha256sum -c CHECKSUMS.sha256 >/dev/null \
+        && python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate"
+    ); then
+      GEOINTEL_BACKUP_LINK_DEST="$candidate"
+      echo "Using verified prior byte snapshot as link-dest: ${candidate}"
+      return 0
+    fi
+    echo "Skipping unusable prior backup link-dest: ${candidate}" >&2
+  done < <(
+    python3 - "$GEOINTEL_BACKUPS_PATH" <<'PY'
+import pathlib
+import sys
+
+root = pathlib.Path(sys.argv[1]).expanduser().resolve()
+candidates = sorted(
+    (
+        path
+        for path in root.iterdir()
+        if path.is_dir() and not path.name.startswith(".") and (path / "manifest.json").is_file()
+    ),
+    key=lambda path: path.stat().st_mtime_ns,
+    reverse=True,
+)
+for candidate in candidates:
+    print(candidate)
+PY
+  )
+  echo "No verified prior byte snapshot found; this deployment will create a full first snapshot."
+}
+
+create_predeploy_backup() {
+  local container_exists="false"
+  local container_running="false"
+  local release_id=""
+  local backup_link_args=()
+  local current_image_id=""
+  local existing_rollback_id=""
+
+  if docker ps -a --format '{{.Names}}' | grep -Fxq geointel; then
+    container_exists="true"
+  fi
+  if [ "$(docker inspect -f '{{.State.Running}}' geointel 2>/dev/null || true)" = "true" ]; then
+    container_running="true"
+  fi
+
+  if [ "$container_exists" = "false" ]; then
+    if [ -f "$GEOINTEL_POSTGIS_DATA_PATH/PG_VERSION" ]; then
+      echo "PostGIS data exists without a running GeoIntel container; refusing an unbacked migration." >&2
+      return 1
+    fi
+    echo "No existing GeoIntel state found; pre-deploy backup is not required for this fresh install."
+    return 0
+  fi
+  if [ "$container_running" != "true" ]; then
+    echo "Existing GeoIntel container is not running; refusing deployment because a consistent backup cannot be created." >&2
+    return 1
+  fi
+
+  current_image_id="$(docker inspect --format '{{.Image}}' geointel)"
+  if ! [[ "$current_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+    echo "Running release does not expose one immutable Docker image ID." >&2
+    return 1
+  fi
+  release_id="predeploy-${GEOINTEL_RELEASE_TOKEN:0:24}-$(date -u +%Y%m%dT%H%M%SZ)-$$"
+  GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"
+  existing_rollback_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" 2>/dev/null || true)"
+  if [ -n "$existing_rollback_id" ] && [ "$existing_rollback_id" != "$current_image_id" ]; then
+    echo "Backup-specific rollback tag already identifies different image bytes." >&2
+    return 1
+  fi
+  docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"
+
+  # This conservative full-copy fallback estimate runs while the existing
+  # release is still healthy. Verified backup-to-backup hardlinks normally
+  # avoid recopying unchanged bytes, but are never assumed for this fail-closed
+  # capacity decision.
+  preflight_backup_capacity
+  select_verified_link_dest
+  if [ -n "$GEOINTEL_BACKUP_LINK_DEST" ]; then
+    backup_link_args=(--link-dest-backup "$GEOINTEL_BACKUP_LINK_DEST")
+  fi
+
+  echo "Quiescing the current backend so the rollback point cannot miss concurrent writes..."
+  if ! docker exec -i geointel python - <<'PY'
+import os
+import pathlib
+import signal
+import time
+
+matches = []
+for item in pathlib.Path("/proc").iterdir():
+    if not item.name.isdigit() or int(item.name) in {os.getpid(), os.getppid()}:
+        continue
+    try:
+        command = (item / "cmdline").read_bytes().replace(b"\0", b" ")
+    except (OSError, PermissionError):
+        continue
+    if b"uvicorn" in command and b"app.main:app" in command:
+        matches.append(int(item.name))
+if not matches:
+    raise SystemExit("Could not identify the running GeoIntel backend")
+for process_id in matches:
+    os.kill(process_id, signal.SIGTERM)
+deadline = time.monotonic() + 60
+remaining = matches
+while remaining and time.monotonic() < deadline:
+    time.sleep(0.25)
+    remaining = [process_id for process_id in remaining if pathlib.Path(f"/proc/{process_id}").exists()]
+if remaining:
+    raise SystemExit(f"Backend did not stop cleanly: {remaining}")
+print(f"Stopped {len(matches)} backend process(es)")
+PY
+  then
+    echo "Could not quiesce the current backend; refusing a potentially inconsistent backup." >&2
+    docker restart geointel >/dev/null || true
+    wait_for_geointel_health || true
+    return 1
+  fi
+
+  GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}"
+  echo "Creating mandatory pre-deploy backup ${release_id}..."
+  if ! bash scripts/backup_release_state.sh \
+      --container geointel \
+      --output-root "$GEOINTEL_BACKUPS_PATH" \
+      --release-id "$release_id" \
+      --storage-path "$GEOINTEL_STORAGE_PATH" \
+      --models-path "$GEOINTEL_MODELS_PATH" \
+      --inventory-mode sha256 \
+      --rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \
+      "${backup_link_args[@]}" \
+    || ! bash scripts/verify_release_backup.sh \
+      --container geointel \
+      --backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then
+    echo "Pre-deploy backup failed; restarting the unchanged current release." >&2
+    docker restart geointel >/dev/null || true
+    wait_for_geointel_health || true
+    GEOINTEL_PREDEPLOY_BACKUP_DIR=""
+    return 1
+  fi
+  echo "Pre-deploy backup verified: ${GEOINTEL_PREDEPLOY_BACKUP_DIR}"
 }
 
 rollback_previous() {
-  if ! docker image inspect "$GEOINTEL_PREVIOUS_IMAGE" >/dev/null 2>&1; then
-    echo "Automatic rollback unavailable: ${GEOINTEL_PREVIOUS_IMAGE} does not exist." >&2
+  if [ -z "$GEOINTEL_PREDEPLOY_BACKUP_DIR" ]; then
+    echo "Automatic rollback unavailable: no verified pre-deploy database backup was created." >&2
     return 1
   fi
-  echo "Rolling back to ${GEOINTEL_PREVIOUS_IMAGE}..."
-  start_image "$GEOINTEL_PREVIOUS_IMAGE"
+  echo "Rolling back database and image to the verified pre-deploy state..."
+  GEOINTEL_DEPLOY_LOCK_HELD=true \
+    bash deploy/unraid/rollback-dockerman-container.sh \
+      --backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR" \
+      --confirm-production-database-restore
 }
 
 docker compose -f docker-compose.unraid.yml config >/dev/null
 
-current_image_id="$(docker inspect --format '{{.Image}}' geointel 2>/dev/null || true)"
-release_image_id="$(
-  docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE" 2>/dev/null || true
-)"
-if (
-  [ -n "$current_image_id" ] &&
-  [ "$current_image_id" != "$release_image_id" ] &&
-  docker image inspect "$current_image_id" >/dev/null 2>&1
-); then
-  docker tag "$current_image_id" "$GEOINTEL_PREVIOUS_IMAGE"
-elif [ -n "$current_image_id" ] && [ "$current_image_id" = "$release_image_id" ]; then
-  echo "Current container already uses ${GEOINTEL_RELEASE_IMAGE}; preserving the existing previous image."
-fi
-
 if docker image inspect "$GEOINTEL_RELEASE_IMAGE" >/dev/null 2>&1; then
   stored_revision="$(
     docker image inspect \
@@ -208,7 +555,10 @@ else
     .
 fi
 
-if ! start_image "$GEOINTEL_RELEASE_IMAGE"; then
+scan_release_image
+create_predeploy_backup
+
+if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"; then
   rollback_previous || true
   exit 1
 fi