Initial public release

scripts/backup_release_state.sh

@@ -0,0 +1,336 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+CONTAINER="geointel"
+OUTPUT_ROOT="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
+RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)"
+STORAGE_PATH=""
+MODELS_PATH=""
+INVENTORY_MODE="metadata"
+ALLOW_INSECURE_PASSWORD="false"
+LINK_DEST_BACKUP=""
+ROLLBACK_IMAGE_TAG=""
+
+usage() {
+  cat <<'EOF'
+Usage: bash scripts/backup_release_state.sh [options]
+
+Creates an atomic, read-only release backup from the running all-in-one
+GeoIntel container. It never deletes or restores application data.
+
+Options:
+  --container NAME                 Docker container (default: geointel)
+  --output-root PATH               Host backup root (default:
+                                   /mnt/user/appdata/geointel/backups)
+  --release-id ID                  Safe backup directory name
+  --storage-path PATH              Host storage path to snapshot byte-for-byte
+  --models-path PATH               Host model path to snapshot byte-for-byte
+  --inventory-mode metadata|sha256 Retained manifest compatibility setting
+  --link-dest-backup PATH          Verified older backup used only to hard-link
+                                   checksum-identical backup-to-backup files
+  --rollback-image-tag TAG         Immutable backup-specific tag bound to the
+                                   running image ID
+  --allow-insecure-password        Complete emergency backup despite an
+                                   empty/default production DB password
+EOF
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --container) CONTAINER="$2"; shift 2 ;;
+    --output-root) OUTPUT_ROOT="$2"; shift 2 ;;
+    --release-id) RELEASE_ID="$2"; shift 2 ;;
+    --storage-path) STORAGE_PATH="$2"; shift 2 ;;
+    --models-path) MODELS_PATH="$2"; shift 2 ;;
+    --inventory-mode) INVENTORY_MODE="$2"; shift 2 ;;
+    --link-dest-backup) LINK_DEST_BACKUP="$2"; shift 2 ;;
+    --rollback-image-tag) ROLLBACK_IMAGE_TAG="$2"; shift 2 ;;
+    --allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;;
+    --help|-h) usage; exit 0 ;;
+    *) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if ! [[ "$RELEASE_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]]; then
+  echo "Unsafe release id: $RELEASE_ID" >&2
+  exit 2
+fi
+if [ "$INVENTORY_MODE" != "metadata" ] && [ "$INVENTORY_MODE" != "sha256" ]; then
+  echo "--inventory-mode must be metadata or sha256" >&2
+  exit 2
+fi
+for required in docker python3 sha256sum; do
+  if ! command -v "$required" >/dev/null 2>&1; then
+    echo "Missing required command: $required" >&2
+    exit 2
+  fi
+done
+
+resolve_source_revision() {
+  local controller_sha="" explicit_sha="${GEOINTEL_BUILD_SHA:-}"
+  local gitea_sha="${GITEA_COMMIT_SHA:-}" github_sha="${GITHUB_SHA:-}"
+  local git_head="" git_top="" git_dirty="false" source=""
+
+  if [ -n "$gitea_sha" ]; then
+    if ! [[ "$gitea_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
+      echo "GITEA_COMMIT_SHA must contain one full 40-character Git commit SHA." >&2
+      return 2
+    fi
+    controller_sha="${gitea_sha,,}"
+    source="GITEA_COMMIT_SHA"
+  fi
+  if [ -n "$github_sha" ]; then
+    if ! [[ "$github_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
+      echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2
+      return 2
+    fi
+    github_sha="${github_sha,,}"
+    if [ -n "$controller_sha" ] && [ "$controller_sha" != "$github_sha" ]; then
+      echo "Controller commit variables disagree." >&2
+      return 2
+    fi
+    controller_sha="$github_sha"
+    source="${source:-GITHUB_SHA}"
+  fi
+  if [ -n "$explicit_sha" ]; then
+    if ! [[ "$explicit_sha" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]]; then
+      echo "GEOINTEL_BUILD_SHA contains an unsafe release revision." >&2
+      return 2
+    fi
+    explicit_sha="${explicit_sha,,}"
+  fi
+  if [ -n "$controller_sha" ]; then
+    if [ -n "$explicit_sha" ] && [ "$explicit_sha" != "$controller_sha" ]; then
+      echo "GEOINTEL_BUILD_SHA differs from the controller revision." >&2
+      return 2
+    fi
+    explicit_sha="$controller_sha"
+  elif [ -n "${GITEA_REPOSITORY:-}${GITHUB_REPOSITORY:-}" ]; then
+    if ! [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]]; then
+      echo "Automated backup requires a full controller or GEOINTEL_BUILD_SHA revision." >&2
+      return 2
+    fi
+    source="GEOINTEL_BUILD_SHA"
+  fi
+
+  if command -v git >/dev/null 2>&1 && git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1; then
+    git_top="$(git -C "$ROOT" rev-parse --show-toplevel 2>/dev/null || true)"
+  fi
+  if [ -n "$git_top" ] && [ "$(cd "$git_top" && pwd -P)" = "$(cd "$ROOT" && pwd -P)" ]; then
+    git_head="$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)"
+    git_head="${git_head,,}"
+    if ! [[ "$git_head" =~ ^[0-9a-f]{40}$ ]]; then
+      echo "Could not resolve a full Git revision from the source checkout." >&2
+      return 2
+    fi
+    if [ -n "$explicit_sha" ] && [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]] && [ "$git_head" != "$explicit_sha" ]; then
+      echo "Source checkout does not match the supplied release revision." >&2
+      return 2
+    fi
+    if [ -n "$(git -C "$ROOT" status --porcelain=v1 2>/dev/null)" ]; then
+      git_dirty="true"
+    fi
+    if [ -z "$explicit_sha" ]; then
+      explicit_sha="$git_head"
+      source="git"
+    fi
+  fi
+
+  if [ -z "$explicit_sha" ]; then
+    echo "Cannot bind backup to a source revision; provide GEOINTEL_BUILD_SHA or a controller SHA." >&2
+    return 2
+  fi
+  SOURCE_REVISION="$explicit_sha"
+  SOURCE_REVISION_SOURCE="${source:-GEOINTEL_BUILD_SHA}"
+  SOURCE_GIT_DIRTY="$git_dirty"
+}
+
+SOURCE_REVISION=""
+SOURCE_REVISION_SOURCE=""
+SOURCE_GIT_DIRTY="false"
+resolve_source_revision
+if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
+  echo "Container '$CONTAINER' is not running." >&2
+  exit 3
+fi
+
+OUTPUT_ROOT="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$OUTPUT_ROOT")"
+mkdir -p "$OUTPUT_ROOT"
+PARTIAL="$OUTPUT_ROOT/.${RELEASE_ID}.partial"
+FINAL="$OUTPUT_ROOT/$RELEASE_ID"
+if [ -e "$PARTIAL" ] || [ -e "$FINAL" ]; then
+  echo "Backup target already exists: $FINAL" >&2
+  exit 3
+fi
+mkdir -p "$PARTIAL"
+
+if [ -n "$LINK_DEST_BACKUP" ]; then
+  LINK_DEST_BACKUP="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve(strict=True))' "$LINK_DEST_BACKUP")"
+  python3 - "$OUTPUT_ROOT" "$LINK_DEST_BACKUP" <<'PY'
+import pathlib
+import sys
+
+root = pathlib.Path(sys.argv[1])
+candidate = pathlib.Path(sys.argv[2])
+try:
+    candidate.relative_to(root)
+except ValueError as exc:
+    raise SystemExit(f"Link-dest backup must remain below {root}") from exc
+if candidate == root or candidate.name.startswith("."):
+    raise SystemExit("Link-dest backup must identify one completed immutable backup")
+PY
+  (
+    cd "$LINK_DEST_BACKUP"
+    sha256sum -c CHECKSUMS.sha256 >/dev/null
+  )
+fi
+
+cleanup_partial() {
+  if [ -d "$PARTIAL" ]; then
+    rm -rf -- "$PARTIAL"
+  fi
+}
+trap cleanup_partial EXIT
+
+DB_NAME="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"' )"
+DB_USER="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"' )"
+if [ -z "$DB_NAME" ] || [ -z "$DB_USER" ]; then
+  echo "Could not resolve database identity from the container." >&2
+  exit 3
+fi
+
+PASSWORD_SECURE="true"
+if ! docker exec "$CONTAINER" sh -c '
+  password="${POSTGRES_PASSWORD:-${GEOINTEL_POSTGRES_PASSWORD:-}}"
+  test -n "$password" &&
+  test "$password" != "geointel" &&
+  test "$password" != "postgres" &&
+  test "$password" != "password"
+'; then
+  PASSWORD_SECURE="false"
+fi
+
+echo "Creating PostgreSQL custom-format dump..."
+docker exec "$CONTAINER" pg_dump \
+  -U "$DB_USER" \
+  -d "$DB_NAME" \
+  -Fc \
+  --no-owner \
+  --no-privileges > "$PARTIAL/database.dump"
+test -s "$PARTIAL/database.dump"
+
+docker exec -i "$CONTAINER" pg_restore --list \
+  < "$PARTIAL/database.dump" \
+  > "$PARTIAL/database.list"
+test -s "$PARTIAL/database.list"
+
+IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")"
+IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")"
+RUNNING_IMAGE_REVISION="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CONTAINER")"
+RUNNING_IMAGE_REVISION_IS_FULL_SHA="false"
+if [[ "$RUNNING_IMAGE_REVISION" =~ ^[0-9A-Fa-f]{40}$ ]]; then
+  RUNNING_IMAGE_REVISION="${RUNNING_IMAGE_REVISION,,}"
+  RUNNING_IMAGE_REVISION_IS_FULL_SHA="true"
+elif ! [[ "$RUNNING_IMAGE_REVISION" =~ ^[A-Za-z0-9._-]{1,128}$ ]]; then
+  echo "Running image has an unsafe or missing OCI revision label." >&2
+  exit 3
+fi
+if [ -n "$ROLLBACK_IMAGE_TAG" ]; then
+  TAGGED_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$ROLLBACK_IMAGE_TAG" 2>/dev/null || true)"
+  if [ "$TAGGED_IMAGE_ID" != "$IMAGE_ID" ]; then
+    echo "Backup-specific rollback tag does not resolve to the running image ID." >&2
+    exit 3
+  fi
+fi
+
+docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \
+  -c "SELECT 'alembic_head', version_num FROM alembic_version
+      UNION ALL SELECT 'postgis_version', postgis_version()
+      UNION ALL SELECT 'database_size_bytes', pg_database_size(current_database())::text
+      ORDER BY 1;" > "$PARTIAL/database-metadata.tsv"
+
+: > "$PARTIAL/table-counts.tsv"
+for table in projects areas datasets dataset_versions vector_features jobs analysis_runs detections segmentations quality_checks metrics exports; do
+  count="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -At \
+    -c "SELECT count(*) FROM public.${table};")"
+  printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv"
+done
+
+snapshot_path() {
+  local source_path="$1"
+  local label="$2"
+  local manifest_path="$PARTIAL/${label}-manifest.tsv"
+  local snapshot_path="$PARTIAL/${label}-snapshot"
+  local link_args=()
+  if [ -z "$source_path" ]; then
+    printf 'not_requested\n' > "$manifest_path"
+    return
+  fi
+  if [ -n "$LINK_DEST_BACKUP" ]; then
+    link_args=(
+      --link-dest-snapshot "$LINK_DEST_BACKUP/${label}-snapshot"
+      --link-dest-manifest "$LINK_DEST_BACKUP/${label}-manifest.tsv"
+    )
+  fi
+  python3 "$ROOT/scripts/release_backup_snapshot.py" create \
+    --source "$source_path" \
+    --snapshot "$snapshot_path" \
+    --manifest "$manifest_path" \
+    --label "$label" \
+    "${link_args[@]}"
+}
+
+snapshot_path "$STORAGE_PATH" storage
+snapshot_path "$MODELS_PATH" models
+
+python3 - "$PARTIAL/manifest.json" <<PY
+import json
+import pathlib
+
+payload = {
+    "schema_version": 1,
+    "release_id": ${RELEASE_ID@Q},
+    "created_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
+    "read_only_source": True,
+    "container": ${CONTAINER@Q},
+    "database_name": ${DB_NAME@Q},
+    "database_user": ${DB_USER@Q},
+    "database_password_secure": ${PASSWORD_SECURE@Q} == "true",
+    "image_id": ${IMAGE_ID@Q},
+    "image_name": ${IMAGE_NAME@Q},
+    "rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None,
+    "backup_tool_revision": ${SOURCE_REVISION@Q},
+    "backup_tool_revision_source": ${SOURCE_REVISION_SOURCE@Q},
+    "backup_tool_git_dirty": ${SOURCE_GIT_DIRTY@Q} == "true",
+    "running_image_revision": ${RUNNING_IMAGE_REVISION@Q},
+    "running_image_revision_is_full_sha": ${RUNNING_IMAGE_REVISION_IS_FULL_SHA@Q} == "true",
+    "inventory_mode": ${INVENTORY_MODE@Q},
+    "storage_inventory_requested": bool(${STORAGE_PATH@Q}),
+    "models_inventory_requested": bool(${MODELS_PATH@Q}),
+    "storage_snapshot_requested": bool(${STORAGE_PATH@Q}),
+    "models_snapshot_requested": bool(${MODELS_PATH@Q}),
+    "link_dest_backup": ${LINK_DEST_BACKUP@Q} or None,
+}
+path = pathlib.Path(__import__("sys").argv[1])
+path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+PY
+
+python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$PARTIAL"
+
+(
+  cd "$PARTIAL"
+  find . -maxdepth 1 -type f ! -name CHECKSUMS.sha256 -printf '%f\n' \
+    | LC_ALL=C sort \
+    | xargs sha256sum > CHECKSUMS.sha256
+)
+
+mv "$PARTIAL" "$FINAL"
+trap - EXIT
+echo "Release backup created: $FINAL"
+
+if [ "$PASSWORD_SECURE" != "true" ] && [ "$ALLOW_INSECURE_PASSWORD" != "true" ]; then
+  echo "SECURITY GATE FAILED: production database password is empty or a known default." >&2
+  echo "The emergency backup is valid, but the release remains blocked." >&2
+  exit 4
+fi