Start autonomous Belgium and North Sea RC

2026-07-18 00:12:34 +02:00
parent 9513f8613e
commit 9c402e0df2
36 changed files with 2235 additions and 115 deletions
@@ -0,0 +1,217 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CONTAINER="geointel"
+OUTPUT_ROOT="backups"
+RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)"
+STORAGE_PATH=""
+MODELS_PATH=""
+INVENTORY_MODE="metadata"
+ALLOW_INSECURE_PASSWORD="false"
+
+usage() {
+  cat <<'EOF'
+Usage: bash scripts/backup_release_state.sh [options]
+
+Creates an atomic, read-only release backup from the running all-in-one
+GeoIntel container. It never deletes or restores application data.
+
+Options:
+  --container NAME                 Docker container (default: geointel)
+  --output-root PATH               Host backup root (default: backups)
+  --release-id ID                  Safe backup directory name
+  --storage-path PATH              Optional host storage path to inventory
+  --models-path PATH               Optional host model path to inventory
+  --inventory-mode metadata|sha256 Hash all inventoried files only with sha256
+  --allow-insecure-password        Complete emergency backup despite an
+                                   empty/default production DB password
+EOF
+}
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --container) CONTAINER="$2"; shift 2 ;;
+    --output-root) OUTPUT_ROOT="$2"; shift 2 ;;
+    --release-id) RELEASE_ID="$2"; shift 2 ;;
+    --storage-path) STORAGE_PATH="$2"; shift 2 ;;
+    --models-path) MODELS_PATH="$2"; shift 2 ;;
+    --inventory-mode) INVENTORY_MODE="$2"; shift 2 ;;
+    --allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;;
+    --help|-h) usage; exit 0 ;;
+    *) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if ! [[ "$RELEASE_ID" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]]; then
+  echo "Unsafe release id: $RELEASE_ID" >&2
+  exit 2
+fi
+if [ "$INVENTORY_MODE" != "metadata" ] && [ "$INVENTORY_MODE" != "sha256" ]; then
+  echo "--inventory-mode must be metadata or sha256" >&2
+  exit 2
+fi
+for required in docker python3 sha256sum git; do
+  if ! command -v "$required" >/dev/null 2>&1; then
+    echo "Missing required command: $required" >&2
+    exit 2
+  fi
+done
+if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
+  echo "Container '$CONTAINER' is not running." >&2
+  exit 3
+fi
+
+OUTPUT_ROOT="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve())' "$OUTPUT_ROOT")"
+mkdir -p "$OUTPUT_ROOT"
+PARTIAL="$OUTPUT_ROOT/.${RELEASE_ID}.partial"
+FINAL="$OUTPUT_ROOT/$RELEASE_ID"
+if [ -e "$PARTIAL" ] || [ -e "$FINAL" ]; then
+  echo "Backup target already exists: $FINAL" >&2
+  exit 3
+fi
+mkdir -p "$PARTIAL"
+
+cleanup_partial() {
+  if [ -d "$PARTIAL" ]; then
+    rm -rf -- "$PARTIAL"
+  fi
+}
+trap cleanup_partial EXIT
+
+DB_NAME="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"' )"
+DB_USER="$(docker exec "$CONTAINER" sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"' )"
+if [ -z "$DB_NAME" ] || [ -z "$DB_USER" ]; then
+  echo "Could not resolve database identity from the container." >&2
+  exit 3
+fi
+
+PASSWORD_SECURE="true"
+if ! docker exec "$CONTAINER" sh -c '
+  password="${POSTGRES_PASSWORD:-${GEOINTEL_POSTGRES_PASSWORD:-}}"
+  test -n "$password" &&
+  test "$password" != "geointel" &&
+  test "$password" != "postgres" &&
+  test "$password" != "password"
+'; then
+  PASSWORD_SECURE="false"
+fi
+
+echo "Creating PostgreSQL custom-format dump..."
+docker exec "$CONTAINER" pg_dump \
+  -U "$DB_USER" \
+  -d "$DB_NAME" \
+  -Fc \
+  --no-owner \
+  --no-privileges > "$PARTIAL/database.dump"
+test -s "$PARTIAL/database.dump"
+
+docker exec -i "$CONTAINER" pg_restore --list \
+  < "$PARTIAL/database.dump" \
+  > "$PARTIAL/database.list"
+test -s "$PARTIAL/database.list"
+
+IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")"
+IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")"
+GIT_COMMIT="$(git rev-parse HEAD)"
+GIT_DIRTY="false"
+if [ -n "$(git status --porcelain=v1)" ]; then
+  GIT_DIRTY="true"
+fi
+
+docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \
+  -c "SELECT 'alembic_head', version_num FROM alembic_version
+      UNION ALL SELECT 'postgis_version', postgis_version()
+      UNION ALL SELECT 'database_size_bytes', pg_database_size(current_database())::text
+      ORDER BY 1;" > "$PARTIAL/database-metadata.tsv"
+
+: > "$PARTIAL/table-counts.tsv"
+for table in projects areas datasets dataset_versions vector_features jobs analysis_runs detections segmentations quality_checks metrics exports; do
+  count="$(docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -At \
+    -c "SELECT count(*) FROM public.${table};")"
+  printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv"
+done
+
+inventory_path() {
+  local source_path="$1"
+  local output_path="$2"
+  if [ -z "$source_path" ]; then
+    printf 'not_requested\n' > "$output_path"
+    return
+  fi
+  python3 - "$source_path" "$output_path" "$INVENTORY_MODE" <<'PY'
+import hashlib
+import os
+import pathlib
+import sys
+
+root = pathlib.Path(sys.argv[1]).expanduser().resolve()
+output = pathlib.Path(sys.argv[2])
+mode = sys.argv[3]
+if not root.is_dir():
+    raise SystemExit(f"Inventory root is not a directory: {root}")
+
+def digest(path: pathlib.Path) -> str:
+    value = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            value.update(chunk)
+    return value.hexdigest()
+
+with output.open("w", encoding="utf-8", newline="\n") as handle:
+    handle.write("relative_path\tsize_bytes\tmtime_ns\tsha256\n")
+    for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()):
+        if path.is_symlink() or not path.is_file():
+            continue
+        stat = path.stat()
+        checksum = digest(path) if mode == "sha256" else ""
+        relative = path.relative_to(root).as_posix()
+        if "\t" in relative or "\n" in relative:
+            raise SystemExit(f"Unsupported inventory path: {relative!r}")
+        handle.write(f"{relative}\t{stat.st_size}\t{stat.st_mtime_ns}\t{checksum}\n")
+PY
+}
+
+inventory_path "$STORAGE_PATH" "$PARTIAL/storage-manifest.tsv"
+inventory_path "$MODELS_PATH" "$PARTIAL/models-manifest.tsv"
+
+python3 - "$PARTIAL/manifest.json" <<PY
+import json
+import pathlib
+
+payload = {
+    "schema_version": 1,
+    "release_id": ${RELEASE_ID@Q},
+    "created_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
+    "read_only_source": True,
+    "container": ${CONTAINER@Q},
+    "database_name": ${DB_NAME@Q},
+    "database_user": ${DB_USER@Q},
+    "database_password_secure": ${PASSWORD_SECURE@Q} == "true",
+    "image_id": ${IMAGE_ID@Q},
+    "image_name": ${IMAGE_NAME@Q},
+    "git_commit": ${GIT_COMMIT@Q},
+    "git_dirty": ${GIT_DIRTY@Q} == "true",
+    "inventory_mode": ${INVENTORY_MODE@Q},
+    "storage_inventory_requested": bool(${STORAGE_PATH@Q}),
+    "models_inventory_requested": bool(${MODELS_PATH@Q}),
+}
+path = pathlib.Path(__import__("sys").argv[1])
+path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+PY
+
+(
+  cd "$PARTIAL"
+  find . -maxdepth 1 -type f ! -name CHECKSUMS.sha256 -printf '%f\n' \
+    | LC_ALL=C sort \
+    | xargs sha256sum > CHECKSUMS.sha256
+)
+
+mv "$PARTIAL" "$FINAL"
+trap - EXIT
+echo "Release backup created: $FINAL"
+
+if [ "$PASSWORD_SECURE" != "true" ] && [ "$ALLOW_INSECURE_PASSWORD" != "true" ]; then
+  echo "SECURITY GATE FAILED: production database password is empty or a known default." >&2
+  echo "The emergency backup is valid, but the release remains blocked." >&2
+  exit 4
+fi