Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(release): make deployment backup and rollback immutable

Browse Source
This commit is contained in:
Jens
2026-08-30 06:00:43 +02:00
parent a0884d64c9
commit c272220277
47 changed files with 3035 additions and 430 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/backup_release_state.sh
+163 -48
View File
@@ -3,12 +3,14 @@ set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
CONTAINER="geointel"
OUTPUT_ROOT="backups"
OUTPUT_ROOT="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
RELEASE_ID="rc-$(date -u +%Y%m%dT%H%M%SZ)"
STORAGE_PATH=""
MODELS_PATH=""
INVENTORY_MODE="metadata"
ALLOW_INSECURE_PASSWORD="false"
LINK_DEST_BACKUP=""
ROLLBACK_IMAGE_TAG=""
usage() {
cat <<'EOF'
@@ -19,11 +21,16 @@ GeoIntel container. It never deletes or restores application data.
Options:
--container NAME Docker container (default: geointel)
--output-root PATH Host backup root (default: backups)
--output-root PATH Host backup root (default:
/mnt/user/appdata/geointel/backups)
--release-id ID Safe backup directory name
--storage-path PATH Optional host storage path to inventory
--models-path PATH Optional host model path to inventory
--inventory-mode metadata|sha256 Hash all inventoried files only with sha256
--storage-path PATH Host storage path to snapshot byte-for-byte
--models-path PATH Host model path to snapshot byte-for-byte
--inventory-mode metadata|sha256 Retained manifest compatibility setting
--link-dest-backup PATH Verified older backup used only to hard-link
checksum-identical backup-to-backup files
--rollback-image-tag TAG Immutable backup-specific tag bound to the
running image ID
--allow-insecure-password Complete emergency backup despite an
empty/default production DB password
EOF
@@ -37,6 +44,8 @@ while [ "$#" -gt 0 ]; do
--storage-path) STORAGE_PATH="$2"; shift 2 ;;
--models-path) MODELS_PATH="$2"; shift 2 ;;
--inventory-mode) INVENTORY_MODE="$2"; shift 2 ;;
--link-dest-backup) LINK_DEST_BACKUP="$2"; shift 2 ;;
--rollback-image-tag) ROLLBACK_IMAGE_TAG="$2"; shift 2 ;;
--allow-insecure-password) ALLOW_INSECURE_PASSWORD="true"; shift ;;
--help|-h) usage; exit 0 ;;
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
@@ -51,12 +60,93 @@ if [ "$INVENTORY_MODE" != "metadata" ] && [ "$INVENTORY_MODE" != "sha256" ]; the
echo "--inventory-mode must be metadata or sha256" >&2
exit 2
fi
for required in docker python3 sha256sum git; do
for required in docker python3 sha256sum; do
if ! command -v "$required" >/dev/null 2>&1; then
echo "Missing required command: $required" >&2
exit 2
fi
done
resolve_source_revision() {
local controller_sha="" explicit_sha="${GEOINTEL_BUILD_SHA:-}"
local gitea_sha="${GITEA_COMMIT_SHA:-}" github_sha="${GITHUB_SHA:-}"
local git_head="" git_dirty="false" source=""
if [ -n "$gitea_sha" ]; then
if ! [[ "$gitea_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
echo "GITEA_COMMIT_SHA must contain one full 40-character Git commit SHA." >&2
return 2
fi
controller_sha="${gitea_sha,,}"
source="GITEA_COMMIT_SHA"
fi
if [ -n "$github_sha" ]; then
if ! [[ "$github_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2
return 2
fi
github_sha="${github_sha,,}"
if [ -n "$controller_sha" ] && [ "$controller_sha" != "$github_sha" ]; then
echo "Controller commit variables disagree." >&2
return 2
fi
controller_sha="$github_sha"
source="${source:-GITHUB_SHA}"
fi
if [ -n "$explicit_sha" ]; then
if ! [[ "$explicit_sha" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$ ]]; then
echo "GEOINTEL_BUILD_SHA contains an unsafe release revision." >&2
return 2
fi
explicit_sha="${explicit_sha,,}"
fi
if [ -n "$controller_sha" ]; then
if [ -n "$explicit_sha" ] && [ "$explicit_sha" != "$controller_sha" ]; then
echo "GEOINTEL_BUILD_SHA differs from the controller revision." >&2
return 2
fi
explicit_sha="$controller_sha"
elif [ -n "${GITEA_REPOSITORY:-}${GITHUB_REPOSITORY:-}" ]; then
if ! [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]]; then
echo "Automated backup requires a full controller or GEOINTEL_BUILD_SHA revision." >&2
return 2
fi
source="GEOINTEL_BUILD_SHA"
fi
if command -v git >/dev/null 2>&1 && git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1; then
git_head="$(git -C "$ROOT" rev-parse HEAD 2>/dev/null || true)"
git_head="${git_head,,}"
if ! [[ "$git_head" =~ ^[0-9a-f]{40}$ ]]; then
echo "Could not resolve a full Git revision from the source checkout." >&2
return 2
fi
if [ -n "$explicit_sha" ] && [[ "$explicit_sha" =~ ^[0-9a-f]{40}$ ]] && [ "$git_head" != "$explicit_sha" ]; then
echo "Source checkout does not match the supplied release revision." >&2
return 2
fi
if [ -n "$(git -C "$ROOT" status --porcelain=v1 2>/dev/null)" ]; then
git_dirty="true"
fi
if [ -z "$explicit_sha" ]; then
explicit_sha="$git_head"
source="git"
fi
fi
if [ -z "$explicit_sha" ]; then
echo "Cannot bind backup to a source revision; provide GEOINTEL_BUILD_SHA or a controller SHA." >&2
return 2
fi
SOURCE_REVISION="$explicit_sha"
SOURCE_REVISION_SOURCE="${source:-GEOINTEL_BUILD_SHA}"
SOURCE_GIT_DIRTY="$git_dirty"
}
SOURCE_REVISION=""
SOURCE_REVISION_SOURCE=""
SOURCE_GIT_DIRTY="false"
resolve_source_revision
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null || true)" != "true" ]; then
echo "Container '$CONTAINER' is not running." >&2
exit 3
@@ -72,6 +162,27 @@ if [ -e "$PARTIAL" ] || [ -e "$FINAL" ]; then
fi
mkdir -p "$PARTIAL"
if [ -n "$LINK_DEST_BACKUP" ]; then
LINK_DEST_BACKUP="$(python3 -c 'import pathlib,sys; print(pathlib.Path(sys.argv[1]).expanduser().resolve(strict=True))' "$LINK_DEST_BACKUP")"
python3 - "$OUTPUT_ROOT" "$LINK_DEST_BACKUP" <<'PY'
import pathlib
import sys
root = pathlib.Path(sys.argv[1])
candidate = pathlib.Path(sys.argv[2])
try:
candidate.relative_to(root)
except ValueError as exc:
raise SystemExit(f"Link-dest backup must remain below {root}") from exc
if candidate == root or candidate.name.startswith("."):
raise SystemExit("Link-dest backup must identify one completed immutable backup")
PY
(
cd "$LINK_DEST_BACKUP"
sha256sum -c CHECKSUMS.sha256 >/dev/null
)
fi
cleanup_partial() {
if [ -d "$PARTIAL" ]; then
rm -rf -- "$PARTIAL"
@@ -113,10 +224,21 @@ test -s "$PARTIAL/database.list"
IMAGE_ID="$(docker inspect -f '{{.Image}}' "$CONTAINER")"
IMAGE_NAME="$(docker inspect -f '{{.Config.Image}}' "$CONTAINER")"
GIT_COMMIT="$(git -C "$ROOT" rev-parse HEAD)"
GIT_DIRTY="false"
if [ -n "$(git -C "$ROOT" status --porcelain=v1)" ]; then
GIT_DIRTY="true"
RUNNING_IMAGE_REVISION="$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$CONTAINER")"
RUNNING_IMAGE_REVISION_IS_FULL_SHA="false"
if [[ "$RUNNING_IMAGE_REVISION" =~ ^[0-9A-Fa-f]{40}$ ]]; then
RUNNING_IMAGE_REVISION="${RUNNING_IMAGE_REVISION,,}"
RUNNING_IMAGE_REVISION_IS_FULL_SHA="true"
elif ! [[ "$RUNNING_IMAGE_REVISION" =~ ^[A-Za-z0-9._-]{1,128}$ ]]; then
echo "Running image has an unsafe or missing OCI revision label." >&2
exit 3
fi
if [ -n "$ROLLBACK_IMAGE_TAG" ]; then
TAGGED_IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$ROLLBACK_IMAGE_TAG" 2>/dev/null || true)"
if [ "$TAGGED_IMAGE_ID" != "$IMAGE_ID" ]; then
echo "Backup-specific rollback tag does not resolve to the running image ID." >&2
exit 3
fi
fi
docker exec "$CONTAINER" psql -X -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" -AtF $'\t' \
@@ -132,48 +254,32 @@ for table in projects areas datasets dataset_versions vector_features jobs analy
printf '%s\t%s\n' "$table" "$count" >> "$PARTIAL/table-counts.tsv"
done
inventory_path() {
snapshot_path() {
local source_path="$1"
local output_path="$2"
local label="$2"
local manifest_path="$PARTIAL/${label}-manifest.tsv"
local snapshot_path="$PARTIAL/${label}-snapshot"
local link_args=()
if [ -z "$source_path" ]; then
printf 'not_requested\n' > "$output_path"
printf 'not_requested\n' > "$manifest_path"
return
fi
python3 - "$source_path" "$output_path" "$INVENTORY_MODE" <<'PY'
import hashlib
import os
import pathlib
import sys
root = pathlib.Path(sys.argv[1]).expanduser().resolve()
output = pathlib.Path(sys.argv[2])
mode = sys.argv[3]
if not root.is_dir():
raise SystemExit(f"Inventory root is not a directory: {root}")
def digest(path: pathlib.Path) -> str:
value = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
value.update(chunk)
return value.hexdigest()
with output.open("w", encoding="utf-8", newline="\n") as handle:
handle.write("relative_path\tsize_bytes\tmtime_ns\tsha256\n")
for path in sorted(root.rglob("*"), key=lambda item: item.as_posix()):
if path.is_symlink() or not path.is_file():
continue
stat = path.stat()
checksum = digest(path) if mode == "sha256" else ""
relative = path.relative_to(root).as_posix()
if "\t" in relative or "\n" in relative:
raise SystemExit(f"Unsupported inventory path: {relative!r}")
handle.write(f"{relative}\t{stat.st_size}\t{stat.st_mtime_ns}\t{checksum}\n")
PY
if [ -n "$LINK_DEST_BACKUP" ]; then
link_args=(
--link-dest-snapshot "$LINK_DEST_BACKUP/${label}-snapshot"
--link-dest-manifest "$LINK_DEST_BACKUP/${label}-manifest.tsv"
)
fi
python3 "$ROOT/scripts/release_backup_snapshot.py" create \
--source "$source_path" \
--snapshot "$snapshot_path" \
--manifest "$manifest_path" \
--label "$label" \
"${link_args[@]}"
}
inventory_path "$STORAGE_PATH" "$PARTIAL/storage-manifest.tsv"
inventory_path "$MODELS_PATH" "$PARTIAL/models-manifest.tsv"
snapshot_path "$STORAGE_PATH" storage
snapshot_path "$MODELS_PATH" models
python3 - "$PARTIAL/manifest.json" <<PY
import json
@@ -190,16 +296,25 @@ payload = {
"database_password_secure": ${PASSWORD_SECURE@Q} == "true",
"image_id": ${IMAGE_ID@Q},
"image_name": ${IMAGE_NAME@Q},
"git_commit": ${GIT_COMMIT@Q},
"git_dirty": ${GIT_DIRTY@Q} == "true",
"rollback_image_tag": ${ROLLBACK_IMAGE_TAG@Q} or None,
"backup_tool_revision": ${SOURCE_REVISION@Q},
"backup_tool_revision_source": ${SOURCE_REVISION_SOURCE@Q},
"backup_tool_git_dirty": ${SOURCE_GIT_DIRTY@Q} == "true",
"running_image_revision": ${RUNNING_IMAGE_REVISION@Q},
"running_image_revision_is_full_sha": ${RUNNING_IMAGE_REVISION_IS_FULL_SHA@Q} == "true",
"inventory_mode": ${INVENTORY_MODE@Q},
"storage_inventory_requested": bool(${STORAGE_PATH@Q}),
"models_inventory_requested": bool(${MODELS_PATH@Q}),
"storage_snapshot_requested": bool(${STORAGE_PATH@Q}),
"models_snapshot_requested": bool(${MODELS_PATH@Q}),
"link_dest_backup": ${LINK_DEST_BACKUP@Q} or None,
}
path = pathlib.Path(__import__("sys").argv[1])
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
PY
python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$PARTIAL"
(
cd "$PARTIAL"
find . -maxdepth 1 -type f ! -name CHECKSUMS.sha256 -printf '%f\n' \