Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
98 lines
3.2 KiB
Bash
98 lines
3.2 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
GEOINTEL_DEPLOY_LOCK_FILE="${GEOINTEL_DEPLOY_LOCK_FILE:-/tmp/geointel-release-deploy.lock}"
|
|
if [ "${GEOINTEL_DEPLOY_LOCK_HELD:-false}" != "true" ]; then
|
|
command -v flock >/dev/null 2>&1 || {
|
|
echo "GeoIntel rollback requires flock to prevent concurrent deployment." >&2
|
|
exit 2
|
|
}
|
|
exec 9>"$GEOINTEL_DEPLOY_LOCK_FILE"
|
|
if ! flock -n 9; then
|
|
echo "Another GeoIntel deployment or rollback is already running." >&2
|
|
exit 3
|
|
fi
|
|
GEOINTEL_DEPLOY_LOCK_HELD="true"
|
|
export GEOINTEL_DEPLOY_LOCK_HELD
|
|
fi
|
|
|
|
GEOINTEL_ROLLBACK_IMAGE="${GEOINTEL_ROLLBACK_IMAGE:-}"
|
|
BACKUP_DIR="${GEOINTEL_ROLLBACK_BACKUP_DIR:-}"
|
|
CONFIRM_RESTORE="false"
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage: bash deploy/unraid/rollback-dockerman-container.sh \
|
|
--backup-dir PATH --confirm-production-database-restore
|
|
|
|
Restores the verified pre-deploy PostgreSQL dump first and only then starts the
|
|
retained previous image. Image-only rollback against an unknown migrated
|
|
schema is deliberately not supported.
|
|
EOF
|
|
}
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--backup-dir) BACKUP_DIR="$2"; shift 2 ;;
|
|
--confirm-production-database-restore) CONFIRM_RESTORE="true"; shift ;;
|
|
--help|-h) usage; exit 0 ;;
|
|
*) echo "Unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
if [ -z "$BACKUP_DIR" ] || [ "$CONFIRM_RESTORE" != "true" ]; then
|
|
echo "Rollback requires a verified pre-deploy backup and explicit database-restore confirmation." >&2
|
|
usage >&2
|
|
exit 2
|
|
fi
|
|
|
|
restore_image_args=()
|
|
if [ -n "$GEOINTEL_ROLLBACK_IMAGE" ]; then
|
|
restore_image_args=(--image "$GEOINTEL_ROLLBACK_IMAGE")
|
|
fi
|
|
echo "Restoring the pre-deploy database before starting its checksum-bound image..."
|
|
bash deploy/unraid/restore-predeploy-database.sh \
|
|
--backup-dir "$BACKUP_DIR" \
|
|
"${restore_image_args[@]}" \
|
|
--confirm-production-database-restore
|
|
|
|
if [ -z "$GEOINTEL_ROLLBACK_IMAGE" ]; then
|
|
GEOINTEL_ROLLBACK_IMAGE="$(python3 - "$BACKUP_DIR/manifest.json" <<'PY'
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
|
|
image_id = json.loads(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")).get("image_id", "")
|
|
if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id):
|
|
raise SystemExit("Backup manifest lacks an immutable rollback image ID")
|
|
print(image_id)
|
|
PY
|
|
)"
|
|
fi
|
|
docker image inspect "$GEOINTEL_ROLLBACK_IMAGE" >/dev/null
|
|
|
|
echo "Starting rollback image ${GEOINTEL_ROLLBACK_IMAGE} with the restored persistent database..."
|
|
GEOINTEL_IMAGE="$GEOINTEL_ROLLBACK_IMAGE" bash deploy/unraid/run-dockerman-container.sh
|
|
|
|
for attempt in $(seq 1 90); do
|
|
status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' geointel 2>/dev/null || true)"
|
|
if [ "$status" = "healthy" ]; then
|
|
LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh
|
|
echo "Rollback completed with healthy image ${GEOINTEL_ROLLBACK_IMAGE}."
|
|
exit 0
|
|
fi
|
|
if [ "$status" = "unhealthy" ] || [ "$status" = "exited" ] || [ "$status" = "dead" ]; then
|
|
docker logs --tail 120 geointel >&2 || true
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
echo "Rollback container did not become healthy." >&2
|
|
docker logs --tail 120 geointel >&2 || true
|
|
exit 1
|