584 lines
21 KiB
Bash
584 lines
21 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
GEOINTEL_DEPLOY_LOCK_FILE="${GEOINTEL_DEPLOY_LOCK_FILE:-/tmp/geointel-release-deploy.lock}"
|
|
if ! command -v flock >/dev/null 2>&1; then
|
|
echo "GeoIntel release deployment requires flock to prevent concurrent container replacement." >&2
|
|
exit 2
|
|
fi
|
|
exec 9>"$GEOINTEL_DEPLOY_LOCK_FILE"
|
|
if ! flock -n 9; then
|
|
echo "Another GeoIntel release deployment is already running." >&2
|
|
exit 3
|
|
fi
|
|
|
|
if [ -f .env ]; then
|
|
set -a
|
|
# shellcheck disable=SC1091
|
|
. ./.env
|
|
set +a
|
|
fi
|
|
|
|
if [ -n "${DEPLOY_GEOINTEL_INSTALL_AI:-}" ]; then
|
|
GEOINTEL_INSTALL_AI="$DEPLOY_GEOINTEL_INSTALL_AI"
|
|
fi
|
|
|
|
GEOINTEL_INSTALL_AI="${GEOINTEL_INSTALL_AI:-true}"
|
|
if [ "$GEOINTEL_INSTALL_AI" != "true" ]; then
|
|
echo "Production release deployment requires the gated AI image (GEOINTEL_INSTALL_AI=true)." >&2
|
|
exit 2
|
|
fi
|
|
GEOINTEL_APP_VERSION="$(tr -d '[:space:]' < VERSION)"
|
|
if ! [[ "$GEOINTEL_APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then
|
|
echo "Invalid semantic version in VERSION: ${GEOINTEL_APP_VERSION}" >&2
|
|
exit 2
|
|
fi
|
|
# Hash of everything that actually lands in the image. This is the honest
|
|
# answer to "does this image need rebuilding?" — unlike a git SHA, it changes
|
|
# when working-tree files change without a commit.
|
|
source_tree_hash() {
|
|
local hash=""
|
|
command -v sha1sum >/dev/null 2>&1 || return 1
|
|
hash="$(
|
|
find backend frontend deploy scripts fixtures VERSION \
|
|
-type f \
|
|
! -path '*/node_modules/*' \
|
|
! -path '*/dist/*' \
|
|
! -path '*/__pycache__/*' \
|
|
! -path '*/.pytest_cache/*' \
|
|
! -name '*.pyc' \
|
|
-print0 2>/dev/null \
|
|
| sort -z \
|
|
| xargs -0 sha1sum 2>/dev/null \
|
|
| sha1sum \
|
|
| cut -c1-40
|
|
)" || return 1
|
|
[ -n "$hash" ] || return 1
|
|
printf '%s' "$hash"
|
|
}
|
|
|
|
resolve_build_sha() {
|
|
local head="" content="" controller_sha="" controller_source=""
|
|
|
|
if [ -n "${GITEA_COMMIT_SHA:-}" ]; then
|
|
controller_sha="$GITEA_COMMIT_SHA"
|
|
controller_source="GITEA_COMMIT_SHA"
|
|
fi
|
|
if [ -n "${GITHUB_SHA:-}" ]; then
|
|
if ! [[ "$GITHUB_SHA" =~ ^[0-9A-Fa-f]{40}$ ]]; then
|
|
echo "GITHUB_SHA must contain one full 40-character Git commit SHA." >&2
|
|
return 2
|
|
fi
|
|
if [ -n "$controller_sha" ] && [ "${controller_sha,,}" != "${GITHUB_SHA,,}" ]; then
|
|
echo "Controller commit variables disagree." >&2
|
|
return 2
|
|
fi
|
|
controller_sha="$GITHUB_SHA"
|
|
controller_source="${controller_source:-GITHUB_SHA}"
|
|
fi
|
|
if [ -n "$controller_sha" ]; then
|
|
if ! [[ "$controller_sha" =~ ^[0-9A-Fa-f]{40}$ ]]; then
|
|
echo "${controller_source} must contain one full 40-character Git commit SHA." >&2
|
|
return 2
|
|
fi
|
|
controller_sha="${controller_sha,,}"
|
|
if [ -n "${GEOINTEL_BUILD_SHA:-}" ] && [ "${GEOINTEL_BUILD_SHA,,}" != "$controller_sha" ]; then
|
|
echo "Explicit build revision differs from the controller revision." >&2
|
|
return 2
|
|
fi
|
|
if command -v git >/dev/null 2>&1 && git rev-parse --git-dir >/dev/null 2>&1; then
|
|
head="$(git rev-parse HEAD 2>/dev/null || true)"
|
|
if [ "${head,,}" != "$controller_sha" ]; then
|
|
echo "Prepared Git checkout does not match the controller revision." >&2
|
|
return 2
|
|
fi
|
|
if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
|
|
echo "Prepared Git checkout contains changes outside the controller revision." >&2
|
|
return 2
|
|
fi
|
|
fi
|
|
printf '%s' "$controller_sha"
|
|
return 0
|
|
fi
|
|
if [ -n "${GITEA_REPOSITORY:-}" ] || [ -n "${GITHUB_REPOSITORY:-}" ]; then
|
|
echo "Automated deployment context is missing GITEA_COMMIT_SHA/GITHUB_SHA." >&2
|
|
return 2
|
|
fi
|
|
|
|
# 1. Explicit override wins.
|
|
if [ -n "${GEOINTEL_BUILD_SHA:-}" ]; then
|
|
printf '%s' "$GEOINTEL_BUILD_SHA"
|
|
return 0
|
|
fi
|
|
|
|
# 2. Git checkout, but only when the working tree matches the commit.
|
|
# A manually copied tree often carries .git along while the files on disk
|
|
# have moved on. Trusting HEAD there produces an unchanged image tag, and
|
|
# the deploy silently reuses the previous image instead of rebuilding.
|
|
if command -v git >/dev/null 2>&1 && git rev-parse --git-dir >/dev/null 2>&1; then
|
|
head="$(git rev-parse HEAD 2>/dev/null || true)"
|
|
if [ -n "$head" ]; then
|
|
if [ -z "$(git status --porcelain 2>/dev/null)" ]; then
|
|
printf '%s' "$head"
|
|
return 0
|
|
fi
|
|
echo "Working tree differs from HEAD; tagging this build by content." >&2
|
|
content="$(source_tree_hash || true)"
|
|
if [ -n "$content" ]; then
|
|
printf '%s-wip%s' "${head:0:12}" "${content:0:12}"
|
|
return 0
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# 3. Manually copied tree with a RELEASE_SHA marker file.
|
|
if [ -f RELEASE_SHA ]; then
|
|
tr -d '[:space:]' < RELEASE_SHA
|
|
return 0
|
|
fi
|
|
|
|
# 4. No git: content hash, so an unchanged redeploy still reuses its image.
|
|
content="$(source_tree_hash || true)"
|
|
if [ -n "$content" ]; then
|
|
printf '%s' "$content"
|
|
return 0
|
|
fi
|
|
|
|
# 5. Last resort: unique per deploy.
|
|
printf 'manual%s' "$(date -u +%Y%m%d%H%M%S)"
|
|
}
|
|
|
|
GEOINTEL_BUILD_SHA="$(resolve_build_sha)"
|
|
if [ -z "$GEOINTEL_BUILD_SHA" ]; then
|
|
echo "Could not determine a build revision for this deployment." >&2
|
|
exit 2
|
|
fi
|
|
export GEOINTEL_BUILD_SHA
|
|
echo "Build revision: ${GEOINTEL_BUILD_SHA}"
|
|
GEOINTEL_RELEASE_TOKEN="$(printf '%s' "$GEOINTEL_BUILD_SHA" | tr -c 'A-Za-z0-9._-' '_' | cut -c1-48)"
|
|
if [ -z "$GEOINTEL_RELEASE_TOKEN" ]; then
|
|
echo "Could not derive a safe release evidence identifier." >&2
|
|
exit 2
|
|
fi
|
|
GEOINTEL_BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
GEOINTEL_IMAGE_REPOSITORY="${GEOINTEL_IMAGE_REPOSITORY:-geointel-all-in-one}"
|
|
if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then
|
|
GEOINTEL_RELEASE_VARIANT="ai"
|
|
else
|
|
GEOINTEL_RELEASE_VARIANT="gis"
|
|
fi
|
|
GEOINTEL_RELEASE_IMAGE="${GEOINTEL_IMAGE_REPOSITORY}:${GEOINTEL_BUILD_SHA}-${GEOINTEL_RELEASE_VARIANT}"
|
|
FRONTEND_URL="${FRONTEND_URL:-http://127.0.0.1:${GEOINTEL_FRONTEND_PORT:-1202}}"
|
|
GEOINTEL_BACKUPS_PATH="${GEOINTEL_BACKUPS_PATH:-/mnt/user/appdata/geointel/backups}"
|
|
GEOINTEL_STORAGE_PATH="${GEOINTEL_STORAGE_PATH:-/mnt/user/appdata/geointel/storage}"
|
|
GEOINTEL_MODELS_PATH="${GEOINTEL_MODELS_PATH:-/mnt/user/appdata/geointel/models}"
|
|
GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}"
|
|
GEOINTEL_DEPLOY_EVIDENCE_DIR="${GEOINTEL_DEPLOY_EVIDENCE_DIR:-artifacts/release-evidence/deploy/${GEOINTEL_RELEASE_TOKEN}-ai}"
|
|
GEOINTEL_PREDEPLOY_BACKUP_DIR=""
|
|
GEOINTEL_RELEASE_IMAGE_ID=""
|
|
GEOINTEL_BACKUP_LINK_DEST=""
|
|
GEOINTEL_PREDEPLOY_ROLLBACK_TAG=""
|
|
|
|
case "$GEOINTEL_DEPLOY_EVIDENCE_DIR" in
|
|
/*|*..*)
|
|
echo "Deployment evidence directory must be repository-relative and must not contain '..'." >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
|
|
wait_for_geointel_health() {
|
|
local status=""
|
|
for attempt in $(seq 1 480); do
|
|
status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' geointel 2>/dev/null || true)"
|
|
if [ "$status" = "healthy" ]; then
|
|
echo "GeoIntel container is healthy after attempt ${attempt}."
|
|
return 0
|
|
fi
|
|
if [ "$status" = "unhealthy" ] || [ "$status" = "exited" ] || [ "$status" = "dead" ]; then
|
|
echo "GeoIntel container entered terminal state: ${status}" >&2
|
|
docker logs --tail 120 geointel >&2 || true
|
|
return 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
echo "GeoIntel container did not become healthy (last state: ${status:-missing})." >&2
|
|
docker logs --tail 120 geointel >&2 || true
|
|
return 1
|
|
}
|
|
|
|
start_image() {
|
|
local image="$1"
|
|
local running_image_id=""
|
|
local running_revision=""
|
|
local running_ai=""
|
|
GEOINTEL_IMAGE="$image" bash deploy/unraid/run-dockerman-container.sh
|
|
wait_for_geointel_health
|
|
running_image_id="$(docker inspect --format '{{.Image}}' geointel)"
|
|
if [ "$running_image_id" != "$image" ]; then
|
|
echo "Running container image ${running_image_id} differs from attested image ${image}." >&2
|
|
return 1
|
|
fi
|
|
running_revision="$(docker inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' geointel)"
|
|
running_ai="$(docker inspect --format '{{index .Config.Labels "io.geointel.ai.enabled"}}' geointel)"
|
|
if [ "$running_revision" != "$GEOINTEL_BUILD_SHA" ] || [ "$running_ai" != "true" ]; then
|
|
echo "Running container labels do not match the attested AI revision." >&2
|
|
return 1
|
|
fi
|
|
echo "Running container matches attested image: ${running_image_id}"
|
|
}
|
|
|
|
scan_release_image() {
|
|
local scanned_image_id=""
|
|
local current_image_id=""
|
|
local inspect_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/image-inspect.json"
|
|
local sbom_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-sbom.spdx.json"
|
|
local vulnerability_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/geointel-container-vulnerabilities.json"
|
|
local attestation_output="${GEOINTEL_DEPLOY_EVIDENCE_DIR}/deployment-attestation.json"
|
|
|
|
scanned_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
|
|
test -n "$scanned_image_id"
|
|
mkdir -p "$ROOT/$GEOINTEL_DEPLOY_EVIDENCE_DIR"
|
|
docker image inspect "$GEOINTEL_RELEASE_IMAGE" > "$ROOT/$inspect_output"
|
|
bash scripts/generate_container_sbom.sh "$GEOINTEL_RELEASE_IMAGE" "$sbom_output"
|
|
bash scripts/scan_container_image.sh "$GEOINTEL_RELEASE_IMAGE" "$vulnerability_output"
|
|
current_image_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_RELEASE_IMAGE")"
|
|
if [ "$current_image_id" != "$scanned_image_id" ]; then
|
|
echo "Release image tag changed while SBOM/scan evidence was being generated." >&2
|
|
return 1
|
|
fi
|
|
test -s "$ROOT/$inspect_output"
|
|
test -s "$ROOT/$sbom_output"
|
|
test -s "$ROOT/$vulnerability_output"
|
|
GEOINTEL_RELEASE_IMAGE_ID="$scanned_image_id"
|
|
python3 - \
|
|
"$ROOT/$attestation_output" \
|
|
"$GEOINTEL_RELEASE_IMAGE" \
|
|
"$GEOINTEL_RELEASE_IMAGE_ID" \
|
|
"$GEOINTEL_BUILD_SHA" \
|
|
"$inspect_output" \
|
|
"$sbom_output" \
|
|
"$vulnerability_output" <<'PY'
|
|
import datetime
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
output, image_tag, image_id, revision, inspect_path, sbom_path, vulnerability_path = sys.argv[1:]
|
|
payload = {
|
|
"schema_version": 1,
|
|
"attested_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
"image_tag": image_tag,
|
|
"image_id": image_id,
|
|
"image_config_digest": image_id,
|
|
"revision": revision,
|
|
"variant": "ai",
|
|
"evidence": {
|
|
"image_inspect": inspect_path,
|
|
"sbom": sbom_path,
|
|
"vulnerabilities": vulnerability_path,
|
|
},
|
|
}
|
|
path = pathlib.Path(output)
|
|
temporary = path.with_suffix(".json.partial")
|
|
temporary.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
temporary.replace(path)
|
|
PY
|
|
test -s "$ROOT/$attestation_output"
|
|
echo "Exact deployment image scanned: ${GEOINTEL_RELEASE_IMAGE_ID}"
|
|
}
|
|
|
|
preflight_backup_capacity() {
|
|
local database_name=""
|
|
local database_user=""
|
|
local database_size_bytes=""
|
|
|
|
database_name="$(docker exec geointel sh -c 'printf %s "${POSTGRES_DB:-${GEOINTEL_POSTGRES_DB:-geointel}}"')"
|
|
database_user="$(docker exec geointel sh -c 'printf %s "${POSTGRES_USER:-${GEOINTEL_POSTGRES_USER:-geointel}}"')"
|
|
database_size_bytes="$(docker exec geointel psql -X -v ON_ERROR_STOP=1 \
|
|
-U "$database_user" -d "$database_name" -Atqc \
|
|
'SELECT pg_database_size(current_database());')"
|
|
mkdir -p "$GEOINTEL_BACKUPS_PATH"
|
|
python3 - \
|
|
"$GEOINTEL_BACKUPS_PATH" \
|
|
"$GEOINTEL_STORAGE_PATH" \
|
|
"$GEOINTEL_MODELS_PATH" \
|
|
"$database_size_bytes" <<'PY'
|
|
import os
|
|
import pathlib
|
|
import shutil
|
|
import stat
|
|
import sys
|
|
|
|
backup_root = pathlib.Path(sys.argv[1]).expanduser().resolve()
|
|
sources = [pathlib.Path(value).expanduser().resolve() for value in sys.argv[2:4]]
|
|
database_bytes = int(sys.argv[4])
|
|
|
|
def retained_bytes(root: pathlib.Path) -> int:
|
|
if not root.is_dir():
|
|
raise SystemExit(f"Mandatory snapshot source is not a directory: {root}")
|
|
total = 0
|
|
for current, directories, files in os.walk(root, topdown=True, followlinks=False):
|
|
current_path = pathlib.Path(current)
|
|
for name in [*directories, *files]:
|
|
path = current_path / name
|
|
details = path.lstat()
|
|
if stat.S_ISLNK(details.st_mode):
|
|
raise SystemExit(f"Mandatory snapshot refuses symlinked content: {path}")
|
|
if name in directories and not stat.S_ISDIR(details.st_mode):
|
|
raise SystemExit(f"Snapshot directory changed during capacity preflight: {path}")
|
|
if name in files:
|
|
if not stat.S_ISREG(details.st_mode):
|
|
raise SystemExit(f"Mandatory snapshot refuses non-regular content: {path}")
|
|
total += details.st_size
|
|
return total
|
|
|
|
source_bytes = sum(retained_bytes(source) for source in sources)
|
|
# Reflink clones are used when the backing filesystem supports them. Budget for
|
|
# a complete copy plus two uncompressed database sizes (dump and isolated
|
|
# restore/cutover recovery) so fallback still fails before the live backend is
|
|
# quiesced rather than midway through the snapshot.
|
|
required = source_bytes + (2 * database_bytes)
|
|
headroom = max(5 * 1024**3, required // 10)
|
|
free = shutil.disk_usage(backup_root).free
|
|
if free < required + headroom:
|
|
raise SystemExit(
|
|
"Insufficient free space for a fail-safe predeploy snapshot: "
|
|
f"required={required + headroom} free={free} source={source_bytes} database={database_bytes}"
|
|
)
|
|
print(
|
|
"Predeploy snapshot capacity: "
|
|
f"source_bytes={source_bytes} database_bytes={database_bytes} free_bytes={free}"
|
|
)
|
|
PY
|
|
}
|
|
|
|
select_verified_link_dest() {
|
|
local candidate=""
|
|
GEOINTEL_BACKUP_LINK_DEST=""
|
|
while IFS= read -r candidate; do
|
|
if (
|
|
cd "$candidate" \
|
|
&& sha256sum -c CHECKSUMS.sha256 >/dev/null \
|
|
&& python3 "$ROOT/scripts/release_backup_snapshot.py" verify-backup --backup-dir "$candidate"
|
|
); then
|
|
GEOINTEL_BACKUP_LINK_DEST="$candidate"
|
|
echo "Using verified prior byte snapshot as link-dest: ${candidate}"
|
|
return 0
|
|
fi
|
|
echo "Skipping unusable prior backup link-dest: ${candidate}" >&2
|
|
done < <(
|
|
python3 - "$GEOINTEL_BACKUPS_PATH" <<'PY'
|
|
import pathlib
|
|
import sys
|
|
|
|
root = pathlib.Path(sys.argv[1]).expanduser().resolve()
|
|
candidates = sorted(
|
|
(
|
|
path
|
|
for path in root.iterdir()
|
|
if path.is_dir() and not path.name.startswith(".") and (path / "manifest.json").is_file()
|
|
),
|
|
key=lambda path: path.stat().st_mtime_ns,
|
|
reverse=True,
|
|
)
|
|
for candidate in candidates:
|
|
print(candidate)
|
|
PY
|
|
)
|
|
echo "No verified prior byte snapshot found; this deployment will create a full first snapshot."
|
|
}
|
|
|
|
create_predeploy_backup() {
|
|
local container_exists="false"
|
|
local container_running="false"
|
|
local release_id=""
|
|
local backup_link_args=()
|
|
local current_image_id=""
|
|
local existing_rollback_id=""
|
|
|
|
if docker ps -a --format '{{.Names}}' | grep -Fxq geointel; then
|
|
container_exists="true"
|
|
fi
|
|
if [ "$(docker inspect -f '{{.State.Running}}' geointel 2>/dev/null || true)" = "true" ]; then
|
|
container_running="true"
|
|
fi
|
|
|
|
if [ "$container_exists" = "false" ]; then
|
|
if [ -f "$GEOINTEL_POSTGIS_DATA_PATH/PG_VERSION" ]; then
|
|
echo "PostGIS data exists without a running GeoIntel container; refusing an unbacked migration." >&2
|
|
return 1
|
|
fi
|
|
echo "No existing GeoIntel state found; pre-deploy backup is not required for this fresh install."
|
|
return 0
|
|
fi
|
|
if [ "$container_running" != "true" ]; then
|
|
echo "Existing GeoIntel container is not running; refusing deployment because a consistent backup cannot be created." >&2
|
|
return 1
|
|
fi
|
|
|
|
current_image_id="$(docker inspect --format '{{.Image}}' geointel)"
|
|
if ! [[ "$current_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
|
|
echo "Running release does not expose one immutable Docker image ID." >&2
|
|
return 1
|
|
fi
|
|
release_id="predeploy-${GEOINTEL_RELEASE_TOKEN:0:24}-$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
|
GEOINTEL_PREDEPLOY_ROLLBACK_TAG="${GEOINTEL_IMAGE_REPOSITORY}:rollback-${release_id}"
|
|
existing_rollback_id="$(docker image inspect --format '{{.Id}}' "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" 2>/dev/null || true)"
|
|
if [ -n "$existing_rollback_id" ] && [ "$existing_rollback_id" != "$current_image_id" ]; then
|
|
echo "Backup-specific rollback tag already identifies different image bytes." >&2
|
|
return 1
|
|
fi
|
|
docker tag "$current_image_id" "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG"
|
|
|
|
# This conservative full-copy fallback estimate runs while the existing
|
|
# release is still healthy. Verified backup-to-backup hardlinks normally
|
|
# avoid recopying unchanged bytes, but are never assumed for this fail-closed
|
|
# capacity decision.
|
|
preflight_backup_capacity
|
|
select_verified_link_dest
|
|
if [ -n "$GEOINTEL_BACKUP_LINK_DEST" ]; then
|
|
backup_link_args=(--link-dest-backup "$GEOINTEL_BACKUP_LINK_DEST")
|
|
fi
|
|
|
|
echo "Quiescing the current backend so the rollback point cannot miss concurrent writes..."
|
|
if ! docker exec -i geointel python - <<'PY'
|
|
import os
|
|
import pathlib
|
|
import signal
|
|
import time
|
|
|
|
matches = []
|
|
for item in pathlib.Path("/proc").iterdir():
|
|
if not item.name.isdigit() or int(item.name) in {os.getpid(), os.getppid()}:
|
|
continue
|
|
try:
|
|
command = (item / "cmdline").read_bytes().replace(b"\0", b" ")
|
|
except (OSError, PermissionError):
|
|
continue
|
|
if b"uvicorn" in command and b"app.main:app" in command:
|
|
matches.append(int(item.name))
|
|
if not matches:
|
|
raise SystemExit("Could not identify the running GeoIntel backend")
|
|
for process_id in matches:
|
|
os.kill(process_id, signal.SIGTERM)
|
|
deadline = time.monotonic() + 60
|
|
remaining = matches
|
|
while remaining and time.monotonic() < deadline:
|
|
time.sleep(0.25)
|
|
remaining = [process_id for process_id in remaining if pathlib.Path(f"/proc/{process_id}").exists()]
|
|
if remaining:
|
|
raise SystemExit(f"Backend did not stop cleanly: {remaining}")
|
|
print(f"Stopped {len(matches)} backend process(es)")
|
|
PY
|
|
then
|
|
echo "Could not quiesce the current backend; refusing a potentially inconsistent backup." >&2
|
|
docker restart geointel >/dev/null || true
|
|
wait_for_geointel_health || true
|
|
return 1
|
|
fi
|
|
|
|
GEOINTEL_PREDEPLOY_BACKUP_DIR="${GEOINTEL_BACKUPS_PATH%/}/${release_id}"
|
|
echo "Creating mandatory pre-deploy backup ${release_id}..."
|
|
if ! bash scripts/backup_release_state.sh \
|
|
--container geointel \
|
|
--output-root "$GEOINTEL_BACKUPS_PATH" \
|
|
--release-id "$release_id" \
|
|
--storage-path "$GEOINTEL_STORAGE_PATH" \
|
|
--models-path "$GEOINTEL_MODELS_PATH" \
|
|
--inventory-mode sha256 \
|
|
--rollback-image-tag "$GEOINTEL_PREDEPLOY_ROLLBACK_TAG" \
|
|
"${backup_link_args[@]}" \
|
|
|| ! bash scripts/verify_release_backup.sh \
|
|
--container geointel \
|
|
--backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR"; then
|
|
echo "Pre-deploy backup failed; restarting the unchanged current release." >&2
|
|
docker restart geointel >/dev/null || true
|
|
wait_for_geointel_health || true
|
|
GEOINTEL_PREDEPLOY_BACKUP_DIR=""
|
|
return 1
|
|
fi
|
|
echo "Pre-deploy backup verified: ${GEOINTEL_PREDEPLOY_BACKUP_DIR}"
|
|
}
|
|
|
|
rollback_previous() {
|
|
if [ -z "$GEOINTEL_PREDEPLOY_BACKUP_DIR" ]; then
|
|
echo "Automatic rollback unavailable: no verified pre-deploy database backup was created." >&2
|
|
return 1
|
|
fi
|
|
echo "Rolling back database and image to the verified pre-deploy state..."
|
|
GEOINTEL_DEPLOY_LOCK_HELD=true \
|
|
bash deploy/unraid/rollback-dockerman-container.sh \
|
|
--backup-dir "$GEOINTEL_PREDEPLOY_BACKUP_DIR" \
|
|
--confirm-production-database-restore
|
|
}
|
|
|
|
docker compose -f docker-compose.unraid.yml config >/dev/null
|
|
|
|
if docker image inspect "$GEOINTEL_RELEASE_IMAGE" >/dev/null 2>&1; then
|
|
stored_revision="$(
|
|
docker image inspect \
|
|
--format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \
|
|
"$GEOINTEL_RELEASE_IMAGE"
|
|
)"
|
|
stored_ai="$(
|
|
docker image inspect \
|
|
--format '{{index .Config.Labels "io.geointel.ai.enabled"}}' \
|
|
"$GEOINTEL_RELEASE_IMAGE"
|
|
)"
|
|
stored_version="$(
|
|
docker image inspect \
|
|
--format '{{index .Config.Labels "org.opencontainers.image.version"}}' \
|
|
"$GEOINTEL_RELEASE_IMAGE"
|
|
)"
|
|
if (
|
|
[ "$stored_revision" != "$GEOINTEL_BUILD_SHA" ] ||
|
|
[ "$stored_ai" != "$GEOINTEL_INSTALL_AI" ] ||
|
|
[ "$stored_version" != "$GEOINTEL_APP_VERSION" ]
|
|
); then
|
|
echo "Immutable release tag has conflicting metadata: ${GEOINTEL_RELEASE_IMAGE}" >&2
|
|
exit 2
|
|
fi
|
|
echo "Reusing existing immutable image ${GEOINTEL_RELEASE_IMAGE}."
|
|
docker tag "$GEOINTEL_RELEASE_IMAGE" "${GEOINTEL_IMAGE_REPOSITORY}:latest"
|
|
else
|
|
docker build \
|
|
--build-arg GEOINTEL_INSTALL_AI="$GEOINTEL_INSTALL_AI" \
|
|
--build-arg GEOINTEL_BUILD_SHA="$GEOINTEL_BUILD_SHA" \
|
|
--build-arg GEOINTEL_BUILD_TIME="$GEOINTEL_BUILD_TIME" \
|
|
--build-arg GEOINTEL_APP_VERSION="$GEOINTEL_APP_VERSION" \
|
|
-f deploy/unraid/Dockerfile.all-in-one \
|
|
-t "$GEOINTEL_RELEASE_IMAGE" \
|
|
-t "${GEOINTEL_IMAGE_REPOSITORY}:latest" \
|
|
.
|
|
fi
|
|
|
|
scan_release_image
|
|
create_predeploy_backup
|
|
|
|
if ! start_image "$GEOINTEL_RELEASE_IMAGE_ID"; then
|
|
rollback_previous || true
|
|
exit 1
|
|
fi
|
|
|
|
if [ -f scripts/live_migration_smoke.sh ]; then
|
|
if ! LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh; then
|
|
rollback_previous || true
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [ -f scripts/verify_browser_runtime.sh ]; then
|
|
if ! bash scripts/verify_browser_runtime.sh "$FRONTEND_URL"; then
|
|
rollback_previous || true
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
echo "Deployed immutable image ${GEOINTEL_RELEASE_IMAGE}."
|
|
docker image inspect \
|
|
--format 'version={{index .Config.Labels "org.opencontainers.image.version"}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} created={{index .Config.Labels "org.opencontainers.image.created"}}' \
|
|
"$GEOINTEL_RELEASE_IMAGE"
|