Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
350 lines
10 KiB
Bash
350 lines
10 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
PYTHON_BIN="${PYTHON_BIN:-}"
|
|
LIVE_SMOKE_USE_DOCKER="${LIVE_SMOKE_USE_DOCKER:-auto}"
|
|
COMPOSE_FILE_PATH="${COMPOSE_FILE:-docker-compose.yml}"
|
|
LIVE_SMOKE_CONTAINER="${LIVE_SMOKE_CONTAINER:-}"
|
|
|
|
run_docker_smoke() {
|
|
service_name="$1"
|
|
cd "$ROOT"
|
|
|
|
echo "== GeoIntel live migration smoke =="
|
|
echo "Using Docker Compose service: ${service_name}."
|
|
|
|
docker compose -f "$COMPOSE_FILE_PATH" exec -T "$service_name" sh <<'CONTAINER_SCRIPT'
|
|
set -eu
|
|
cd /app
|
|
|
|
echo "Database URL is read from backend container settings / DATABASE_URL."
|
|
|
|
python - <<'PY'
|
|
import time
|
|
|
|
from sqlalchemy import text
|
|
from sqlalchemy.exc import OperationalError
|
|
|
|
from app.db.session import get_engine
|
|
|
|
last_error = None
|
|
for attempt in range(1, 61):
|
|
try:
|
|
with get_engine().connect() as connection:
|
|
connection.execute(text("SELECT 1"))
|
|
print(f"Database connection: ok after attempt {attempt}")
|
|
break
|
|
except OperationalError as exc:
|
|
last_error = exc
|
|
print(f"Database not ready yet ({attempt}/60): {exc}")
|
|
time.sleep(2)
|
|
else:
|
|
raise SystemExit(f"Database did not become ready: {last_error}")
|
|
PY
|
|
|
|
python -m alembic upgrade head
|
|
|
|
python - <<'PY'
|
|
from sqlalchemy import text
|
|
|
|
from app.db.session import get_engine
|
|
|
|
required_objects = [
|
|
"public.projects",
|
|
"public.areas",
|
|
"public.datasets",
|
|
"public.jobs",
|
|
"public.vector_features",
|
|
"public.quality_checks",
|
|
"public.metrics",
|
|
"public.analysis_runs",
|
|
"public.detections",
|
|
"public.segmentations",
|
|
"public.ix_vector_features_geometry",
|
|
"public.ix_detections_geometry",
|
|
"public.ix_segmentations_geometry",
|
|
]
|
|
|
|
with get_engine().connect() as connection:
|
|
postgis_version = connection.execute(text("SELECT PostGIS_Version()")).scalar()
|
|
print(f"PostGIS version: {postgis_version}")
|
|
collation_row = connection.execute(
|
|
text(
|
|
"SELECT datname, datcollversion, pg_database_collation_actual_version(oid) AS actual_version "
|
|
"FROM pg_database WHERE datname = current_database()"
|
|
)
|
|
).mappings().one()
|
|
stored_version = collation_row["datcollversion"]
|
|
actual_version = collation_row["actual_version"]
|
|
if stored_version != actual_version:
|
|
database_name = str(collation_row["datname"]).replace('"', '""')
|
|
print(
|
|
"COLLATION_VERSION_MISMATCH: "
|
|
f"database={collation_row['datname']} stored={stored_version} actual={actual_version}"
|
|
)
|
|
print(
|
|
"After reviewing backups and indexes, acknowledge with: "
|
|
f'ALTER DATABASE "{database_name}" REFRESH COLLATION VERSION;'
|
|
)
|
|
else:
|
|
print("Database collation version: ok")
|
|
|
|
missing = [
|
|
object_name
|
|
for object_name in required_objects
|
|
if connection.execute(text("SELECT to_regclass(:object_name)"), {"object_name": object_name}).scalar() is None
|
|
]
|
|
if missing:
|
|
raise SystemExit(f"Missing expected migrated schema objects: {', '.join(missing)}")
|
|
print("Required runtime schema objects: ok")
|
|
PY
|
|
|
|
HEAD_COUNT="$(python -m alembic heads | grep -c 'head')"
|
|
if [ "$HEAD_COUNT" -ne 1 ]; then
|
|
echo "Expected exactly one Alembic head, found $HEAD_COUNT" >&2
|
|
python -m alembic heads >&2
|
|
exit 1
|
|
fi
|
|
|
|
python -m alembic heads
|
|
echo "== Live migration smoke passed =="
|
|
CONTAINER_SCRIPT
|
|
}
|
|
|
|
run_container_smoke() {
|
|
container_name="$1"
|
|
cd "$ROOT"
|
|
|
|
echo "== GeoIntel live migration smoke =="
|
|
echo "Using Docker container: ${container_name}."
|
|
|
|
docker exec -i "$container_name" sh <<'CONTAINER_SCRIPT'
|
|
set -eu
|
|
cd /app
|
|
|
|
echo "Database URL is read from backend container settings / DATABASE_URL."
|
|
|
|
python - <<'PY'
|
|
import time
|
|
|
|
from sqlalchemy import text
|
|
from sqlalchemy.exc import OperationalError
|
|
|
|
from app.db.session import get_engine
|
|
|
|
last_error = None
|
|
for attempt in range(1, 61):
|
|
try:
|
|
with get_engine().connect() as connection:
|
|
connection.execute(text("SELECT 1"))
|
|
print(f"Database connection: ok after attempt {attempt}")
|
|
break
|
|
except OperationalError as exc:
|
|
last_error = exc
|
|
print(f"Database not ready yet ({attempt}/60): {exc}")
|
|
time.sleep(2)
|
|
else:
|
|
raise SystemExit(f"Database did not become ready: {last_error}")
|
|
PY
|
|
|
|
python -m alembic upgrade head
|
|
|
|
python - <<'PY'
|
|
from sqlalchemy import text
|
|
|
|
from app.db.session import get_engine
|
|
|
|
required_objects = [
|
|
"public.projects",
|
|
"public.areas",
|
|
"public.datasets",
|
|
"public.jobs",
|
|
"public.vector_features",
|
|
"public.quality_checks",
|
|
"public.metrics",
|
|
"public.analysis_runs",
|
|
"public.detections",
|
|
"public.segmentations",
|
|
"public.ix_vector_features_geometry",
|
|
"public.ix_detections_geometry",
|
|
"public.ix_segmentations_geometry",
|
|
]
|
|
|
|
with get_engine().connect() as connection:
|
|
postgis_version = connection.execute(text("SELECT PostGIS_Version()")).scalar()
|
|
print(f"PostGIS version: {postgis_version}")
|
|
collation_row = connection.execute(
|
|
text(
|
|
"SELECT datname, datcollversion, pg_database_collation_actual_version(oid) AS actual_version "
|
|
"FROM pg_database WHERE datname = current_database()"
|
|
)
|
|
).mappings().one()
|
|
stored_version = collation_row["datcollversion"]
|
|
actual_version = collation_row["actual_version"]
|
|
if stored_version != actual_version:
|
|
database_name = str(collation_row["datname"]).replace('"', '""')
|
|
print(
|
|
"COLLATION_VERSION_MISMATCH: "
|
|
f"database={collation_row['datname']} stored={stored_version} actual={actual_version}"
|
|
)
|
|
print(
|
|
"After reviewing backups and indexes, acknowledge with: "
|
|
f'ALTER DATABASE "{database_name}" REFRESH COLLATION VERSION;'
|
|
)
|
|
else:
|
|
print("Database collation version: ok")
|
|
|
|
missing = [
|
|
object_name
|
|
for object_name in required_objects
|
|
if connection.execute(text("SELECT to_regclass(:object_name)"), {"object_name": object_name}).scalar() is None
|
|
]
|
|
if missing:
|
|
raise SystemExit(f"Missing expected migrated schema objects: {', '.join(missing)}")
|
|
print("Required runtime schema objects: ok")
|
|
PY
|
|
|
|
HEAD_COUNT="$(python -m alembic heads | grep -c 'head')"
|
|
if [ "$HEAD_COUNT" -ne 1 ]; then
|
|
echo "Expected exactly one Alembic head, found $HEAD_COUNT" >&2
|
|
python -m alembic heads >&2
|
|
exit 1
|
|
fi
|
|
|
|
python -m alembic heads
|
|
echo "== Live migration smoke passed =="
|
|
CONTAINER_SCRIPT
|
|
}
|
|
|
|
if [ -n "$LIVE_SMOKE_CONTAINER" ] && command -v docker >/dev/null 2>&1; then
|
|
if docker ps --format '{{.Names}}' | grep -qx "$LIVE_SMOKE_CONTAINER"; then
|
|
run_container_smoke "$LIVE_SMOKE_CONTAINER"
|
|
exit 0
|
|
fi
|
|
fi
|
|
|
|
if [ "$LIVE_SMOKE_USE_DOCKER" != "0" ] && command -v docker >/dev/null 2>&1 && [ -f "$ROOT/$COMPOSE_FILE_PATH" ]; then
|
|
cd "$ROOT"
|
|
for service_name in geointel backend; do
|
|
if docker compose -f "$COMPOSE_FILE_PATH" ps -q "$service_name" >/dev/null 2>&1 \
|
|
&& [ -n "$(docker compose -f "$COMPOSE_FILE_PATH" ps -q "$service_name" 2>/dev/null)" ]; then
|
|
if [ "$LIVE_SMOKE_USE_DOCKER" = "1" ] \
|
|
|| docker compose -f "$COMPOSE_FILE_PATH" ps --status running "$service_name" | grep -q "$service_name"; then
|
|
run_docker_smoke "$service_name"
|
|
exit 0
|
|
fi
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [ -z "$PYTHON_BIN" ]; then
|
|
for candidate in python3 python.exe python; do
|
|
if command -v "$candidate" >/dev/null 2>&1 && "$candidate" -c "import sys" >/dev/null 2>&1; then
|
|
PYTHON_BIN="$candidate"
|
|
break
|
|
fi
|
|
done
|
|
fi
|
|
|
|
if [ -z "$PYTHON_BIN" ]; then
|
|
echo "No usable Python interpreter found. Set PYTHON_BIN=/path/to/python and retry." >&2
|
|
exit 1
|
|
fi
|
|
|
|
cd "$ROOT/backend"
|
|
|
|
echo "== GeoIntel live migration smoke =="
|
|
echo "Using Python: $PYTHON_BIN"
|
|
echo "Database URL is read from backend settings / DATABASE_URL."
|
|
|
|
"$PYTHON_BIN" - <<'PY'
|
|
import time
|
|
|
|
from sqlalchemy import text
|
|
from sqlalchemy.exc import OperationalError
|
|
|
|
from app.db.session import get_engine
|
|
|
|
last_error = None
|
|
for attempt in range(1, 61):
|
|
try:
|
|
with get_engine().connect() as connection:
|
|
connection.execute(text("SELECT 1"))
|
|
print(f"Database connection: ok after attempt {attempt}")
|
|
break
|
|
except OperationalError as exc:
|
|
last_error = exc
|
|
print(f"Database not ready yet ({attempt}/60): {exc}")
|
|
time.sleep(2)
|
|
else:
|
|
raise SystemExit(f"Database did not become ready: {last_error}")
|
|
PY
|
|
|
|
"$PYTHON_BIN" -m alembic upgrade head
|
|
|
|
"$PYTHON_BIN" - <<'PY'
|
|
from sqlalchemy import text
|
|
|
|
from app.db.session import get_engine
|
|
|
|
required_objects = [
|
|
"public.projects",
|
|
"public.areas",
|
|
"public.datasets",
|
|
"public.jobs",
|
|
"public.vector_features",
|
|
"public.quality_checks",
|
|
"public.metrics",
|
|
"public.analysis_runs",
|
|
"public.detections",
|
|
"public.segmentations",
|
|
"public.ix_vector_features_geometry",
|
|
"public.ix_detections_geometry",
|
|
"public.ix_segmentations_geometry",
|
|
]
|
|
|
|
with get_engine().connect() as connection:
|
|
postgis_version = connection.execute(text("SELECT PostGIS_Version()")).scalar()
|
|
print(f"PostGIS version: {postgis_version}")
|
|
collation_row = connection.execute(
|
|
text(
|
|
"SELECT datname, datcollversion, pg_database_collation_actual_version(oid) AS actual_version "
|
|
"FROM pg_database WHERE datname = current_database()"
|
|
)
|
|
).mappings().one()
|
|
stored_version = collation_row["datcollversion"]
|
|
actual_version = collation_row["actual_version"]
|
|
if stored_version != actual_version:
|
|
database_name = str(collation_row["datname"]).replace('"', '""')
|
|
print(
|
|
"COLLATION_VERSION_MISMATCH: "
|
|
f"database={collation_row['datname']} stored={stored_version} actual={actual_version}"
|
|
)
|
|
print(
|
|
"After reviewing backups and indexes, acknowledge with: "
|
|
f'ALTER DATABASE "{database_name}" REFRESH COLLATION VERSION;'
|
|
)
|
|
else:
|
|
print("Database collation version: ok")
|
|
|
|
missing = [
|
|
object_name
|
|
for object_name in required_objects
|
|
if connection.execute(text("SELECT to_regclass(:object_name)"), {"object_name": object_name}).scalar() is None
|
|
]
|
|
if missing:
|
|
raise SystemExit(f"Missing expected migrated schema objects: {', '.join(missing)}")
|
|
print("Required runtime schema objects: ok")
|
|
PY
|
|
|
|
HEAD_COUNT="$("$PYTHON_BIN" -m alembic heads | grep -c 'head')"
|
|
if [ "$HEAD_COUNT" -ne 1 ]; then
|
|
echo "Expected exactly one Alembic head, found $HEAD_COUNT" >&2
|
|
"$PYTHON_BIN" -m alembic heads >&2
|
|
exit 1
|
|
fi
|
|
|
|
"$PYTHON_BIN" -m alembic heads
|
|
echo "== Live migration smoke passed =="
|