diff --git a/CHANGELOG.md b/CHANGELOG.md index 1af010f5..ebe52b4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ - Added `deploy/unraid/nginx-all-in-one.conf` with localhost backend proxying inside the same container. - Added a PNG DockerMan icon and made the Unraid template name match the running `geointel` container. - Added DockerMan labels to the all-in-one Compose service so Unraid can associate the running container with web UI and icon metadata. +- Added `deploy/unraid/run-dockerman-container.sh` so repo deploys automatically replace Compose-owned containers with a DockerMan-native `geointel` container while preserving/migrating persisted data. - Updated Tower deploy scripts to install `/boot/config/plugins/dockerMan/templates-user/my-geointel.xml` and `/boot/config/plugins/dockerMan/images/geointel-icon.png`. - Updated Tower deploy scripts to stop the old multi-container stack without removing volumes and start the all-in-one stack. - Updated the Unraid template so the Docker can be edited from Unraid with one web port, storage path, PostGIS data path and app icon. diff --git a/README.md b/README.md index c8622c05..9058e000 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ make readiness ## Unraid / Tower deployment -GeoIntel runs on Unraid as an all-in-one Docker container. The container embeds PostGIS, runs the FastAPI backend internally, and serves the frontend through nginx on one editable web port. +GeoIntel runs on Unraid as an all-in-one DockerMan-native container. The container embeds PostGIS, runs the FastAPI backend internally, and serves the frontend through nginx on one editable web port. Unraid template assets live in: @@ -85,7 +85,8 @@ Copy the Unraid env template to `.env` in the checkout and edit ports/paths ther cd /mnt/user/appdata/geointel cp deploy/unraid/geointel.env.example .env nano .env -docker compose -f docker-compose.unraid.yml up -d --build +docker compose -f docker-compose.unraid.yml build geointel +bash deploy/unraid/run-dockerman-container.sh ``` Common editable values: @@ -98,7 +99,7 @@ GEOINTEL_POSTGIS_DATA_PATH=/mnt/user/appdata/geointel/postgres-data The backend and PostGIS ports are intentionally not exposed to the LAN in the all-in-one runtime. See `deploy/unraid/README.md` for full setup, port-change and cleanup notes. -On Tower/Unraid, `scripts/deploy_tower.ps1` and `scripts/deploy_tower.sh` also install the editable DockerMan template as `/boot/config/plugins/dockerMan/templates-user/my-geointel.xml` and the PNG icon as `/boot/config/plugins/dockerMan/images/geointel-icon.png`. +On Tower/Unraid, `scripts/deploy_tower.ps1` and `scripts/deploy_tower.sh` automatically install the editable DockerMan template as `/boot/config/plugins/dockerMan/templates-user/my-geointel.xml`, install the PNG icon as `/boot/config/plugins/dockerMan/images/geointel-icon.png`, remove any old Compose-owned `geointel` container and start the final container with DockerMan labels. ## Sprint 2 quick start diff --git a/backend/tests/test_sprint31_unraid_template.py b/backend/tests/test_sprint31_unraid_template.py index 84d2a53b..c66d42f6 100644 --- a/backend/tests/test_sprint31_unraid_template.py +++ b/backend/tests/test_sprint31_unraid_template.py @@ -58,7 +58,9 @@ def test_unraid_readme_explains_port_changes_and_safe_cleanup() -> None: assert "cp deploy/unraid/geointel.env.example .env" in readme assert "GEOINTEL_FRONTEND_PORT=1203" in readme - assert "docker compose -f docker-compose.unraid.yml up -d --build" in readme + assert "docker compose -f docker-compose.unraid.yml build geointel" in readme + assert "bash deploy/unraid/run-dockerman-container.sh" in readme + assert "net.unraid.docker.managed=dockerman" in readme assert "curl -fsS" in readme assert "docker builder prune -af" in readme assert "Avoid broad volume pruning" in readme @@ -67,6 +69,7 @@ def test_unraid_readme_explains_port_changes_and_safe_cleanup() -> None: def test_unraid_all_in_one_runtime_starts_embedded_postgis_backend_and_nginx() -> None: dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") start_script = (ROOT / "deploy" / "unraid" / "all-in-one-start.sh").read_text(encoding="utf-8") + dockerman_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8") nginx_config = (ROOT / "deploy" / "unraid" / "nginx-all-in-one.conf").read_text(encoding="utf-8") dockerignore = (ROOT / ".dockerignore").read_text(encoding="utf-8") @@ -85,6 +88,10 @@ def test_unraid_all_in_one_runtime_starts_embedded_postgis_backend_and_nginx() - assert "python -m alembic upgrade head" in start_script assert "uvicorn app.main:app --host 127.0.0.1 --port 8000 &" in start_script assert 'exec nginx -g "daemon off;"' in start_script + assert "docker run -d" in dockerman_script + assert "--label net.unraid.docker.managed=dockerman" in dockerman_script + assert "migrate_compose_volume_if_needed" in dockerman_script + assert "docker rm -f geointel" in dockerman_script assert "proxy_pass http://127.0.0.1:8000/api/" in nginx_config assert "proxy_pass http://127.0.0.1:8000/health" in nginx_config assert "location = /geointel-icon.png" in nginx_config @@ -98,13 +105,18 @@ def test_tower_deploy_uses_single_container_unraid_compose() -> None: bash = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8") for script in (powershell, bash): - assert "/boot/config/plugins/dockerMan/templates-user/my-geointel.xml" in script - assert "/boot/config/plugins/dockerMan/images/geointel-icon.png" in script - assert "docker compose down --remove-orphans || true" in script assert "docker compose -f docker-compose.unraid.yml config" in script assert "docker compose -f docker-compose.unraid.yml build geointel" in script - assert "docker compose -f docker-compose.unraid.yml up -d" in script - assert "COMPOSE_FILE=docker-compose.unraid.yml bash scripts/live_migration_smoke.sh" in script + assert "bash deploy/unraid/run-dockerman-container.sh" in script + assert "LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh" in script + + +def test_live_migration_smoke_supports_dockerman_native_container() -> None: + script = (ROOT / "scripts" / "live_migration_smoke.sh").read_text(encoding="utf-8") + + assert 'LIVE_SMOKE_CONTAINER="${LIVE_SMOKE_CONTAINER:-}"' in script + assert "run_container_smoke()" in script + assert 'docker exec -i "$container_name" sh' in script def test_frontend_and_unraid_icon_assets_are_present() -> None: diff --git a/deploy/unraid/README.md b/deploy/unraid/README.md index 1e36e52b..2213fc75 100644 --- a/deploy/unraid/README.md +++ b/deploy/unraid/README.md @@ -1,6 +1,6 @@ # GeoIntel Unraid all-in-one container -GeoIntel can run on Unraid as one Docker container. +GeoIntel can run on Unraid as one DockerMan-native container. Inside that single container: @@ -25,9 +25,10 @@ http://:${GEOINTEL_FRONTEND_PORT}/geointel-icon.png ## Files -- `docker-compose.unraid.yml`: recommended single-container Compose stack. +- `docker-compose.unraid.yml`: build definition for the all-in-one image. - `deploy/unraid/Dockerfile.all-in-one`: builds the single container. - `deploy/unraid/all-in-one-start.sh`: starts embedded PostGIS, backend and nginx. +- `deploy/unraid/run-dockerman-container.sh`: starts/replaces the running container with DockerMan labels and editable Unraid metadata. - `deploy/unraid/nginx-all-in-one.conf`: frontend and API proxy config for one container. - `deploy/unraid/geointel.env.example`: copy to `.env` and edit ports/paths. - `deploy/unraid/geointel-unraid-template.xml`: Unraid/DockerMan metadata for editable fields. @@ -60,7 +61,7 @@ net.unraid.docker.icon=/boot/config/plugins/dockerMan/images/geointel-icon.png These labels are required because a plain Compose container can run correctly while still missing the normal Unraid edit/icon controls. -## First setup with Compose Manager +## First setup from the repo From the Unraid shell: @@ -71,9 +72,12 @@ cd /mnt/user/appdata/geointel cp deploy/unraid/geointel.env.example .env nano .env docker compose -f docker-compose.unraid.yml config -docker compose -f docker-compose.unraid.yml up -d --build +docker compose -f docker-compose.unraid.yml build geointel +bash deploy/unraid/run-dockerman-container.sh ``` +The repository deploy scripts run the same flow automatically. They build the image, install the DockerMan template/icon, remove any old Compose-owned `geointel` container, preserve/migrate the PostGIS data path and start the final container with DockerMan labels. + Validate: ```bash @@ -95,7 +99,8 @@ GEOINTEL_CORS_ORIGINS=http://localhost:1203,http://127.0.0.1:1203,http://192.168 Apply: ```bash -docker compose -f docker-compose.unraid.yml up -d --build +docker compose -f docker-compose.unraid.yml build geointel +bash deploy/unraid/run-dockerman-container.sh ``` ## Persistent paths @@ -117,7 +122,8 @@ GEOINTEL_POSTGIS_DATA_PATH=/mnt/user/appdata/geointel/postgres-data cd /mnt/user/appdata/geointel git fetch origin main git reset --hard origin/main -docker compose -f docker-compose.unraid.yml up -d --build +docker compose -f docker-compose.unraid.yml build geointel +bash deploy/unraid/run-dockerman-container.sh ``` ## Safe cleanup diff --git a/deploy/unraid/run-dockerman-container.sh b/deploy/unraid/run-dockerman-container.sh new file mode 100644 index 00000000..d94bf024 --- /dev/null +++ b/deploy/unraid/run-dockerman-container.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT" + +if [ -f .env ]; then + set -a + # shellcheck disable=SC1091 + . ./.env + set +a +fi + +GEOINTEL_FRONTEND_PORT="${GEOINTEL_FRONTEND_PORT:-1202}" +GEOINTEL_STORAGE_PATH="${GEOINTEL_STORAGE_PATH:-/mnt/user/appdata/geointel/storage}" +GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}" +GEOINTEL_POSTGRES_DB="${GEOINTEL_POSTGRES_DB:-geointel}" +GEOINTEL_POSTGRES_USER="${GEOINTEL_POSTGRES_USER:-geointel}" +GEOINTEL_POSTGRES_PASSWORD="${GEOINTEL_POSTGRES_PASSWORD:-geointel}" +GEOINTEL_CORS_ORIGINS="${GEOINTEL_CORS_ORIGINS:-http://localhost:${GEOINTEL_FRONTEND_PORT},http://127.0.0.1:${GEOINTEL_FRONTEND_PORT},http://192.168.10.150:${GEOINTEL_FRONTEND_PORT}}" +GEOINTEL_MAX_UPLOAD_MB="${GEOINTEL_MAX_UPLOAD_MB:-500}" + +install_dockerman_metadata() { + if [ -d /boot/config/plugins/dockerMan ]; then + mkdir -p /boot/config/plugins/dockerMan/templates-user /boot/config/plugins/dockerMan/images + cp deploy/unraid/geointel-unraid-template.xml /boot/config/plugins/dockerMan/templates-user/my-geointel.xml + cp deploy/unraid/geointel-icon.png /boot/config/plugins/dockerMan/images/geointel-icon.png + fi +} + +migrate_compose_volume_if_needed() { + if [ -f "${GEOINTEL_POSTGIS_DATA_PATH}/PG_VERSION" ]; then + return 0 + fi + + local compose_volume_path + compose_volume_path="$(docker volume inspect geointel_geointel_postgis --format '{{ .Mountpoint }}' 2>/dev/null || true)" + if [ -z "$compose_volume_path" ] || [ ! -f "${compose_volume_path}/PG_VERSION" ]; then + return 0 + fi + + echo "Migrating existing Compose PostGIS volume to ${GEOINTEL_POSTGIS_DATA_PATH}..." + mkdir -p "$GEOINTEL_POSTGIS_DATA_PATH" + cp -a "${compose_volume_path}/." "$GEOINTEL_POSTGIS_DATA_PATH/" +} + +install_dockerman_metadata + +docker compose down --remove-orphans || true + +if docker ps -a --format '{{.Names}}' | grep -qx geointel; then + docker rm -f geointel +fi + +mkdir -p "$GEOINTEL_STORAGE_PATH" "$GEOINTEL_POSTGIS_DATA_PATH" +migrate_compose_volume_if_needed + +docker run -d \ + --name geointel \ + --restart unless-stopped \ + --label net.unraid.docker.managed=dockerman \ + --label 'net.unraid.docker.webui=http://[IP]:[PORT:80]/' \ + --label net.unraid.docker.icon=/boot/config/plugins/dockerMan/images/geointel-icon.png \ + -p "${GEOINTEL_FRONTEND_PORT}:80" \ + -e GEOINTEL_POSTGRES_DB="$GEOINTEL_POSTGRES_DB" \ + -e GEOINTEL_POSTGRES_USER="$GEOINTEL_POSTGRES_USER" \ + -e GEOINTEL_POSTGRES_PASSWORD="$GEOINTEL_POSTGRES_PASSWORD" \ + -e GEOINTEL_STORAGE_ROOT=/app/storage \ + -e GEOINTEL_CORS_ORIGINS="$GEOINTEL_CORS_ORIGINS" \ + -e GEOINTEL_MAX_UPLOAD_MB="$GEOINTEL_MAX_UPLOAD_MB" \ + -v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data" \ + -v "${GEOINTEL_STORAGE_PATH}:/app/storage" \ + geointel-all-in-one:latest + +docker ps --filter name=geointel diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 3be74e2b..c1f6917f 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1518,3 +1518,15 @@ Changed: Expected Unraid behavior: - Refresh the Unraid Docker page after deploy so DockerMan reloads the user template metadata. - The running `geointel` container should have template-backed editable fields and a PNG icon. + +## Sprint 32 automatic DockerMan-native deploy follow-up (2026-06-17) + +Changed: +- Added `deploy/unraid/run-dockerman-container.sh` as the single source of truth for the final Unraid container launch. +- Updated Tower deploy scripts so repository deploys build the image with Compose but start the final container with `docker run` and DockerMan labels. +- The launch script installs the DockerMan template/icon, removes old `geointel` containers, preserves/migrates the old Compose PostGIS volume when needed and starts the final `geointel` container as DockerMan-managed. +- Updated live migration smoke to support direct container execution through `LIVE_SMOKE_CONTAINER=geointel`. + +Expected Unraid behavior: +- A deploy from the repo should no longer leave the final app as a plain Compose-owned container. +- The running `geointel` container should expose `net.unraid.docker.managed=dockerman`, web UI metadata and icon metadata immediately after deploy. diff --git a/scripts/deploy_tower.ps1 b/scripts/deploy_tower.ps1 index f7175084..8d873fa3 100644 --- a/scripts/deploy_tower.ps1 +++ b/scripts/deploy_tower.ps1 @@ -33,20 +33,12 @@ git reset --hard 'origin/$RemoteBranch' git branch -M '$RemoteBranch' chmod +x scripts/*.sh backend/docker_start.sh deploy/unraid/*.sh || true -if [ -d /boot/config/plugins/dockerMan ]; then - mkdir -p /boot/config/plugins/dockerMan/templates-user /boot/config/plugins/dockerMan/images - cp deploy/unraid/geointel-unraid-template.xml /boot/config/plugins/dockerMan/templates-user/my-geointel.xml - cp deploy/unraid/geointel-icon.png /boot/config/plugins/dockerMan/images/geointel-icon.png -fi - -docker compose down --remove-orphans || true docker compose -f docker-compose.unraid.yml config >/dev/null docker compose -f docker-compose.unraid.yml build geointel -docker compose -f docker-compose.unraid.yml up -d -docker compose -f docker-compose.unraid.yml ps +bash deploy/unraid/run-dockerman-container.sh if [ -x scripts/live_migration_smoke.sh ]; then - COMPOSE_FILE=docker-compose.unraid.yml bash scripts/live_migration_smoke.sh + LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh fi if [ -x scripts/verify_browser_runtime.sh ]; then diff --git a/scripts/deploy_tower.sh b/scripts/deploy_tower.sh index 2af2d92c..75d9b641 100755 --- a/scripts/deploy_tower.sh +++ b/scripts/deploy_tower.sh @@ -34,20 +34,12 @@ git fetch origin "$REMOTE_BRANCH" git checkout -B "$REMOTE_BRANCH" "origin/$REMOTE_BRANCH" chmod +x scripts/*.sh backend/docker_start.sh deploy/unraid/*.sh || true -if [[ -d /boot/config/plugins/dockerMan ]]; then - mkdir -p /boot/config/plugins/dockerMan/templates-user /boot/config/plugins/dockerMan/images - cp deploy/unraid/geointel-unraid-template.xml /boot/config/plugins/dockerMan/templates-user/my-geointel.xml - cp deploy/unraid/geointel-icon.png /boot/config/plugins/dockerMan/images/geointel-icon.png -fi - -docker compose down --remove-orphans || true docker compose -f docker-compose.unraid.yml config >/dev/null docker compose -f docker-compose.unraid.yml build geointel -docker compose -f docker-compose.unraid.yml up -d -docker compose -f docker-compose.unraid.yml ps +bash deploy/unraid/run-dockerman-container.sh if [[ -x scripts/live_migration_smoke.sh ]]; then - COMPOSE_FILE=docker-compose.unraid.yml bash scripts/live_migration_smoke.sh + LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh fi if [[ -x scripts/verify_browser_runtime.sh ]]; then diff --git a/scripts/live_migration_smoke.sh b/scripts/live_migration_smoke.sh index 3bd16d83..2b662c6a 100755 --- a/scripts/live_migration_smoke.sh +++ b/scripts/live_migration_smoke.sh @@ -5,6 +5,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" PYTHON_BIN="${PYTHON_BIN:-}" LIVE_SMOKE_USE_DOCKER="${LIVE_SMOKE_USE_DOCKER:-auto}" COMPOSE_FILE_PATH="${COMPOSE_FILE:-docker-compose.yml}" +LIVE_SMOKE_CONTAINER="${LIVE_SMOKE_CONTAINER:-}" run_docker_smoke() { service_name="$1" @@ -91,6 +92,98 @@ echo "== Live migration smoke passed ==" CONTAINER_SCRIPT } +run_container_smoke() { + container_name="$1" + cd "$ROOT" + + echo "== GeoIntel live migration smoke ==" + echo "Using Docker container: ${container_name}." + + docker exec -i "$container_name" sh <<'CONTAINER_SCRIPT' +set -eu +cd /app + +echo "Database URL is read from backend container settings / DATABASE_URL." + +python - <<'PY' +import time + +from sqlalchemy import text +from sqlalchemy.exc import OperationalError + +from app.db.session import get_engine + +last_error = None +for attempt in range(1, 61): + try: + with get_engine().connect() as connection: + connection.execute(text("SELECT 1")) + print(f"Database connection: ok after attempt {attempt}") + break + except OperationalError as exc: + last_error = exc + print(f"Database not ready yet ({attempt}/60): {exc}") + time.sleep(2) +else: + raise SystemExit(f"Database did not become ready: {last_error}") +PY + +python -m alembic upgrade head + +python - <<'PY' +from sqlalchemy import text + +from app.db.session import get_engine + +required_objects = [ + "public.projects", + "public.areas", + "public.datasets", + "public.jobs", + "public.vector_features", + "public.quality_checks", + "public.metrics", + "public.analysis_runs", + "public.detections", + "public.segmentations", + "public.ix_vector_features_geometry", + "public.ix_detections_geometry", + "public.ix_segmentations_geometry", +] + +with get_engine().connect() as connection: + postgis_version = connection.execute(text("SELECT PostGIS_Version()")).scalar() + print(f"PostGIS version: {postgis_version}") + + missing = [ + object_name + for object_name in required_objects + if connection.execute(text("SELECT to_regclass(:object_name)"), {"object_name": object_name}).scalar() is None + ] + if missing: + raise SystemExit(f"Missing expected migrated schema objects: {', '.join(missing)}") + print("Required runtime schema objects: ok") +PY + +HEAD_COUNT="$(python -m alembic heads | grep -c 'head')" +if [ "$HEAD_COUNT" -ne 1 ]; then + echo "Expected exactly one Alembic head, found $HEAD_COUNT" >&2 + python -m alembic heads >&2 + exit 1 +fi + +python -m alembic heads +echo "== Live migration smoke passed ==" +CONTAINER_SCRIPT +} + +if [ -n "$LIVE_SMOKE_CONTAINER" ] && command -v docker >/dev/null 2>&1; then + if docker ps --format '{{.Names}}' | grep -qx "$LIVE_SMOKE_CONTAINER"; then + run_container_smoke "$LIVE_SMOKE_CONTAINER" + exit 0 + fi +fi + if [ "$LIVE_SMOKE_USE_DOCKER" != "0" ] && command -v docker >/dev/null 2>&1 && [ -f "$ROOT/$COMPOSE_FILE_PATH" ]; then cd "$ROOT" for service_name in geointel backend; do