From 8b09bee84a6572ca82f8b3b364d66e8808db40e2 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 5 Jul 2026 00:25:12 +0200 Subject: [PATCH] Add reusable GIS run mode and AI runtime opt-in --- .env.example | 7 +++ CHANGELOG.md | 2 + backend/Dockerfile | 6 ++- backend/README.md | 20 ++++++++ backend/tests/test_docker_runtime_config.py | 33 ++++++++++++- ..._sprint116_operational_gis_map_workflow.py | 6 +++ .../tests/test_sprint31_unraid_template.py | 5 +- deploy/unraid/Dockerfile.all-in-one | 7 ++- deploy/unraid/README.md | 13 ++++-- deploy/unraid/geointel.env.example | 15 ++++++ deploy/unraid/run-dockerman-container.sh | 22 ++++++++- docker-compose.unraid.yml | 12 +++++ docker-compose.yml | 12 +++++ docs/AI_PIPELINES.md | 8 ++++ docs/CODEX_EXECUTION_LOG.md | 32 +++++++++++++ docs/TODO.md | 2 + frontend/README.md | 2 +- frontend/src/App.tsx | 1 + frontend/src/components/map/MapWorkspace.tsx | 46 ++++++++++++++++++- frontend/src/styles/app.css | 9 ++++ scripts/README.md | 12 +++++ scripts/deploy_tower.ps1 | 3 +- scripts/deploy_tower.sh | 2 +- 23 files changed, 262 insertions(+), 15 deletions(-) diff --git a/.env.example b/.env.example index 754b6466..b86fd5c4 100644 --- a/.env.example +++ b/.env.example @@ -7,8 +7,13 @@ MAX_UPLOAD_MB=500 CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202 YOLO_ENABLED=false YOLO_MODEL_PATH= +YOLO_MODEL_ID=yolo-configured +YOLO_MODEL_DISPLAY_NAME=Configured YOLO detector YOLO_MODEL_VERSION= +YOLO_DEVICE=cpu +YOLO_IMAGE_SIZE=640 YOLO_MAX_TILES=100 +YOLO_BATCH_SIZE=1 ENABLE_GRB_WFS=false GRB_WFS_URL= OSM_OVERPASS_URL=https://overpass-api.de/api/interpreter @@ -26,7 +31,9 @@ VITE_MAP_STYLE_URL= # Docker Compose / Unraid GEOINTEL_FRONTEND_PORT=1202 GEOINTEL_BACKEND_PORT=8000 +GEOINTEL_INSTALL_AI=false GEOINTEL_STORAGE_PATH=./storage +GEOINTEL_MODELS_PATH=./models GEOINTEL_POSTGIS_DATA_PATH=./postgres-data GEOINTEL_POSTGRES_DB=geointel GEOINTEL_POSTGRES_USER=geointel diff --git a/CHANGELOG.md b/CHANGELOG.md index a4c492e7..0db539ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ - Added an Operational GIS run panel that reuses AOI or active layer extents to query persisted PostGIS `vector_features` through the existing bbox selection flow. - Added a basemap policy notice when the public OpenStreetMap fallback is active and a guided operational workflow for query, derived dataset, QA/QC and export handoff. - Added a one-click full GIS workflow action that runs persisted selection, saves the derived dataset, saves a GeoJSON export and optionally runs QA/QC against the selected reference dataset. +- Added a full-workflow run mode selector so repeated Map QA/QC runs can reuse the latest saved derived dataset instead of creating duplicate dataset/export artifacts. +- Added an opt-in Docker/Unraid AI build path (`GEOINTEL_INSTALL_AI=true`) for installing optional PyTorch/Ultralytics dependencies while keeping the default GIS runtime lightweight and import-safe. - Added static regression coverage for the road basemap, attribution, basemap policy notice, database layer selector and persisted operational GIS workflow wiring. ## Sprint 115 QA/QC and Exports usability layout pass (2026-07-04) diff --git a/backend/Dockerfile b/backend/Dockerfile index effc85f4..288b2853 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -2,6 +2,8 @@ FROM python:3.12-slim WORKDIR /app +ARG GEOINTEL_INSTALL_AI=false + RUN apt-get update && apt-get install -y --no-install-recommends \ gcc \ gdal-bin \ @@ -15,7 +17,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY pyproject.toml README.md /app/ COPY app /app/app RUN pip install --no-cache-dir --upgrade pip setuptools -RUN pip install --no-cache-dir ".[gis]" +RUN extras=".[gis]" \ + && if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then extras=".[gis,ai]"; fi \ + && pip install --no-cache-dir "$extras" COPY . /app RUN python scripts/gis_import_smoke.py diff --git a/backend/README.md b/backend/README.md index dfdb2030..132e2cc4 100644 --- a/backend/README.md +++ b/backend/README.md @@ -216,6 +216,17 @@ cd backend python -m pip install -e .[ai] ``` +Docker and Unraid builds keep AI dependencies disabled by default. To build an +image with local PyTorch/Ultralytics support, set: + +```bash +GEOINTEL_INSTALL_AI=true +``` + +The default remains `false` so normal GIS deployments do not install the large AI +runtime. GeoIntel still requires an explicit local model path and never downloads +weights automatically. + Configured YOLO requires: ```bash @@ -235,6 +246,15 @@ In Docker, run the same smoke through the backend container: docker compose exec -T backend python scripts/yolo_preflight.py --model-path /absolute/path/to/local-model.pt --tile-manifest-path /absolute/path/to/manifest.json --check-model-load --json ``` +In the all-in-one Unraid runtime, place model files under the configured models +directory, mounted as `/app/models` by default: + +```bash +GEOINTEL_MODELS_PATH=/mnt/user/appdata/geointel/models +YOLO_ENABLED=true +YOLO_MODEL_PATH=/app/models/local-model.pt +``` + The smoke loads only the supplied local model file, does not run inference and does not download weights. diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py index dd93e643..df71953a 100644 --- a/backend/tests/test_docker_runtime_config.py +++ b/backend/tests/test_docker_runtime_config.py @@ -8,7 +8,7 @@ def test_backend_dockerfile_copies_package_sources_before_pip_install() -> None: dockerfile = ROOT / "backend" / "Dockerfile" lines = dockerfile.read_text(encoding="utf-8").splitlines() - pip_install_index = lines.index('RUN pip install --no-cache-dir ".[gis]"') + pip_install_index = lines.index('RUN extras=".[gis]" \\') preceding = "\n".join(lines[:pip_install_index]) assert "COPY pyproject.toml README.md /app/" in preceding @@ -18,7 +18,9 @@ def test_backend_dockerfile_copies_package_sources_before_pip_install() -> None: def test_backend_dockerfile_installs_approved_gis_runtime_stack() -> None: dockerfile = (ROOT / "backend" / "Dockerfile").read_text(encoding="utf-8") - assert 'RUN pip install --no-cache-dir ".[gis]"' in dockerfile + assert "ARG GEOINTEL_INSTALL_AI=false" in dockerfile + assert 'extras=".[gis]"' in dockerfile + assert 'extras=".[gis,ai]"' in dockerfile assert "RUN python scripts/gis_import_smoke.py" in dockerfile assert "gdal-bin" in dockerfile assert "libgdal-dev" in dockerfile @@ -37,6 +39,16 @@ def test_backend_pyproject_exposes_gis_optional_dependency_group() -> None: assert '"ultralytics>=8.3,<9"' not in pyproject.split("gis = [", 1)[1].split("]", 1)[0] +def test_all_in_one_dockerfile_can_opt_into_ai_dependencies_without_base_install() -> None: + dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") + + assert "ARG GEOINTEL_INSTALL_AI=false" in dockerfile + assert 'extras=".[gis]"' in dockerfile + assert 'extras=".[gis,ai]"' in dockerfile + assert "python scripts/gis_import_smoke.py" in dockerfile + assert "yolo_preflight.py" in dockerfile + + def test_compose_does_not_require_missing_root_env_file() -> None: compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") @@ -60,6 +72,7 @@ def test_compose_exposes_frontend_on_configurable_host_port_with_cors_origin() - def test_env_example_uses_runtime_env_names_read_by_backend_and_frontend() -> None: env_example = (ROOT / ".env.example").read_text(encoding="utf-8") + assert "GEOINTEL_INSTALL_AI=false" in env_example assert "YOLO_ENABLED=false" in env_example assert "YOLO_MODEL_PATH=" in env_example assert "YOLO_MAX_TILES=100" in env_example @@ -182,3 +195,19 @@ def test_gis_import_smoke_script_checks_runtime_imports() -> None: def test_backend_docker_context_contains_gis_import_smoke_script() -> None: assert (ROOT / "backend" / "scripts" / "gis_import_smoke.py").exists() + + +def test_unraid_deploy_passes_ai_build_arg_and_yolo_runtime_env() -> None: + deploy_ps1 = (ROOT / "scripts" / "deploy_tower.ps1").read_text(encoding="utf-8") + deploy_sh = (ROOT / "scripts" / "deploy_tower.sh").read_text(encoding="utf-8") + run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8") + + assert "--build-arg GEOINTEL_INSTALL_AI=${GEOINTEL_INSTALL_AI:-false}" in deploy_sh + assert "--build-arg GEOINTEL_INSTALL_AI='$InstallAi'" in deploy_ps1 + assert "[string]$InstallAi" in deploy_ps1 + + assert 'YOLO_ENABLED="${YOLO_ENABLED:-false}"' in run_script + assert '-e YOLO_ENABLED="$YOLO_ENABLED"' in run_script + assert '-e YOLO_MODEL_PATH="$YOLO_MODEL_PATH"' in run_script + assert '-e YOLO_MAX_TILES="$YOLO_MAX_TILES"' in run_script + assert "-v \"${GEOINTEL_MODELS_PATH}:/app/models\"" in run_script diff --git a/backend/tests/test_sprint116_operational_gis_map_workflow.py b/backend/tests/test_sprint116_operational_gis_map_workflow.py index 8c9c5876..7d884482 100644 --- a/backend/tests/test_sprint116_operational_gis_map_workflow.py +++ b/backend/tests/test_sprint116_operational_gis_map_workflow.py @@ -38,9 +38,15 @@ def test_map_workspace_can_select_persisted_database_layer_and_run_query() -> No assert "runFullGisWorkflow" in map_workspace assert "fullWorkflowStatus" in map_workspace assert "Query, save, QA and export" in map_workspace + assert "fullWorkflowMode" in map_workspace + assert "Create new dataset/export" in map_workspace + assert "Reuse latest saved dataset for QA" in map_workspace + assert "Reusing latest saved dataset for QA/QC" in map_workspace + assert "latestSelectionDataset" in map_workspace assert "selectedMapDatasetId=" in app_shell assert ".basemap-policy-notice" in styles assert ".guided-gis-flow" in styles assert ".guided-gis-batch-status" in styles + assert ".guided-gis-run-mode" in styles assert ".gis-test-run-surface" in styles assert ".gis-test-run-grid" in styles diff --git a/backend/tests/test_sprint31_unraid_template.py b/backend/tests/test_sprint31_unraid_template.py index 1dff17e7..024310a2 100644 --- a/backend/tests/test_sprint31_unraid_template.py +++ b/backend/tests/test_sprint31_unraid_template.py @@ -58,7 +58,7 @@ def test_unraid_readme_explains_port_changes_and_safe_cleanup() -> None: assert "cp deploy/unraid/geointel.env.example .env" in readme assert "GEOINTEL_FRONTEND_PORT=1203" in readme - assert "docker build -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest ." in readme + assert "docker build --build-arg GEOINTEL_INSTALL_AI=${GEOINTEL_INSTALL_AI:-false} -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest ." in readme assert "bash deploy/unraid/run-dockerman-container.sh" in readme assert "net.unraid.docker.managed=dockerman" in readme assert "curl -fsS" in readme @@ -111,7 +111,8 @@ def test_tower_deploy_uses_single_container_unraid_compose() -> None: for script in (powershell, bash): assert "docker compose -f docker-compose.unraid.yml config" in script - assert "docker build -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest ." in script + assert "--build-arg GEOINTEL_INSTALL_AI=" in script + assert "-f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest ." in script assert "docker compose -f docker-compose.unraid.yml build geointel" not in script assert "bash deploy/unraid/run-dockerman-container.sh" in script assert "LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh" in script diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 4da69976..77dc9848 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -8,6 +8,8 @@ RUN npm run build FROM postgres:16-bookworm AS runtime +ARG GEOINTEL_INSTALL_AI=false + ENV GEOINTEL_ENV=production \ GEOINTEL_API_PREFIX=/api/v1 \ GEOINTEL_STORAGE_ROOT=/app/storage \ @@ -43,8 +45,11 @@ COPY --from=frontend-build /frontend/dist/ /usr/share/nginx/html/ RUN /usr/bin/python3.11 -m venv /opt/geointel/venv \ && pip install --no-cache-dir --upgrade pip setuptools \ - && pip install --no-cache-dir ".[gis]" \ + && extras=".[gis]" \ + && if [ "$GEOINTEL_INSTALL_AI" = "true" ]; then extras=".[gis,ai]"; fi \ + && pip install --no-cache-dir "$extras" \ && python scripts/gis_import_smoke.py \ + && python scripts/yolo_preflight.py --json >/tmp/geointel-yolo-preflight.json \ && chmod +x /usr/local/bin/geointel-all-in-one-start \ && rm -f /etc/nginx/sites-enabled/default \ && mkdir -p /app/storage /run/nginx /var/log/nginx diff --git a/deploy/unraid/README.md b/deploy/unraid/README.md index f01f1493..3f813b52 100644 --- a/deploy/unraid/README.md +++ b/deploy/unraid/README.md @@ -72,16 +72,21 @@ cd /mnt/user/appdata/geointel cp deploy/unraid/geointel.env.example .env nano .env docker compose -f docker-compose.unraid.yml config -docker build -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest . +docker build --build-arg GEOINTEL_INSTALL_AI=${GEOINTEL_INSTALL_AI:-false} -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest . bash deploy/unraid/run-dockerman-container.sh ``` -The repository deploy scripts run the same flow automatically. They validate the Compose reference, build the image with plain `docker build`, install the DockerMan template/icon, remove any old Compose-owned `geointel` container, preserve/migrate the PostGIS data path and start the final container with DockerMan labels. +The repository deploy scripts run the same flow automatically. They validate the Compose reference, build the image with the `GEOINTEL_INSTALL_AI` build arg, install the DockerMan template/icon, remove any old Compose-owned `geointel` container, preserve/migrate the PostGIS data path and start the final container with DockerMan labels. Database credentials are runtime configuration, not image metadata. The all-in-one image does not bake `GEOINTEL_POSTGRES_PASSWORD` into the Dockerfile; set it through `.env`, the Unraid template or `docker run -e`. +AI dependencies are opt-in. Leave `GEOINTEL_INSTALL_AI=false` for the default +GIS-only image. Set `GEOINTEL_INSTALL_AI=true`, mount models through +`GEOINTEL_MODELS_PATH` and configure `YOLO_ENABLED=true` plus +`YOLO_MODEL_PATH=/app/models/.pt` only when you have a local model file. + Validate: ```bash @@ -119,7 +124,7 @@ GEOINTEL_CORS_ORIGINS=http://localhost:1203,http://127.0.0.1:1203,http://192.168 Apply: ```bash -docker build -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest . +docker build --build-arg GEOINTEL_INSTALL_AI=${GEOINTEL_INSTALL_AI:-false} -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest . bash deploy/unraid/run-dockerman-container.sh ``` @@ -142,7 +147,7 @@ GEOINTEL_POSTGIS_DATA_PATH=/mnt/user/appdata/geointel/postgres-data cd /mnt/user/appdata/geointel git fetch origin main git reset --hard origin/main -docker build -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest . +docker build --build-arg GEOINTEL_INSTALL_AI=${GEOINTEL_INSTALL_AI:-false} -f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest . bash deploy/unraid/run-dockerman-container.sh ``` diff --git a/deploy/unraid/geointel.env.example b/deploy/unraid/geointel.env.example index 0669bfb8..c09ea416 100644 --- a/deploy/unraid/geointel.env.example +++ b/deploy/unraid/geointel.env.example @@ -7,6 +7,9 @@ GEOINTEL_FRONTEND_PORT=1202 # Persisted application artifacts: uploads, tiles, masks, reports and exports. GEOINTEL_STORAGE_PATH=/mnt/user/appdata/geointel/storage +# Local AI model files mounted into the container as /app/models. +GEOINTEL_MODELS_PATH=/mnt/user/appdata/geointel/models + # Embedded PostGIS data directory for the all-in-one container. GEOINTEL_POSTGIS_DATA_PATH=/mnt/user/appdata/geointel/postgres-data @@ -20,3 +23,15 @@ GEOINTEL_CORS_ORIGINS=http://localhost:1202,http://127.0.0.1:1202,http://192.168 # Upload guard in MiB. GEOINTEL_MAX_UPLOAD_MB=500 + +# Optional configured-YOLO runtime. Keep disabled unless a local model is mounted. +GEOINTEL_INSTALL_AI=false +YOLO_ENABLED=false +YOLO_MODEL_PATH= +YOLO_MODEL_ID=yolo-configured +YOLO_MODEL_DISPLAY_NAME=Configured YOLO detector +YOLO_MODEL_VERSION= +YOLO_DEVICE=cpu +YOLO_IMAGE_SIZE=640 +YOLO_MAX_TILES=100 +YOLO_BATCH_SIZE=1 diff --git a/deploy/unraid/run-dockerman-container.sh b/deploy/unraid/run-dockerman-container.sh index d94bf024..68f179c5 100644 --- a/deploy/unraid/run-dockerman-container.sh +++ b/deploy/unraid/run-dockerman-container.sh @@ -13,12 +13,22 @@ fi GEOINTEL_FRONTEND_PORT="${GEOINTEL_FRONTEND_PORT:-1202}" GEOINTEL_STORAGE_PATH="${GEOINTEL_STORAGE_PATH:-/mnt/user/appdata/geointel/storage}" +GEOINTEL_MODELS_PATH="${GEOINTEL_MODELS_PATH:-/mnt/user/appdata/geointel/models}" GEOINTEL_POSTGIS_DATA_PATH="${GEOINTEL_POSTGIS_DATA_PATH:-/mnt/user/appdata/geointel/postgres-data}" GEOINTEL_POSTGRES_DB="${GEOINTEL_POSTGRES_DB:-geointel}" GEOINTEL_POSTGRES_USER="${GEOINTEL_POSTGRES_USER:-geointel}" GEOINTEL_POSTGRES_PASSWORD="${GEOINTEL_POSTGRES_PASSWORD:-geointel}" GEOINTEL_CORS_ORIGINS="${GEOINTEL_CORS_ORIGINS:-http://localhost:${GEOINTEL_FRONTEND_PORT},http://127.0.0.1:${GEOINTEL_FRONTEND_PORT},http://192.168.10.150:${GEOINTEL_FRONTEND_PORT}}" GEOINTEL_MAX_UPLOAD_MB="${GEOINTEL_MAX_UPLOAD_MB:-500}" +YOLO_ENABLED="${YOLO_ENABLED:-false}" +YOLO_MODEL_PATH="${YOLO_MODEL_PATH:-}" +YOLO_MODEL_ID="${YOLO_MODEL_ID:-yolo-configured}" +YOLO_MODEL_DISPLAY_NAME="${YOLO_MODEL_DISPLAY_NAME:-Configured YOLO detector}" +YOLO_MODEL_VERSION="${YOLO_MODEL_VERSION:-}" +YOLO_DEVICE="${YOLO_DEVICE:-cpu}" +YOLO_IMAGE_SIZE="${YOLO_IMAGE_SIZE:-640}" +YOLO_MAX_TILES="${YOLO_MAX_TILES:-100}" +YOLO_BATCH_SIZE="${YOLO_BATCH_SIZE:-1}" install_dockerman_metadata() { if [ -d /boot/config/plugins/dockerMan ]; then @@ -52,7 +62,7 @@ if docker ps -a --format '{{.Names}}' | grep -qx geointel; then docker rm -f geointel fi -mkdir -p "$GEOINTEL_STORAGE_PATH" "$GEOINTEL_POSTGIS_DATA_PATH" +mkdir -p "$GEOINTEL_STORAGE_PATH" "$GEOINTEL_MODELS_PATH" "$GEOINTEL_POSTGIS_DATA_PATH" migrate_compose_volume_if_needed docker run -d \ @@ -68,8 +78,18 @@ docker run -d \ -e GEOINTEL_STORAGE_ROOT=/app/storage \ -e GEOINTEL_CORS_ORIGINS="$GEOINTEL_CORS_ORIGINS" \ -e GEOINTEL_MAX_UPLOAD_MB="$GEOINTEL_MAX_UPLOAD_MB" \ + -e YOLO_ENABLED="$YOLO_ENABLED" \ + -e YOLO_MODEL_PATH="$YOLO_MODEL_PATH" \ + -e YOLO_MODEL_ID="$YOLO_MODEL_ID" \ + -e YOLO_MODEL_DISPLAY_NAME="$YOLO_MODEL_DISPLAY_NAME" \ + -e YOLO_MODEL_VERSION="$YOLO_MODEL_VERSION" \ + -e YOLO_DEVICE="$YOLO_DEVICE" \ + -e YOLO_IMAGE_SIZE="$YOLO_IMAGE_SIZE" \ + -e YOLO_MAX_TILES="$YOLO_MAX_TILES" \ + -e YOLO_BATCH_SIZE="$YOLO_BATCH_SIZE" \ -v "${GEOINTEL_POSTGIS_DATA_PATH}:/var/lib/postgresql/data" \ -v "${GEOINTEL_STORAGE_PATH}:/app/storage" \ + -v "${GEOINTEL_MODELS_PATH}:/app/models" \ geointel-all-in-one:latest docker ps --filter name=geointel diff --git a/docker-compose.unraid.yml b/docker-compose.unraid.yml index 5ea5f4bb..b0d8c31c 100644 --- a/docker-compose.unraid.yml +++ b/docker-compose.unraid.yml @@ -3,6 +3,8 @@ services: build: context: . dockerfile: deploy/unraid/Dockerfile.all-in-one + args: + GEOINTEL_INSTALL_AI: ${GEOINTEL_INSTALL_AI:-false} image: geointel-all-in-one:latest container_name: geointel labels: @@ -16,11 +18,21 @@ services: GEOINTEL_STORAGE_ROOT: /app/storage GEOINTEL_CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202} GEOINTEL_MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500} + YOLO_ENABLED: ${YOLO_ENABLED:-false} + YOLO_MODEL_PATH: ${YOLO_MODEL_PATH:-} + YOLO_MODEL_ID: ${YOLO_MODEL_ID:-yolo-configured} + YOLO_MODEL_DISPLAY_NAME: ${YOLO_MODEL_DISPLAY_NAME:-Configured YOLO detector} + YOLO_MODEL_VERSION: ${YOLO_MODEL_VERSION:-} + YOLO_DEVICE: ${YOLO_DEVICE:-cpu} + YOLO_IMAGE_SIZE: ${YOLO_IMAGE_SIZE:-640} + YOLO_MAX_TILES: ${YOLO_MAX_TILES:-100} + YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1} ports: - "${GEOINTEL_FRONTEND_PORT:-1202}:80" volumes: - ${GEOINTEL_POSTGIS_DATA_PATH:-geointel_postgis}:/var/lib/postgresql/data - ${GEOINTEL_STORAGE_PATH:-./storage}:/app/storage + - ${GEOINTEL_MODELS_PATH:-./models}:/app/models restart: unless-stopped volumes: diff --git a/docker-compose.yml b/docker-compose.yml index fb8e5326..4231bf96 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,15 +16,27 @@ services: backend: build: context: ./backend + args: + GEOINTEL_INSTALL_AI: ${GEOINTEL_INSTALL_AI:-false} environment: DATABASE_URL: postgresql+psycopg://${GEOINTEL_POSTGRES_USER:-geointel}:${GEOINTEL_POSTGRES_PASSWORD:-geointel}@db:5432/${GEOINTEL_POSTGRES_DB:-geointel} STORAGE_ROOT: /app/storage CORS_ORIGINS: ${GEOINTEL_CORS_ORIGINS:-http://localhost:1202,http://127.0.0.1:1202} MAX_UPLOAD_MB: ${GEOINTEL_MAX_UPLOAD_MB:-500} + YOLO_ENABLED: ${YOLO_ENABLED:-false} + YOLO_MODEL_PATH: ${YOLO_MODEL_PATH:-} + YOLO_MODEL_ID: ${YOLO_MODEL_ID:-yolo-configured} + YOLO_MODEL_DISPLAY_NAME: ${YOLO_MODEL_DISPLAY_NAME:-Configured YOLO detector} + YOLO_MODEL_VERSION: ${YOLO_MODEL_VERSION:-} + YOLO_DEVICE: ${YOLO_DEVICE:-cpu} + YOLO_IMAGE_SIZE: ${YOLO_IMAGE_SIZE:-640} + YOLO_MAX_TILES: ${YOLO_MAX_TILES:-100} + YOLO_BATCH_SIZE: ${YOLO_BATCH_SIZE:-1} ports: - "${GEOINTEL_BACKEND_PORT:-8000}:8000" volumes: - ${GEOINTEL_STORAGE_PATH:-./storage}:/app/storage + - ${GEOINTEL_MODELS_PATH:-./models}:/app/models - ./fixtures:/app/fixtures:ro command: sh /app/docker_start.sh depends_on: diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 74f447ac..2179b7df 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -84,8 +84,16 @@ Ultralytics/PyTorch compatibility, but it still does not run tile prediction and does not download weights. It cannot be combined with `--assume-dependencies` because that would turn the smoke into a false positive. +Docker and Unraid runtime support remains opt-in. Set `GEOINTEL_INSTALL_AI=true` +at build time to install the backend `.[gis,ai]` extra into the container. Leave +it unset or `false` for the default GIS-only image. Runtime model files should be +mounted into the container, for example `/app/models/local-model.pt`, and enabled +with `YOLO_ENABLED=true` plus `YOLO_MODEL_PATH=/app/models/local-model.pt`. +GeoIntel never downloads weights automatically. + Environment variables: +- `GEOINTEL_INSTALL_AI` - `YOLO_ENABLED` - `YOLO_MODEL_PATH` - `YOLO_MODEL_ID` diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 5b22f654..92bd5851 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,35 @@ +## Sprint 117 Reusable GIS run and AI runtime opt-in (2026-07-05) + +Changed: +- Added a Map workspace full-run mode selector with `Create new dataset/export` and `Reuse latest saved dataset for QA`. +- Reuse mode runs QA/QC against the latest saved derived map-selection dataset without creating another derived dataset/export pair. +- Added opt-in Docker and Unraid AI build support through `GEOINTEL_INSTALL_AI=true`; default builds still install only the GIS runtime. +- Passed YOLO runtime environment variables and a `/app/models` volume into the all-in-one Unraid container so local PyTorch/Ultralytics models can be mounted explicitly. +- Updated `.env.example`, `backend/README.md`, `frontend/README.md`, `scripts/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`. +- Added regression coverage in `backend/tests/test_sprint116_operational_gis_map_workflow.py` and `backend/tests/test_docker_runtime_config.py`. + +Validation: +- RED: `python -m pytest backend\tests\test_sprint116_operational_gis_map_workflow.py backend\tests\test_docker_runtime_config.py -q` failed before implementation because `fullWorkflowMode`, AI build args and YOLO runtime env wiring were absent. +- `python -m pytest backend\tests\test_sprint116_operational_gis_map_workflow.py backend\tests\test_docker_runtime_config.py -q` passed: 22 tests. +- `cd frontend && npm run typecheck` passed. +- `cd frontend && npm run build` passed. +- `python -m compileall backend/app` passed. +- `python -m py_compile scripts\yolo_preflight.py backend\scripts\yolo_preflight.py` passed. +- `python -m pytest backend\tests\test_sprint31_unraid_template.py backend\tests\test_docker_runtime_config.py -q` passed: 27 tests. +- `cd backend && python -m pytest -q` passed: 366 tests. +- `bash scripts/run_readiness_check.sh` passed: 366 backend tests plus frontend typecheck/build. +- `cd backend && python -m alembic heads` passed: `202606120900 (head)`. +- `cd backend && python -m alembic upgrade head --sql` passed. +- `bash -n scripts/live_migration_smoke.sh; bash -n scripts/deploy_tower.sh; bash -n deploy/unraid/run-dockerman-container.sh` passed. +- Local Codex host could not run `docker compose config` because Docker is not installed in this Windows environment; Tower Docker validation is required after push/deploy. + +Limitations: +- `GEOINTEL_INSTALL_AI=true` installs optional PyTorch/Ultralytics dependencies but still requires a user-provided local model file; GeoIntel does not download weights. +- Reuse mode intentionally reuses only the latest saved map-selection dataset for QA/QC. It does not delete or mutate older derived datasets/exports. + +Next recommended pass: +- Run full readiness, deploy Tower, and browser-verify both Map run modes plus configured-YOLO preflight status in the live container. + ## Sprint 116 Operational GIS map workflow (2026-07-04) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index 40df0ac4..8e886689 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -78,6 +78,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add selected map feature extraction with highlight, property table, copy and GeoJSON download. - [x] Add operational GIS map workflow with road basemap, persisted database layer selection and AOI/layer `vector_features` query run. - [x] Add basemap policy notice and guided GIS query-to-QA/export workflow in the Map workspace. +- [x] Add reusable latest-result mode for repeated Map QA/QC runs without duplicate derived artifacts. +- [x] Add opt-in Docker/Unraid AI build/runtime path for local PyTorch/Ultralytics YOLO operation. - [x] Add one-click full GIS workflow action for query, derived dataset, QA/QC and export handoff. - [x] Add QA/QC workspace result hierarchy and filter density polish. - [x] Add Change Detection panel hierarchy and analysis workspace density polish. diff --git a/frontend/README.md b/frontend/README.md index 3a53ad76..a78c6bb9 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -12,7 +12,7 @@ The Map workspace defaults to an OpenStreetMap road basemap with visible attribu When the public OpenStreetMap fallback is active, the Map workspace shows a basemap usage notice. This keeps the local/demo default honest and reminds operators to configure a managed style URL before production or heavier tile traffic. -Operational GIS testing is now available directly in the Map workspace. Users can choose a persisted vector database layer, load it on the map, reuse the selected AOI or active layer extent, run the existing persisted `vector_features` bbox query, save the result as a derived dataset, export the selection GeoJSON, choose a reference dataset and launch QA/QC without creating fake data or a parallel backend path. The guided workflow also includes a one-click full run action that executes query, derived dataset save, GeoJSON export and optional QA/QC in sequence with visible status. +Operational GIS testing is now available directly in the Map workspace. Users can choose a persisted vector database layer, load it on the map, reuse the selected AOI or active layer extent, run the existing persisted `vector_features` bbox query, save the result as a derived dataset, export the selection GeoJSON, choose a reference dataset and launch QA/QC without creating fake data or a parallel backend path. The guided workflow also includes a one-click full run action that executes query, derived dataset save, GeoJSON export and optional QA/QC in sequence with visible status. For repeated review, switch the run mode from `Create new dataset/export` to `Reuse latest saved dataset for QA`; this reruns QA/QC against the latest saved derived dataset without creating another dataset/export pair. QA/QC and Exports follow the same calmer density model. QA/QC keeps metric evidence, feature ids and raw findings available but compresses provenance and history surfaces so review starts from the selected check and map evidence actions. Exports uses denser handoff cards, latest-artifact cards and history filters so artifact creation and download paths are easier to scan. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index cb977df1..d47b7fd7 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -860,6 +860,7 @@ function App(): JSX.Element { latestSelectionExportPath={latestSelectionExport?.path ?? null} selectionDatasetSaving={selectionDatasetSaving} selectionDatasetError={selectionDatasetError} + latestSelectionDataset={latestSelectionDataset} latestSelectionDatasetName={latestSelectionDataset?.name ?? null} mapQaReferenceDatasets={referenceDatasets} selectedMapQaReferenceDatasetId={selectedMapQaReferenceDatasetId} diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index d3976836..ed08f3ab 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -201,6 +201,7 @@ interface MapWorkspaceProps { latestSelectionExportPath: string | null selectionDatasetSaving: boolean selectionDatasetError: string | null + latestSelectionDataset: DatasetCreateResponse | null latestSelectionDatasetName: string | null mapQaReferenceDatasets: DatasetCreateResponse[] selectedMapQaReferenceDatasetId: string @@ -258,6 +259,7 @@ export function MapWorkspace({ latestSelectionExportPath, selectionDatasetSaving, selectionDatasetError, + latestSelectionDataset, latestSelectionDatasetName, mapQaReferenceDatasets, selectedMapQaReferenceDatasetId, @@ -290,6 +292,7 @@ export function MapWorkspace({ const [fullWorkflowRunning, setFullWorkflowRunning] = useState(false) const [fullWorkflowStatus, setFullWorkflowStatus] = useState('Ready to run persisted GIS workflow.') const [fullWorkflowError, setFullWorkflowError] = useState(null) + const [fullWorkflowMode, setFullWorkflowMode] = useState<'new' | 'reuse'>('new') const selectedMapArea = areas.find((area) => area.id === selectedMapAreaId) const featureProperties = selectedMapFeature?.properties ?? null const featureSummaryEntries = featureProperties @@ -409,6 +412,30 @@ export function MapWorkspace({ const runFullGisWorkflow = async () => { const bbox = currentSelectionBbox ?? selectedAreaBbox ?? activeLayerBbox + if (fullWorkflowMode === 'reuse') { + if (!latestSelectionDataset) { + setFullWorkflowError('Save a map selection dataset before reusing the latest result.') + return + } + if (!selectedMapQaReferenceDatasetId) { + setFullWorkflowError('Select a reference dataset before reusing the latest result for QA/QC.') + return + } + setFullWorkflowRunning(true) + setFullWorkflowError(null) + try { + setFullWorkflowStatus('Reusing latest saved dataset for QA/QC...') + const qaResult = await onRunMapSelectionQa(latestSelectionDataset) + setFullWorkflowStatus(qaResult ? 'Reused latest saved dataset and completed QA/QC.' : 'Latest saved dataset reused, but QA/QC did not complete.') + } catch (error) { + setFullWorkflowError(error instanceof Error ? error.message : 'Full GIS workflow failed.') + setFullWorkflowStatus('Workflow stopped.') + } finally { + setFullWorkflowRunning(false) + } + return + } + if (!selectedMapDataset || !bbox) { setFullWorkflowError('Select a database layer and AOI/layer extent before running the full workflow.') return @@ -753,9 +780,26 @@ export function MapWorkspace({
+