Harden AI Docker runtime imports
This commit is contained in:
@@ -16,6 +16,7 @@
|
||||
- Added a one-click full GIS workflow action that runs persisted selection, saves the derived dataset, saves a GeoJSON export and optionally runs QA/QC against the selected reference dataset.
|
||||
- Added a full-workflow run mode selector so repeated Map QA/QC runs can reuse the latest saved derived dataset instead of creating duplicate dataset/export artifacts.
|
||||
- Added an opt-in Docker/Unraid AI build path (`GEOINTEL_INSTALL_AI=true`) for installing optional PyTorch/Ultralytics dependencies while keeping the default GIS runtime lightweight and import-safe.
|
||||
- Hardened the AI Docker runtime with OpenCV native libraries required by Ultralytics and made YOLO dependency detection use real imports instead of optimistic module discovery.
|
||||
- Added static regression coverage for the road basemap, attribution, basemap policy notice, database layer selector and persisted operational GIS workflow wiring.
|
||||
|
||||
## Sprint 115 QA/QC and Exports usability layout pass (2026-07-04)
|
||||
|
||||
@@ -7,10 +7,17 @@ ARG GEOINTEL_INSTALL_AI=false
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
gdal-bin \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
libgdal-dev \
|
||||
libgeos-dev \
|
||||
libproj-dev \
|
||||
libpq-dev \
|
||||
libsm6 \
|
||||
libx11-6 \
|
||||
libxcb1 \
|
||||
libxext6 \
|
||||
libxrender1 \
|
||||
proj-bin \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
|
||||
@@ -227,6 +227,11 @@ The default remains `false` so normal GIS deployments do not install the large A
|
||||
runtime. GeoIntel still requires an explicit local model path and never downloads
|
||||
weights automatically.
|
||||
|
||||
AI-enabled Docker images include the native OpenCV runtime libraries required by
|
||||
Ultralytics. Dependency availability is checked with real `torch` and
|
||||
`ultralytics` imports, so missing shared libraries are reported as
|
||||
`dependency_unavailable` instead of being treated as configured.
|
||||
|
||||
Configured YOLO requires:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -14,7 +13,12 @@ class YoloDetectionAdapter:
|
||||
|
||||
@staticmethod
|
||||
def dependencies_available() -> bool:
|
||||
return importlib.util.find_spec("ultralytics") is not None and importlib.util.find_spec("torch") is not None
|
||||
try:
|
||||
import torch # noqa: F401
|
||||
import ultralytics # noqa: F401
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
def load_model(self, model_path: Path):
|
||||
if not model_path.exists() or not model_path.is_file():
|
||||
|
||||
@@ -27,6 +27,9 @@ def test_backend_dockerfile_installs_approved_gis_runtime_stack() -> None:
|
||||
assert "libgeos-dev" in dockerfile
|
||||
assert "libproj-dev" in dockerfile
|
||||
assert "proj-bin" in dockerfile
|
||||
assert "libxcb1" in dockerfile
|
||||
assert "libgl1" in dockerfile
|
||||
assert "libglib2.0-0" in dockerfile
|
||||
|
||||
|
||||
def test_backend_pyproject_exposes_gis_optional_dependency_group() -> None:
|
||||
@@ -47,6 +50,9 @@ def test_all_in_one_dockerfile_can_opt_into_ai_dependencies_without_base_install
|
||||
assert 'extras=".[gis,ai]"' in dockerfile
|
||||
assert "python scripts/gis_import_smoke.py" in dockerfile
|
||||
assert "yolo_preflight.py" in dockerfile
|
||||
assert "libxcb1" in dockerfile
|
||||
assert "libgl1" in dockerfile
|
||||
assert "libglib2.0-0" in dockerfile
|
||||
|
||||
|
||||
def test_compose_does_not_require_missing_root_env_file() -> None:
|
||||
|
||||
@@ -12,6 +12,8 @@ from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, objects=None) -> None:
|
||||
@@ -171,6 +173,14 @@ def test_yolo_configured_model_reports_configured_with_local_model_and_dependenc
|
||||
assert model.version == settings.yolo_model_version
|
||||
|
||||
|
||||
def test_yolo_dependency_check_uses_real_imports_not_find_spec() -> None:
|
||||
source = (ROOT / "backend" / "app" / "services" / "yolo_adapter.py").read_text(encoding="utf-8")
|
||||
|
||||
assert 'find_spec("ultralytics")' not in source
|
||||
assert "import ultralytics" in source
|
||||
assert "import torch" in source
|
||||
|
||||
|
||||
def test_yolo_run_requires_tile_manifest_path(tmp_path: Path) -> None:
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
|
||||
@@ -21,10 +21,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
curl \
|
||||
gdal-bin \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
libgdal-dev \
|
||||
libgeos-dev \
|
||||
libpq-dev \
|
||||
libproj-dev \
|
||||
libsm6 \
|
||||
libx11-6 \
|
||||
libxcb1 \
|
||||
libxext6 \
|
||||
libxrender1 \
|
||||
nginx \
|
||||
postgresql-16-postgis-3 \
|
||||
postgresql-16-postgis-3-scripts \
|
||||
|
||||
@@ -86,6 +86,8 @@ AI dependencies are opt-in. Leave `GEOINTEL_INSTALL_AI=false` for the default
|
||||
GIS-only image. Set `GEOINTEL_INSTALL_AI=true`, mount models through
|
||||
`GEOINTEL_MODELS_PATH` and configure `YOLO_ENABLED=true` plus
|
||||
`YOLO_MODEL_PATH=/app/models/<model>.pt` only when you have a local model file.
|
||||
The AI-enabled image installs PyTorch/Ultralytics plus the native OpenCV runtime
|
||||
libraries needed for Ultralytics imports; it still never downloads model weights.
|
||||
|
||||
Validate:
|
||||
|
||||
|
||||
@@ -5,8 +5,9 @@ Changed:
|
||||
- Reuse mode runs QA/QC against the latest saved derived map-selection dataset without creating another derived dataset/export pair.
|
||||
- Added opt-in Docker and Unraid AI build support through `GEOINTEL_INSTALL_AI=true`; default builds still install only the GIS runtime.
|
||||
- Passed YOLO runtime environment variables and a `/app/models` volume into the all-in-one Unraid container so local PyTorch/Ultralytics models can be mounted explicitly.
|
||||
- Hardened the AI image path after Tower validation showed `torch` imported but `ultralytics` failed on a missing OpenCV native library. The Dockerfiles now include the required OpenCV runtime shared libraries and YOLO dependency detection performs real imports instead of `find_spec` checks.
|
||||
- Updated `.env.example`, `backend/README.md`, `frontend/README.md`, `scripts/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`.
|
||||
- Added regression coverage in `backend/tests/test_sprint116_operational_gis_map_workflow.py` and `backend/tests/test_docker_runtime_config.py`.
|
||||
- Added regression coverage in `backend/tests/test_sprint116_operational_gis_map_workflow.py`, `backend/tests/test_sprint8b_yolo_foundation.py` and `backend/tests/test_docker_runtime_config.py`.
|
||||
|
||||
Validation:
|
||||
- RED: `python -m pytest backend\tests\test_sprint116_operational_gis_map_workflow.py backend\tests\test_docker_runtime_config.py -q` failed before implementation because `fullWorkflowMode`, AI build args and YOLO runtime env wiring were absent.
|
||||
@@ -16,8 +17,13 @@ Validation:
|
||||
- `python -m compileall backend/app` passed.
|
||||
- `python -m py_compile scripts\yolo_preflight.py backend\scripts\yolo_preflight.py` passed.
|
||||
- `python -m pytest backend\tests\test_sprint31_unraid_template.py backend\tests\test_docker_runtime_config.py -q` passed: 27 tests.
|
||||
- `cd backend && python -m pytest -q` passed: 366 tests.
|
||||
- `bash scripts/run_readiness_check.sh` passed: 366 backend tests plus frontend typecheck/build.
|
||||
- RED: `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py backend\tests\test_docker_runtime_config.py -q` failed before the runtime hardening because YOLO dependency detection still used `find_spec` and the Dockerfiles lacked OpenCV native runtime libraries.
|
||||
- `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py backend\tests\test_docker_runtime_config.py -q` passed: 30 tests.
|
||||
- `python -m compileall backend/app` passed after the AI runtime hardening.
|
||||
- `cd backend && python -m pytest -q` passed: 367 tests.
|
||||
- `cd frontend && npm run typecheck` passed.
|
||||
- `cd frontend && npm run build` passed.
|
||||
- `bash scripts/run_readiness_check.sh` passed: 367 backend tests plus frontend typecheck/build.
|
||||
- `cd backend && python -m alembic heads` passed: `202606120900 (head)`.
|
||||
- `cd backend && python -m alembic upgrade head --sql` passed.
|
||||
- `bash -n scripts/live_migration_smoke.sh; bash -n scripts/deploy_tower.sh; bash -n deploy/unraid/run-dockerman-container.sh` passed.
|
||||
|
||||
Reference in New Issue
Block a user