From add4768a52ef64a6fe356076b63c19e0f047fb17 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 6 Jul 2026 20:25:09 +0200 Subject: [PATCH] Harden YOLO tile inference inputs --- CHANGELOG.md | 1 + backend/README.md | 5 ++ backend/app/services/yolo_adapter.py | 60 +++++++++++++++--- .../tests/test_sprint8b_yolo_foundation.py | 61 +++++++++++++++++++ docs/AI_PIPELINES.md | 1 + docs/CODEX_EXECUTION_LOG.md | 20 ++++-- docs/TODO.md | 1 + 7 files changed, 137 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f14b56c2..9b2b7e37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ - The helper refuses no-model and ambiguous multi-model states, and only applies env changes when `--apply` is provided. - Documented the Tower flow for placing model files under `/mnt/user/appdata/geointel/models`, applying the env update and restarting/redeploying the all-in-one container. - Added regression coverage for no-model, multi-model, dry-run and env-file apply behavior. +- Hardened configured YOLO inference so single-band raster tiles are converted to temporary RGB prediction images and model runtime errors are returned as typed detection failures instead of raw server errors. ## Sprint 116 Operational GIS map workflow (2026-07-04) diff --git a/backend/README.md b/backend/README.md index ffd7137d..acd1137a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -275,6 +275,11 @@ python scripts/configure_yolo_model.py \ The smoke loads only the supplied local model file, does not run inference and does not download weights. +Configured YOLO inference uses raster tile artifacts from the existing tile +manifest flow. Single-band or otherwise non-RGB tile images are converted to a +temporary RGB prediction image before inference; georeferencing still comes +from the persisted tile manifest transform/bounds metadata. + Optional tuning: ```bash diff --git a/backend/app/services/yolo_adapter.py b/backend/app/services/yolo_adapter.py index 4b8f0824..362d8a28 100644 --- a/backend/app/services/yolo_adapter.py +++ b/backend/app/services/yolo_adapter.py @@ -1,7 +1,10 @@ from __future__ import annotations +from contextlib import contextmanager from pathlib import Path +import tempfile from typing import Any +from collections.abc import Iterator from app.core.config import Settings from app.core.errors import AppError @@ -62,13 +65,24 @@ class YoloDetectionAdapter: details={"tile_path": str(tile_path)}, status_code=422, ) - results = model.predict( - source=str(tile_path), - conf=float(confidence_threshold), - imgsz=int(self.settings.yolo_image_size), - device=self.settings.yolo_device, - verbose=False, - ) + try: + with _prediction_source(tile_path) as prediction_source: + results = model.predict( + source=prediction_source, + conf=float(confidence_threshold), + imgsz=int(self.settings.yolo_image_size), + device=self.settings.yolo_device, + verbose=False, + ) + except AppError: + raise + except Exception as exc: + raise AppError( + code="DETECTION_INFERENCE_FAILED", + message="Configured YOLO inference failed for a raster tile", + details={"tile_path": str(tile_path), "error": str(exc)}, + status_code=503, + ) from exc detections: list[dict[str, Any]] = [] for result in results: @@ -102,3 +116,35 @@ def _to_list(value: Any) -> list[Any]: if hasattr(value, "tolist"): return value.tolist() return list(value) + + +@contextmanager +def _prediction_source(tile_path: Path) -> Iterator[str]: + temp_path: Path | None = None + try: + try: + from PIL import Image + except Exception: + yield str(tile_path) + return + + try: + with Image.open(tile_path) as image: + if image.mode == "RGB" and len(image.getbands()) == 3: + yield str(tile_path) + return + + rgb_image = image.convert("RGB") + with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as handle: + temp_path = Path(handle.name) + rgb_image.save(temp_path) + yield str(temp_path) + return + except Exception: + if temp_path is not None: + raise + yield str(tile_path) + return + finally: + if temp_path is not None: + temp_path.unlink(missing_ok=True) diff --git a/backend/tests/test_sprint8b_yolo_foundation.py b/backend/tests/test_sprint8b_yolo_foundation.py index 7debf96e..ce99594e 100644 --- a/backend/tests/test_sprint8b_yolo_foundation.py +++ b/backend/tests/test_sprint8b_yolo_foundation.py @@ -7,10 +7,12 @@ from uuid import uuid4 import pytest from app.core.config import Settings +from app.core.errors import AppError from app.models import AnalysisRun, Dataset, Detection, Job, Project from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon from app.services.detection_service import DetectionService from app.services.model_registry_service import ModelRegistryService +from app.services.yolo_adapter import YoloDetectionAdapter ROOT = Path(__file__).resolve().parents[2] @@ -75,6 +77,33 @@ class MockYoloAdapter: ] +class RecordingPredictModel: + def __init__(self) -> None: + self.seen_sources: list[dict] = [] + + def predict(self, *, source, conf, imgsz, device, verbose): + from PIL import Image + + with Image.open(source) as image: + self.seen_sources.append( + { + "path": str(source), + "mode": image.mode, + "bands": len(image.getbands()), + "conf": conf, + "imgsz": imgsz, + "device": device, + "verbose": verbose, + } + ) + return [] + + +class ExplodingPredictModel: + def predict(self, *, source, conf, imgsz, device, verbose): + raise RuntimeError("expected input[1, 1, 480, 640] to have 3 channels, but got 1 channels instead") + + def _project_and_dataset(dataset_type: str = "raster"): project_id = uuid4() dataset_id = uuid4() @@ -312,3 +341,35 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No assert detections[0].properties_json == {"adapter": "mock", "tile_index": 0} assert runs[0].status == "success" assert jobs[0].status == "success" + + +def test_yolo_adapter_converts_single_band_tiles_to_rgb_before_prediction(tmp_path: Path) -> None: + Image = pytest.importorskip("PIL.Image") + tile_path = tmp_path / "single_band_tile.tif" + Image.new("L", (16, 16), 128).save(tile_path) + model = RecordingPredictModel() + settings = _settings(tmp_path, yolo_image_size=64, yolo_device="cpu") + + detections = YoloDetectionAdapter(settings).predict_tile(model, tile_path, confidence_threshold=0.25) + + assert detections == [] + assert model.seen_sources[0]["mode"] == "RGB" + assert model.seen_sources[0]["bands"] == 3 + assert model.seen_sources[0]["path"] != str(tile_path) + assert model.seen_sources[0]["conf"] == 0.25 + assert model.seen_sources[0]["imgsz"] == 64 + assert model.seen_sources[0]["device"] == "cpu" + assert model.seen_sources[0]["verbose"] is False + + +def test_yolo_adapter_wraps_prediction_runtime_errors(tmp_path: Path) -> None: + tile_path = tmp_path / "tile.tif" + tile_path.write_bytes(b"not an image but present") + settings = _settings(tmp_path) + + with pytest.raises(AppError) as exc_info: + YoloDetectionAdapter(settings).predict_tile(ExplodingPredictModel(), tile_path, confidence_threshold=0.25) + + assert exc_info.value.code == "DETECTION_INFERENCE_FAILED" + assert "Configured YOLO inference failed for a raster tile" in exc_info.value.message + assert exc_info.value.details["tile_path"] == str(tile_path) diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 3d166805..42c291a3 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -44,6 +44,7 @@ Sprint 8B adds an import-safe real YOLO adapter path: - `yolo-configured` reports `not_configured` until `YOLO_ENABLED=true`, `YOLO_MODEL_PATH` points to an existing local model file and optional AI dependencies are installed. - GeoIntel never downloads model weights automatically. - Real YOLO inference uses an existing raster tile manifest generated by the raster tile operation. +- YOLO raster tiles are normalized to RGB for inference when the tile artifact is not already a 3-band RGB image; the persisted georeferencing still comes from the tile manifest. - YOLO pixel boxes are converted to EPSG:4326 detection polygons from tile transform or tile bounds metadata. - Detection runs remain synchronous behind the existing job and analysis-run persistence boundary for Sprint 8B. diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 71f37cc8..e0240ee7 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -4285,14 +4285,19 @@ Changed: - Added `scripts/configure_yolo_model.py` to configure an existing local YOLO model into the deployment `.env` file without downloading model weights, loading a model or running inference. - The helper scans a mounted model directory for `.pt`, `.onnx` and `.engine` files, refuses no-model and ambiguous multi-model states, and writes env updates only when `--apply` is provided. - Added regression coverage in `backend/tests/test_sprint119_yolo_model_configuration.py` for no local model, ambiguous model selection, dry-run single model selection and env-file apply behavior. -- Updated `scripts/README.md`, `deploy/unraid/README.md`, `backend/README.md`, `docs/TODO.md` and `CHANGELOG.md`. +- Downloaded the official Ultralytics `yolov8n.pt` smoke model to Tower under `/mnt/user/appdata/geointel/models/yolov8n.pt`, recorded checksum `f59b3d833e2ff32e194b5bb8e08d211dc7c5bdf144b90d2c8412c47ccfc83b36`, and applied the env configuration with the local helper. +- Hardened `YoloDetectionAdapter.predict_tile` so non-RGB raster tile artifacts are converted to a temporary RGB image before YOLO inference while georeferencing remains driven by the tile manifest. +- Wrapped YOLO prediction runtime errors as typed `DETECTION_INFERENCE_FAILED` `AppError`s instead of leaking raw runtime exceptions through FastAPI. +- Updated `scripts/README.md`, `deploy/unraid/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`. Tested: - Red step: `python -m pytest backend\tests\test_sprint119_yolo_model_configuration.py -q` failed while `scripts/configure_yolo_model.py` was absent. +- Red step: `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py -q` failed because single-band TIFF tiles were passed through as mode `L` and prediction runtime errors leaked as raw `RuntimeError`. - `python -m pytest backend\tests\test_sprint119_yolo_model_configuration.py -q` (`4 passed`) +- `python -m pytest backend\tests\test_sprint8b_yolo_foundation.py -q` (`12 passed`) - `python -m py_compile scripts\configure_yolo_model.py` - `python -m compileall backend/app` -- `cd backend && python -m pytest -q` (`375 passed`, existing Pydantic protected-namespace warnings remain) +- `cd backend && python -m pytest -q` (`377 passed`, existing Pydantic protected-namespace warnings remain) - `cd frontend && npm run typecheck` - `cd frontend && npm run build` - `bash scripts/run_readiness_check.sh` (`Run readiness check passed`) @@ -4305,16 +4310,21 @@ Tested: - Deploy-time browser runtime verification passed for frontend, API proxy and icon. - Live API check passed: `GET /api/v1/detection/yolo/preflight` returned canonical `data` with `status=not_configured`, `YOLO_ENABLED=false`, `torch_version=2.12.1`, `ultralytics_version=8.4.89`, `will_download_models=false` and `will_run_inference=false`. - Tower helper dry-run passed: `python scripts/configure_yolo_model.py --models-dir /mnt/user/appdata/geointel/models --env-file .env --json` returned `status=no_model_found`, empty candidates and no env updates. +- Tower model apply passed: `python scripts/configure_yolo_model.py --models-dir /mnt/user/appdata/geointel/models --env-file .env --model-file /mnt/user/appdata/geointel/models/yolov8n.pt --apply --json` returned `status=applied`, `YOLO_ENABLED=true` and `YOLO_MODEL_PATH=/app/models/yolov8n.pt`. +- Live YOLO preflight with generated demo raster tile manifest passed with `status=ready`, `model_load_ok=true`, `manifest_valid=true`, `tile_paths_exist=true`, `tile_count=1`, `will_download_models=false` and `will_run_inference=false`. +- Live inference smoke before the RGB adapter fix reproduced the runtime bug: YOLOv8n expected 3 channels but the demo tile was single-band (`input[1, 1, 480, 640]`). Open: -- No local YOLO model file is currently present on Tower under `/mnt/user/appdata/geointel/models`, so runtime YOLO activation remains intentionally not configured until the operator places a real model file. +- A local YOLO smoke model is now present and configured on Tower, but it is the generic COCO `yolov8n.pt` model. It proves the runtime path, not production-quality aerial building detection. Limitations: -- Operational configuration helper only; no AI inference behavior, model download behavior, backend API contract, database migration, provider fetching or frontend product workflow changed. +- The bundled Tower model file was placed as an operator/runtime artifact under appdata, not committed to Git. +- The configured model is a generic COCO model and should be replaced by a suitable aerial/building detector for meaningful GIS output. +- No model download behavior was added to the application; the manual operator placement remains explicit. - If multiple local model files are present, the operator must choose one with `--model-file` so GeoIntel does not silently activate the wrong model. Next recommended pass: -- Place a real local model under `/mnt/user/appdata/geointel/models`, run the configurator with `--apply`, redeploy/restart the all-in-one container, then run the YOLO preflight with `--check-model-load` before any detection test run. +- Deploy the RGB adapter fix to Tower, rerun the real YOLO inference smoke against the generated demo raster tile manifest, then replace `yolov8n.pt` with a domain-appropriate aerial/building detector before evaluating QA/QC quality. ## Sprint 104 AI Lab action guardrails (2026-06-24) diff --git a/docs/TODO.md b/docs/TODO.md index 9a3d9cea..6029e533 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -389,3 +389,4 @@ This file now starts with the current implementation status. Older preparation/b - [x] Persist QA/QC feature-level evidence for matches, false positives and false negatives. - [x] Render persisted QA/QC feature-level evidence as Map workspace overlays. - [x] Add safe local YOLO model env configuration helper for Unraid/Tower runtime activation. +- [x] Harden configured YOLO inference for single-band raster tiles and wrapped runtime errors.