From 71c2cd9411cb6add6f90b98fbf54e1598044988a Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 7 Jul 2026 01:50:24 +0200 Subject: [PATCH] Fix YOLO class normalization --- CHANGELOG.md | 9 ++++- backend/README.md | 6 ++- backend/app/services/detection_service.py | 14 +++++-- .../tests/test_sprint8b_yolo_foundation.py | 39 +++++++++++++++++++ docs/AI_PIPELINES.md | 6 ++- docs/CODEX_EXECUTION_LOG.md | 26 ++++++++++++- docs/TODO.md | 3 +- scripts/README.md | 9 +++-- 8 files changed, 98 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85596ae3..9bb50397 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ # Changelog +## Sprint 123 YOLO class normalization and real-data inference fix (2026-07-07) + +- Fixed configured-YOLO class filtering so model labels such as `Building` match operator/domain filters such as `building`. +- Persisted configured-YOLO class names as canonical lowercase values while preserving the original model label in detection provenance. +- Added regression coverage for the mixed-case YOLO class route that caused the Geel real-data smoke to persist zero detections. +- Confirmed through direct Tower inference that the active local building model returns raw detections on the prepared Geel orthophoto tile; the remaining work is threshold/QA calibration rather than model availability. + ## Sprint 122 Real operator data availability and raster metadata fix (2026-07-07) - Created Tower operator sample artifacts under `/mnt/user/appdata/geointel/storage/operator-data`: @@ -15,7 +22,7 @@ - Fixed raster upload metadata mapping so uploaded rasters persist canonical `bounds_json`, `resolution_json` and `bands_json` from extracted raster metadata. - Added regression coverage for raster upload metadata mapping. - Deployed the fix to Tower and ran the real-data detection + QA workflow against `http://192.168.10.150:1202`. -- The workflow passed with persisted raster/reference datasets, tile manifest, AnalysisRun, QualityCheck and detection GeoJSON export. The configured evaluation model returned zero detections on the Geel AOI, so model quality/calibration remains a follow-up. +- The workflow passed with persisted raster/reference datasets, tile manifest, AnalysisRun, QualityCheck and detection GeoJSON export. A follow-up pass identified case-sensitive class filtering as the reason the initial Geel run persisted zero detections. ## Sprint 121 Real data detection and QA workflow smoke (2026-07-07) diff --git a/backend/README.md b/backend/README.md index 4e0323d9..07a99196 100644 --- a/backend/README.md +++ b/backend/README.md @@ -367,8 +367,10 @@ dataset through the normal dataset service, generates raster tiles, selects a local model asset, runs configured YOLO detection, compares persisted detections against persisted `vector_features`, persists QA/QC rows and exports the detection GeoJSON. It never seeds demo detections, enables fixture mode, -fetches live providers or downloads model weights. Current V1 upload support is -limited to GeoTIFF-style rasters and GeoJSON/JSON reference vectors. +fetches live providers or downloads model weights. Configured-YOLO model class +labels are normalized to lowercase for filtering and persisted detections while +the original model label is retained in detection provenance. Current V1 upload +support is limited to GeoTIFF-style rasters and GeoJSON/JSON reference vectors. ### Run backend diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index 64067fc8..dca7c2d9 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -541,13 +541,14 @@ class DetectionService: model_path = Path(settings.yolo_model_path or "").expanduser() adapter = yolo_adapter_class(settings) model = adapter.load_model(model_path) - allowed_classes = set(class_filter) + allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)} persisted: list[Detection] = [] manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") or "EPSG:4326" for tile in manifest["tiles"]: tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser()) for raw in adapter.predict_tile(model, tile_path, confidence_threshold): - class_name = str(raw.get("class_name") or "") + model_class_name = str(raw.get("class_name") or "").strip() + class_name = DetectionService._canonical_class_name(model_class_name) confidence = float(raw.get("confidence", 0.0)) if allowed_classes and class_name not in allowed_classes: continue @@ -557,6 +558,9 @@ class DetectionService: if not isinstance(bbox, list): raise AppError(code="DETECTION_INVALID_BBOX", message="YOLO adapter returned a detection without bbox", status_code=422) geometry = pixel_bbox_to_epsg4326_polygon(bbox=bbox, tile=tile, crs=tile.get("crs") or manifest_crs) + properties = dict(raw.get("properties") or {}) + if model_class_name and model_class_name != class_name: + properties.setdefault("model_class_name", model_class_name) detection = Detection( id=uuid.uuid4(), project_id=project_id, @@ -575,7 +579,7 @@ class DetectionService: "y_max": float(bbox[3]), }, source_tile_path=str(tile_path), - properties_json={**dict(raw.get("properties") or {}), "tile_index": tile.get("index")}, + properties_json={**properties, "tile_index": tile.get("index")}, ) db.add(detection) persisted.append(detection) @@ -584,6 +588,10 @@ class DetectionService: db.refresh(detection) return persisted + @staticmethod + def _canonical_class_name(value: Any) -> str: + return str(value or "").strip().casefold() + @staticmethod def _load_tile_manifest(tile_manifest_path: str | None, max_tiles: int) -> dict[str, Any]: if not tile_manifest_path: diff --git a/backend/tests/test_sprint8b_yolo_foundation.py b/backend/tests/test_sprint8b_yolo_foundation.py index ce99594e..8dd71732 100644 --- a/backend/tests/test_sprint8b_yolo_foundation.py +++ b/backend/tests/test_sprint8b_yolo_foundation.py @@ -77,6 +77,18 @@ class MockYoloAdapter: ] +class MixedCaseYoloAdapter(MockYoloAdapter): + def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]: + return [ + { + "class_name": "Building", + "confidence": 0.91, + "bbox": [10.0, 20.0, 30.0, 40.0], + "properties": {"adapter": "mock"}, + } + ] + + class RecordingPredictModel: def __init__(self) -> None: self.seen_sources: list[dict] = [] @@ -343,6 +355,33 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No assert jobs[0].status == "success" +def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_path: Path) -> None: + db, project_id, dataset_id = _project_and_dataset() + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"local weights") + settings = _settings(tmp_path, yolo_model_path=str(model_path)) + manifest_path = _manifest(tmp_path, tile_count=1) + + result = DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-configured", + confidence_threshold=0.5, + class_filter=["building"], + tile_manifest_path=str(manifest_path), + settings=settings, + yolo_adapter_class=MixedCaseYoloAdapter, + ) + + detections = [item for item in db.added if isinstance(item, Detection)] + + assert result.status == "success" + assert result.detection_count == 1 + assert detections[0].class_name == "building" + assert detections[0].properties_json["model_class_name"] == "Building" + + def test_yolo_adapter_converts_single_band_tiles_to_rgb_before_prediction(tmp_path: Path) -> None: Image = pytest.importorskip("PIL.Image") tile_path = tmp_path / "single_band_tile.tif" diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 4cf3b209..9de2d4c3 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -46,6 +46,7 @@ Sprint 8B adds an import-safe real YOLO adapter path: - Real YOLO inference uses an existing raster tile manifest generated by the raster tile operation. - YOLO raster tiles are normalized to RGB for inference when the tile artifact is not already a 3-band RGB image; the persisted georeferencing still comes from the tile manifest. - YOLO pixel boxes are converted to EPSG:4326 detection polygons from tile transform or tile bounds metadata. +- YOLO class labels are normalized to lowercase for persisted detection records and filtering, while the original model label remains available in detection provenance. - Detection runs remain synchronous behind the existing job and analysis-run persistence boundary for Sprint 8B. ### Sprint 13 YOLO operational preflight @@ -162,8 +163,9 @@ The script verifies the full persisted chain: It refuses to run without a real GeoTIFF-style raster and GeoJSON/JSON reference vector. It does not seed demo data, use `fixture_mode`, fetch live providers or -download model weights. A zero detection count is valid as runtime evidence but -does not prove the model is useful for the target imagery. +download model weights. A zero detection count is valid as runtime evidence only +when the selected model genuinely returns no usable detections after canonical +class filtering; it does not prove the model is useful for the target imagery. ### Sprint 8C detection visualization and QA status diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 3e1a319d..3259aa53 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,25 @@ +## Sprint 123 YOLO class normalization and real-data inference fix (2026-07-07) + +Changed: +- Investigated the Geel real-data smoke that persisted zero detections despite the configured building model being available. +- Confirmed on Tower that `/app/models/yolov8n-building-segmentation.pt` reports model class `Building` and returns 4 raw detections at confidence `0.5` on the same real Geel tile manifest. +- Fixed configured-YOLO detection persistence so model class names are compared case-insensitively against `class_filter`, persisted as canonical lowercase domain classes, and preserve the original model class name in `properties_json.model_class_name`. +- Added regression coverage in `backend/tests/test_sprint8b_yolo_foundation.py`. + +Validation: +- RED: `python -m pytest backend/tests/test_sprint8b_yolo_foundation.py::test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class -q` failed with `detection_count=0` because `Building` did not match `building`. +- `python -m pytest backend/tests/test_sprint8b_yolo_foundation.py::test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class -q` passed. +- `python -m pytest backend/tests/test_sprint8b_yolo_foundation.py backend/tests/test_model_asset_catalog.py backend/tests/test_sprint121_real_data_detection_qa_smoke.py backend/tests/test_sprint122_raster_upload_metadata_mapping.py -q` passed: 20 tests. + +Open: +- Full readiness, Tower deploy and repeated live real-data smoke still need to be run for this pass. + +Limitations: +- This fixes class routing and persistence, not model quality. Thresholds, false positives and reference IoU quality still need calibration on larger local orthophoto samples. + +Next recommended pass: +- Redeploy to Tower, rerun the Geel real-data smoke, then inspect persisted detections and QA metrics to choose practical confidence/IoU defaults. + ## Sprint 122 Real operator data availability and raster metadata fix (2026-07-07) Changed: @@ -28,11 +50,11 @@ Validation: - detection export: `0081c230-1766-4ff3-8df0-e47236f529d1` Limitations: -- The workflow is now operational against real operator data, but the active evaluation model returned zero detections on the Geel sample AOI. This is model/data quality evidence, not a platform failure. +- The workflow is operational against real operator data. A follow-up class-normalization pass found that the active evaluation model returned `Building` while the workflow filtered on `building`; see Sprint 123. - The prepared files are runtime artifacts on Tower, not repository fixtures. Next recommended pass: -- Calibrate model selection and confidence/class handling against real Flemish orthophotos, or replace the evaluation model with a detector better aligned to aerial building footprints. +- Redeploy the class-normalization fix, rerun the real-data smoke and calibrate confidence/IoU thresholds against persisted detection and QA metrics. ## Sprint 121 Real data detection and QA workflow smoke (2026-07-07) diff --git a/docs/TODO.md b/docs/TODO.md index e0df6857..1361625a 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -93,7 +93,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add Export/System handoff hierarchy and provider registry density polish. - [x] Add operator-provided real raster/reference detection + QA workflow smoke. - [x] Validate the configured building model on a real georeferenced Kempen orthophoto/GeoTIFF with persisted reference vectors and QA/QC metrics. -- [ ] Calibrate or replace the evaluation building model after real orthophoto validation returned zero detections on the Geel sample AOI. +- [x] Fix configured-YOLO mixed-case class labels so `Building` model output matches `building` domain filters. +- [ ] Calibrate confidence, IoU and model selection against persisted Geel detections and additional local orthophoto/reference samples. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index 0fab0102..36f6bd99 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -180,9 +180,12 @@ metadata, tiles the raster, selects a mounted local model asset, verifies read-only YOLO preflight, runs configured YOLO detection, runs detection QA against persisted `vector_features`, and exports the detection run as GeoJSON. It does not seed demo data, enable fixture detections, fetch external data or -download model weights. A zero detection count is accepted operationally, but -must be interpreted as model/data quality evidence rather than as a successful -building extraction result. +download model weights. Configured-YOLO model class labels are normalized to +lowercase for filtering and persisted detections, while the original model label +is retained in detection provenance. A zero detection count is accepted +operationally only when the selected model genuinely returns no usable +detections after class filtering; it must be interpreted as model/data quality +evidence rather than as a successful building extraction result. Docker images install only the GIS runtime by default. To build a local/Tower image with PyTorch/Ultralytics available for the configured-YOLO preflight and