diff --git a/CHANGELOG.md b/CHANGELOG.md index 9bb50397..cf530952 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,13 @@ # Changelog -## Sprint 123 YOLO class normalization and real-data inference fix (2026-07-07) +## Sprint 123 YOLO class and tile CRS normalization (2026-07-07) - Fixed configured-YOLO class filtering so model labels such as `Building` match operator/domain filters such as `building`. - Persisted configured-YOLO class names as canonical lowercase values while preserving the original model label in detection provenance. - Added regression coverage for the mixed-case YOLO class route that caused the Geel real-data smoke to persist zero detections. - Confirmed through direct Tower inference that the active local building model returns raw detections on the prepared Geel orthophoto tile; the remaining work is threshold/QA calibration rather than model availability. +- Fixed raster tile manifest CRS propagation so generated tile manifests include source CRS metadata required to convert YOLO pixel boxes to WGS84 Detection GeoJSON coordinates. ## Sprint 122 Real operator data availability and raster metadata fix (2026-07-07) diff --git a/backend/README.md b/backend/README.md index 07a99196..bf894b8c 100644 --- a/backend/README.md +++ b/backend/README.md @@ -369,7 +369,9 @@ detections against persisted `vector_features`, persists QA/QC rows and exports the detection GeoJSON. It never seeds demo detections, enables fixture mode, fetches live providers or downloads model weights. Configured-YOLO model class labels are normalized to lowercase for filtering and persisted detections while -the original model label is retained in detection provenance. Current V1 upload +the original model label is retained in detection provenance. Raster tile +manifests generated for AI handoff include source CRS metadata so pixel-space +model outputs can be transformed to WGS84 GeoJSON coordinates. Current V1 upload support is limited to GeoTIFF-style rasters and GeoJSON/JSON reference vectors. ### Run backend diff --git a/backend/app/services/raster_operations_service.py b/backend/app/services/raster_operations_service.py index 43a3b2b9..c580187f 100644 --- a/backend/app/services/raster_operations_service.py +++ b/backend/app/services/raster_operations_service.py @@ -939,8 +939,13 @@ class RasterOperationsService: manifest_tiles: list[dict[str, Any]] = [] tile_paths: list[str] = [] + source_crs: str | None = None with rasterio.open(dataset.storage_path) as source: + raw_source_crs = getattr(source, "crs", None) + source_crs = raw_source_crs.to_string() if hasattr(raw_source_crs, "to_string") else ( + str(raw_source_crs) if raw_source_crs else dataset.crs + ) source_count = getattr(source, "count", 0) if not source_count: source_count = 1 @@ -981,6 +986,7 @@ class RasterOperationsService: "pixel_window": [int(xoff), int(yoff), int(tile_width), int(tile_height)], "bounds": RasterOperationsService._window_bounds_to_list(bounds), "transform": [float(item) for item in transform.to_gdal()], + "crs": source_crs, "index": tile_index, }, ) @@ -994,10 +1000,14 @@ class RasterOperationsService: except AppError: source_metadata = {"bounds": [0.0, 0.0, 0.0, 0.0]} bounds = source_metadata.get("bounds", [0.0, 0.0, 0.0, 0.0]) + manifest_crs = source_crs or source_metadata.get("crs") or dataset.crs manifest_payload = { "tile_set_id": tile_set_id, "source_dataset_id": str(dataset.id), "source_raster_id": str(dataset.id), + "crs": manifest_crs, + "source_crs": manifest_crs, + "dataset_crs": dataset.crs, "bounds": [float(value) for value in bounds], "tile_size": int(tile_size), "overlap": int(overlap), diff --git a/backend/tests/test_raster_operations_service.py b/backend/tests/test_raster_operations_service.py index 3d2a7777..73b8eabb 100644 --- a/backend/tests/test_raster_operations_service.py +++ b/backend/tests/test_raster_operations_service.py @@ -606,6 +606,10 @@ def test_raster_tile_returns_manifest_payload(monkeypatch, tmp_path) -> None: float(window.yoff + window.height), ) + class FakeCRS: + def to_string(self): + return "EPSG:31370" + class FakeSource: width = 10 height = 10 @@ -614,6 +618,7 @@ def test_raster_tile_returns_manifest_payload(monkeypatch, tmp_path) -> None: self.profile = {"width": self.width, "height": self.height, "count": 1, "dtype": "uint8", "transform": None} self.transform = None self.nodata = 0 + self.crs = FakeCRS() def read(self, *args, **kwargs): return FakeArray() @@ -670,7 +675,10 @@ def test_raster_tile_returns_manifest_payload(monkeypatch, tmp_path) -> None: assert payload["manifest"]["overlap"] == 1 assert payload["manifest"]["source_dataset_id"] == str(dataset_id) assert payload["manifest"]["source_raster_id"] == str(dataset_id) + assert payload["manifest"]["crs"] == "EPSG:31370" + assert payload["manifest"]["source_crs"] == "EPSG:31370" assert payload["manifest"]["count"] == payload["count"] + assert payload["manifest"]["tiles"][0]["crs"] == "EPSG:31370" assert payload["manifest"]["tiles"][0]["bounds"] == [0.0, 0.0, 4.0, 4.0] assert payload["manifest"]["ai_inference"] is False assert payload["manifest"]["tile_server"] is None diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 9de2d4c3..6468e46b 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -187,7 +187,7 @@ Elke tile moet opslaan: - pixel window - geospatial bounds - transform -- CRS +- CRS, and the manifest must also carry source CRS metadata - tile size - overlap diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 3259aa53..1f5be967 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,24 +1,42 @@ -## Sprint 123 YOLO class normalization and real-data inference fix (2026-07-07) +## Sprint 123 YOLO class and tile CRS normalization (2026-07-07) Changed: - Investigated the Geel real-data smoke that persisted zero detections despite the configured building model being available. - Confirmed on Tower that `/app/models/yolov8n-building-segmentation.pt` reports model class `Building` and returns 4 raw detections at confidence `0.5` on the same real Geel tile manifest. - Fixed configured-YOLO detection persistence so model class names are compared case-insensitively against `class_filter`, persisted as canonical lowercase domain classes, and preserve the original model class name in `properties_json.model_class_name`. -- Added regression coverage in `backend/tests/test_sprint8b_yolo_foundation.py`. +- Found a second live GIS correctness issue: generated tile manifests carried Lambert bounds/transforms but no CRS, so detection GeoJSON could expose EPSG:31370 coordinates as if they were EPSG:4326. +- Fixed raster tile manifest generation to include `crs`, `source_crs` and `dataset_crs` on the manifest and `crs` on each tile entry when the source raster CRS is known. +- Added regression coverage in `backend/tests/test_sprint8b_yolo_foundation.py` and `backend/tests/test_raster_operations_service.py`. Validation: - RED: `python -m pytest backend/tests/test_sprint8b_yolo_foundation.py::test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class -q` failed with `detection_count=0` because `Building` did not match `building`. - `python -m pytest backend/tests/test_sprint8b_yolo_foundation.py::test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class -q` passed. - `python -m pytest backend/tests/test_sprint8b_yolo_foundation.py backend/tests/test_model_asset_catalog.py backend/tests/test_sprint121_real_data_detection_qa_smoke.py backend/tests/test_sprint122_raster_upload_metadata_mapping.py -q` passed: 20 tests. +- RED: `python -m pytest backend/tests/test_raster_operations_service.py::test_raster_tile_returns_manifest_payload -q` failed because the tile manifest had no `crs`. +- `python -m pytest backend/tests/test_raster_operations_service.py::test_raster_tile_returns_manifest_payload -q` passed. +- `python -m compileall backend/app` passed. +- `bash scripts/run_readiness_check.sh` passed: 387 backend tests, Alembic head check, frontend typecheck/build and shell syntax checks. +- Tower deploy from commit `71c2cd9` passed with `GEOINTEL_INSTALL_AI=true`. +- Deploy-time live migration smoke passed; PostGIS reported `3.6 USE_GEOS=1 USE_PROJ=1 USE_STATS=1` and Alembic head was `202606120900`. +- Deploy-time browser runtime verification passed for `http://192.168.10.150:1202`. +- Real-data smoke after the class-normalization deploy passed and persisted 4 detections: + - project: `cb80638d-dbef-48ac-b19c-cec7c3efc96e` + - raster dataset: `ae0ff76d-70c0-404f-b777-54d14517179a` + - reference dataset: `8ac01b4f-bd6a-4d6a-b625-a0950ae0f3eb` + - analysis run: `7ba34274-411d-45e3-8f54-c37baec598b1` + - quality check: `66907e6a-9ed7-4b0c-976f-5ad1ba9b8b7a` + - detection export: `516d37a3-2305-48a2-a3dd-b56f70eb055e` + - persisted detections used canonical `class_name=building` and preserved `model_class_name=Building`. Open: -- Full readiness, Tower deploy and repeated live real-data smoke still need to be run for this pass. +- Redeploy the tile-CRS manifest fix and rerun the real-data smoke to verify Detection GeoJSON coordinates are in EPSG:4326. Limitations: -- This fixes class routing and persistence, not model quality. Thresholds, false positives and reference IoU quality still need calibration on larger local orthophoto samples. +- This fixes class routing, persistence and future tile manifest CRS propagation. Existing tile manifests generated before this fix remain missing CRS and should be regenerated before AI runs. +- Thresholds, false positives and reference IoU quality still need calibration on larger local orthophoto samples. Next recommended pass: -- Redeploy to Tower, rerun the Geel real-data smoke, then inspect persisted detections and QA metrics to choose practical confidence/IoU defaults. +- Redeploy the tile-CRS fix, rerun the Geel real-data smoke, confirm WGS84 Detection GeoJSON coordinates and then calibrate practical confidence/IoU defaults. ## Sprint 122 Real operator data availability and raster metadata fix (2026-07-07) diff --git a/docs/TODO.md b/docs/TODO.md index 1361625a..6eba12a5 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -94,6 +94,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add operator-provided real raster/reference detection + QA workflow smoke. - [x] Validate the configured building model on a real georeferenced Kempen orthophoto/GeoTIFF with persisted reference vectors and QA/QC metrics. - [x] Fix configured-YOLO mixed-case class labels so `Building` model output matches `building` domain filters. +- [x] Persist CRS metadata in raster tile manifests so AI detections can be transformed to WGS84 GeoJSON correctly. - [ ] Calibrate confidence, IoU and model selection against persisted Geel detections and additional local orthophoto/reference samples. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index 36f6bd99..b5adce3f 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -182,7 +182,9 @@ against persisted `vector_features`, and exports the detection run as GeoJSON. It does not seed demo data, enable fixture detections, fetch external data or download model weights. Configured-YOLO model class labels are normalized to lowercase for filtering and persisted detections, while the original model label -is retained in detection provenance. A zero detection count is accepted +is retained in detection provenance. Raster tile manifests generated by the +workflow include source CRS metadata so persisted detection GeoJSON coordinates +can be transformed to WGS84. A zero detection count is accepted operationally only when the selected model genuinely returns no usable detections after class filtering; it must be interpreted as model/data quality evidence rather than as a successful building extraction result.