from __future__ import annotations from datetime import UTC, datetime from pathlib import Path from uuid import uuid4 import numpy as np import pytest import rasterio from fastapi.testclient import TestClient from geoalchemy2.shape import from_shape from pyproj import Transformer from rasterio.io import MemoryFile from rasterio.transform import from_origin from shapely.geometry import MultiPolygon, box from app.core.config import Settings from app.core.errors import AppError from app.db.session import get_db from app.main import app from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot from app.schemas.orthophoto import OrthophotoAcquireRequest from app.services.dataset_consumption_gate_service import DatasetConsumptionGate from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService from tests.frontend_contract import read_feature ROOT = Path(__file__).resolve().parents[2] class FakeSession: def __init__(self, rows: dict[tuple[type, object], object] | None = None, query_result=None): self.rows = rows or {} self.query_result = query_result self.added: list[object] = [] def get(self, model, row_id): row = self.rows.get((model, row_id)) if row is not None: return row return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None) def add(self, row): self.added.append(row) def flush(self): # The governed importer persists source identities and immutable # snapshots before the Dataset. Mirror the database-generated UUIDs # so this harness exercises that Phase 2 path rather than the legacy # no-registry fallback. for row in self.added: if getattr(row, "id", None) is None: row.id = uuid4() def commit(self): return None def rollback(self): return None def refresh(self, row): return row def query(self, model): rows = [ row for (row_model, _row_id), row in self.rows.items() if row_model is model and isinstance(row, model) ] rows.extend(row for row in self.added if isinstance(row, model)) if isinstance(self.query_result, model): rows.append(self.query_result) elif isinstance(self.query_result, list): rows.extend(row for row in self.query_result if isinstance(row, model)) return FakeQuery(rows) class FakeQuery: def __init__(self, results): self.results = list(results) def filter(self, *_args): return self def order_by(self, *_args): return self def first(self): return self.results[0] if self.results else None def one_or_none(self): return self.first() class FakeImageResponse: def __init__(self, content: bytes): self.content = content self.headers = { "Content-Type": "image/tiff", "Content-Length": str(len(content)), } def __enter__(self): return self def __exit__(self, *_args): return None def read(self, limit: int) -> bytes: return self.content[:limit] def _selection_payload( *, side_m: float = 512.0, force_refresh: bool = True, area_id=None, product_key: str = "most_recent", resolution_m: float | None = None, ) -> OrthophotoAcquireRequest: west, south = 199_000.0, 210_000.0 transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True) min_lon, min_lat = transformer.transform(west, south) max_lon, max_lat = transformer.transform(west + side_m, south + side_m) return OrthophotoAcquireRequest( bbox={ "min_x": min_lon, "min_y": min_lat, "max_x": max_lon, "max_y": max_lat, "crs": "EPSG:4326", }, area_id=area_id, product_key=product_key, force_refresh=force_refresh, resolution_m=resolution_m, ) def _source_tiff(width: int, height: int) -> bytes: pixels = np.zeros((3, height, width), dtype=np.uint8) pixels[0, :, :] = 92 pixels[1, :, :] = 126 pixels[2, :, :] = 84 with MemoryFile() as memory: with memory.open( driver="GTiff", width=width, height=height, count=3, dtype="uint8", transform=from_origin(0, height, 1, 1), ) as output: output.write(pixels) return memory.read() def test_orthophoto_request_is_bounded_and_uses_official_wms_contract() -> None: settings = Settings(_env_file=None) prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(), settings) # A north-up WGS84 rectangle becomes slightly wider after the bounded # EPSG:31370 transform; the service must still keep it near the requested scale. assert 500 <= prepared["width"] <= 540 assert 500 <= prepared["height"] <= 540 assert prepared["params"]["CRS"] == "EPSG:31370" assert prepared["params"]["LAYERS"] == "Ortho" assert "geo.api.vlaanderen.be/OMWRGBMRVL/wms" in prepared["request_url"] assert len(prepared["request_hash"]) == 64 def test_training_request_can_use_native_resolution_but_not_oversample_source() -> None: settings = Settings(_env_file=None) prepared = OrthophotoAcquisitionService._prepared_request( _selection_payload(product_key="wallonia_latest", resolution_m=0.25), settings ) assert 2_000 <= prepared["width"] <= 2_120 assert prepared["resolution_m"] == 0.25 with pytest.raises(AppError) as exc_info: OrthophotoAcquisitionService._prepared_request( _selection_payload(product_key="wallonia_latest", resolution_m=0.1), settings ) assert exc_info.value.code == "ORTHOPHOTO_RESOLUTION_EXCEEDS_SOURCE" def test_orthophoto_product_registry_exposes_only_governed_official_layers() -> None: settings = Settings(_env_file=None) products = OrthophotoAcquisitionService.list_products(settings) keys = [item["key"] for item in products] assert keys[0] == "most_recent" assert {"2025", "2012", "2008_2011", "2000_2003", "1979_1990", "1971"}.issubset(keys) assert next(item for item in products if item["key"] == "most_recent")["supports_detection"] is True detection_keys = {item["key"] for item in products if item["supports_detection"]} assert {"most_recent", "wallonia_latest", "wallonia_2024", "wallonia_2023", "brussels_latest", "brussels_2025", "2025"} <= detection_keys by_key = {item["key"]: item for item in products} assert by_key["wallonia_latest"]["provider"] == "spw_orthophoto" assert by_key["wallonia_latest"]["coverage_zone"] == "wallonia" assert by_key["brussels_latest"]["provider"] == "urbis_orthophoto" assert by_key["brussels_latest"]["coverage_zone"] == "brussels" @pytest.mark.parametrize( ("product_key", "provider", "layer", "coverage_zone"), [ ("wallonia_latest", "spw_orthophoto", "0", "wallonia"), ("brussels_latest", "urbis_orthophoto", "Ortho", "brussels"), ], ) def test_regional_orthophoto_products_bind_provider_and_governed_scope( tmp_path, product_key: str, provider: str, layer: str, coverage_zone: str ) -> None: project_id = uuid4() payload = _selection_payload(product_key=product_key) scope = Area( id=uuid4(), project_id=project_id, name="Wallonia" if coverage_zone == "wallonia" else "Brussels-Capital Region", geometry=from_shape( MultiPolygon( [ box( payload.bbox.min_x - 0.01, payload.bbox.min_y - 0.01, payload.bbox.max_x + 0.01, payload.bbox.max_y + 0.01, ) ] ), srid=4326, ), ) db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")}, query_result=scope) settings = Settings(_env_file=None, storage_root=str(tmp_path), orthophoto_resolution_m=1.0) prepared = OrthophotoAcquisitionService._prepared_request(payload, settings) result = OrthophotoAcquisitionService.acquire( db, project_id, payload, settings=settings, opener=lambda *_args, **_kwargs: FakeImageResponse( _source_tiff(prepared["width"], prepared["height"]) ), ) dataset = next(row for row in db.added if isinstance(row, Dataset)) source = next(row for row in db.added if isinstance(row, SourceRegistry)) snapshot = next(row for row in db.added if isinstance(row, SourceSnapshot)) assert result["provider"] == provider assert result["layer"] == layer assert dataset.source_name == provider assert dataset.source_metadata["coverage_zone"] == coverage_zone assert dataset.source_metadata["license_note"] assert dataset.provenance_metadata["request_url"].startswith(prepared["product"].wms_url) assert dataset.source_registry_id == source.id assert dataset.source_snapshot_id == snapshot.id assert dataset.validation_status == "passed" assert dataset.provenance_status == "complete" assert dataset.quarantine_status == "not_quarantined" assert snapshot.source_registry_id == source.id assert snapshot.checksum_sha256 == dataset.checksum_sha256 assert snapshot.ingest_status == "ingested" assert snapshot.freshness_status == "current" dataset.source_registry = source dataset.source_snapshot = snapshot assert DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference").eligible is True prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings) assert prepared["params"]["LAYERS"] == "OKZPAN71VL" assert prepared["wms_url"] == "https://geo.api.vlaanderen.be/OKZ/wms" with pytest.raises(AppError) as exc_info: OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="arbitrary-layer"), settings) assert exc_info.value.code == "ORTHOPHOTO_PRODUCT_NOT_SUPPORTED" @pytest.mark.parametrize( ("side_m", "expected_code"), [(64.0, "ORTHOPHOTO_SELECTION_TOO_SMALL"), (1_200.0, "ORTHOPHOTO_SELECTION_TOO_LARGE")], ) def test_orthophoto_request_rejects_unsafe_selection_sizes(side_m: float, expected_code: str) -> None: with pytest.raises(AppError) as exc_info: OrthophotoAcquisitionService._prepared_request(_selection_payload(side_m=side_m), Settings(_env_file=None)) assert exc_info.value.code == expected_code def test_orthophoto_acquisition_persists_georeferenced_raster_and_provenance(tmp_path) -> None: project_id = uuid4() area_id = uuid4() payload = _selection_payload(area_id=area_id) area_geometry = MultiPolygon( [ box( payload.bbox.min_x - 0.01, payload.bbox.min_y - 0.01, payload.bbox.max_x + 0.01, payload.bbox.max_y + 0.01, ) ] ) db = FakeSession( { (Project, project_id): Project(id=project_id, name="Mol operationele werkruimte"), (Area, area_id): Area( id=area_id, project_id=project_id, name="Gemeente Mol", geometry=from_shape(area_geometry, srid=4326), ), } ) settings = Settings(_env_file=None, storage_root=str(tmp_path), orthophoto_resolution_m=1.0) prepared = OrthophotoAcquisitionService._prepared_request(payload, settings) response = FakeImageResponse(_source_tiff(prepared["width"], prepared["height"])) result = OrthophotoAcquisitionService.acquire( db, project_id, payload, settings=settings, opener=lambda *_args, **_kwargs: response, ) datasets = [row for row in db.added if isinstance(row, Dataset)] versions = [row for row in db.added if isinstance(row, DatasetVersion)] assert len(datasets) == 1 assert len(versions) == 1 dataset = datasets[0] source = next(row for row in db.added if isinstance(row, SourceRegistry)) snapshot = next(row for row in db.added if isinstance(row, SourceSnapshot)) assert result["output_dataset_id"] == str(dataset.id) assert result["reused"] is False assert dataset.project_id == project_id assert dataset.area_id == area_id assert dataset.dataset_type == "raster" assert dataset.dataset_role == "source" assert dataset.source_name == "digitaal_vlaanderen_orthophoto" assert dataset.crs == "EPSG:31370" assert dataset.provenance_metadata["acquisition"] == "explicit_bounded_map_selection" assert dataset.provenance_metadata["request_hash"] == prepared["request_hash"] assert dataset.source_registry_id == source.id assert dataset.source_snapshot_id == snapshot.id assert dataset.validation_status == "passed" assert dataset.provenance_status == "complete" assert dataset.lineage_status == "not_applicable" assert dataset.quarantine_status == "not_quarantined" assert snapshot.source_registry_id == source.id assert snapshot.checksum_sha256 == dataset.checksum_sha256 assert snapshot.freshness_status == "current" assert dataset.source_metadata["attribution"].startswith("Bron: Orthofotomozaiek Vlaanderen") assert dataset.storage_path is not None with rasterio.open(dataset.storage_path) as stored: assert stored.crs.to_epsg() == 31370 assert stored.count == 3 assert stored.width == prepared["width"] assert stored.height == prepared["height"] assert list(stored.bounds) == pytest.approx(prepared["bbox_epsg31370"], abs=0.01) def test_orthophoto_acquisition_rejects_selection_outside_persisted_area() -> None: project_id = uuid4() area_id = uuid4() payload = _selection_payload(area_id=area_id) db = FakeSession( { (Project, project_id): Project(id=project_id, name="Mol"), (Area, area_id): Area( id=area_id, project_id=project_id, name="Unrelated area", geometry=from_shape(MultiPolygon([box(3.0, 50.0, 3.1, 50.1)]), srid=4326), ), } ) with pytest.raises(AppError) as exc_info: OrthophotoAcquisitionService.acquire(db, project_id, payload, settings=Settings(_env_file=None)) assert exc_info.value.code == "ORTHOPHOTO_SELECTION_OUTSIDE_AREA" def test_orthophoto_acquisition_reuses_fresh_exact_request_without_provider_call(tmp_path) -> None: project_id = uuid4() payload = _selection_payload(force_refresh=False) prepared = OrthophotoAcquisitionService._prepared_request(payload, Settings(_env_file=None)) stored_path = tmp_path / "cached.tif" stored_path.write_bytes(b"persisted") cached = Dataset( id=uuid4(), project_id=project_id, name=f"orthofoto_most_recent_{prepared['request_hash'][:12]}.tif", dataset_type="raster", source="Digitaal Vlaanderen", source_name="digitaal_vlaanderen_orthophoto", status="ready", storage_path=str(stored_path), imported_at=datetime.now(UTC), ) db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}, query_result=cached) result = OrthophotoAcquisitionService.acquire( db, project_id, payload, settings=Settings(_env_file=None), opener=lambda *_args, **_kwargs: pytest.fail("fresh cached request must not call the provider"), ) assert result["output_dataset_id"] == str(cached.id) assert result["reused"] is True assert db.added == [] def test_orthophoto_provider_rejects_non_image_response() -> None: response = FakeImageResponse(b"invalid layer") response.headers["Content-Type"] = "text/xml" with pytest.raises(AppError) as exc_info: OrthophotoAcquisitionService._fetch( "https://geo.api.vlaanderen.be/OMWRGBMRVL/wms", Settings(_env_file=None), opener=lambda *_args, **_kwargs: response, ) assert exc_info.value.code == "ORTHOPHOTO_PROVIDER_INVALID_RESPONSE" def test_historical_orthophoto_persists_temporal_product_provenance(tmp_path) -> None: project_id = uuid4() payload = _selection_payload(product_key="2020") db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}) settings = Settings(_env_file=None, storage_root=str(tmp_path), orthophoto_resolution_m=1.0) prepared = OrthophotoAcquisitionService._prepared_request(payload, settings) response = FakeImageResponse(_source_tiff(prepared["width"], prepared["height"])) result = OrthophotoAcquisitionService.acquire( db, project_id, payload, settings=settings, opener=lambda *_args, **_kwargs: response, ) dataset = next(row for row in db.added if isinstance(row, Dataset)) assert result["product_key"] == "2020" assert result["supports_detection"] is False assert dataset.observed_at.year == 2020 assert dataset.temporal_granularity == "year" assert dataset.source_metadata["layer"] == "OMWRGB20VL" assert dataset.source_metadata["product_key"] == "2020" assert dataset.provenance_metadata["spatial_hash"] == prepared["spatial_hash"] def test_persisted_orthophoto_renders_browser_png(tmp_path) -> None: project_id = uuid4() dataset_id = uuid4() path = tmp_path / "ortho.tif" path.write_bytes(_source_tiff(32, 24)) dataset = Dataset( id=dataset_id, project_id=project_id, name="ortho.tif", dataset_type="raster", source="Digitaal Vlaanderen", source_name="digitaal_vlaanderen_orthophoto", status="ready", storage_path=str(path), ) db = FakeSession({(Dataset, dataset_id): dataset}) png = OrthophotoAcquisitionService.render_png(db, project_id, dataset_id) assert png.startswith(b"\x89PNG\r\n\x1a\n") def test_orthophoto_endpoint_returns_canonical_job_envelope(monkeypatch) -> None: project_id = uuid4() output_dataset_id = uuid4() db = FakeSession() monkeypatch.setattr( OrthophotoAcquisitionService, "acquire", lambda *_args, **_kwargs: { "output_dataset_id": str(output_dataset_id), "reused": False, "provider": "digitaal_vlaanderen_orthophoto", }, ) payload = _selection_payload(force_refresh=False).model_dump(mode="json") app.dependency_overrides[get_db] = lambda: db try: response = TestClient(app).post(f"/api/v1/projects/{project_id}/datasets/orthophoto/acquire", json=payload) finally: app.dependency_overrides.clear() assert response.status_code == 200 body = response.json() assert set(body) == {"data"} assert body["data"]["status"] == "success" assert body["data"]["job_type"] == "raster.orthophoto.acquire" assert body["data"]["output_dataset_id"] == str(output_dataset_id) assert body["data"]["result_json"]["provider"] == "digitaal_vlaanderen_orthophoto" assert any(isinstance(row, Job) for row in db.added) def test_orthophoto_product_endpoint_returns_canonical_envelope() -> None: project_id = uuid4() db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}) app.dependency_overrides[get_db] = lambda: db try: response = TestClient(app).get(f"/api/v1/projects/{project_id}/datasets/orthophoto/products") finally: app.dependency_overrides.clear() assert response.status_code == 200 body = response.json() assert set(body) == {"data"} assert body["data"]["total"] == len(body["data"]["items"]) assert body["data"]["items"][0]["key"] == "most_recent" def test_frontend_connects_map_selection_to_existing_detection_and_qa_flows() -> None: app_source = read_feature("shell") hook_source = (ROOT / "frontend" / "src" / "hooks" / "useMapOrthophotoAnalysis.ts").read_text(encoding="utf-8") map_source = read_feature("map_workspace") assert "useMapOrthophotoAnalysis" in app_source assert "onRunOrthophotoAnalysis={mapOrthophotoAnalysis.run}" in app_source assert "datasetsApi.acquireOrthophoto" in hook_source assert "prepareAndRunDetection(datasetId)" in hook_source assert "compareDetectionRunWithReference" in hook_source assert "compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false, iouThreshold)" in app_source assert "MAP_BUILDING_QA_IOU_THRESHOLD = 0.25" in hook_source assert "vervoerregio|operationele grens" in app_source assert "selectionBbox: mapSelectionBbox" in app_source assert "Maak de rechthoek minstens 128 bij 128 meter groot." in hook_source assert "Herken gebouwen" in map_source assert "Officieel luchtbeeld, lokaal AI-model" in map_source def test_unraid_runtime_exposes_bounded_orthophoto_settings() -> None: compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8") runner = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8") template = (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(encoding="utf-8") for name in ("ORTHOPHOTO_ENABLED", "ORTHOPHOTO_WMS_URL", "ORTHOPHOTO_RESOLUTION_M", "ORTHOPHOTO_MAX_SIDE_M"): assert name in compose assert name in runner assert name in template