from __future__ import annotations from http.client import IncompleteRead from pathlib import Path from types import SimpleNamespace from uuid import uuid4 import numpy as np import pytest from fastapi.testclient import TestClient from pyproj import Transformer from rasterio.io import MemoryFile from rasterio.transform import from_origin from app.core.config import Settings from app.core.errors import AppError from app.db.session import get_db from app.main import app from app.models import Area, Dataset, Job, Project from app.schemas.thematic_raster import ThematicRasterAcquireRequest, ThematicRasterSelectionRequest from app.schemas.assistant import AssistantQueryRequest from app.services.geo_assistant_service import GeoAssistantService from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService from app.services.dataset_service import DatasetService from app.services.vector_feature_service import VectorFeatureService ROOT = Path(__file__).resolve().parents[2] class FakeQuery: def __init__(self, result=None): self.result = result def filter(self, *_args): return self def order_by(self, *_args): return self def first(self): return self.result def all(self): return self.result if isinstance(self.result, list) else [] class FakeSession: def __init__(self, rows=None, query_result=None): self.rows = rows or {} self.query_result = query_result self.added = [] def get(self, model, row_id): row = self.rows.get((model, row_id)) if row is not None: return row return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None) def add(self, row): self.added.append(row) def commit(self): return None def rollback(self): return None def refresh(self, row): return row def query(self, _model): return FakeQuery(self.query_result) class FakeResponse: def __init__(self, content: bytes): self.content = content self.headers = {"Content-Type": "image/tiff", "Content-Length": str(len(content))} def __enter__(self): return self def __exit__(self, *_args): return None def read(self, limit: int): return self.content[:limit] class IncompleteResponse(FakeResponse): def read(self, limit: int): raise IncompleteRead(self.content[:limit]) def payload(product_key: str = "space_occupation_2025", *, side_m: float = 1000.0) -> ThematicRasterAcquireRequest: transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True) min_x, min_y = transformer.transform(200_000, 210_000) max_x, max_y = transformer.transform(200_000 + side_m, 210_000 + side_m) return ThematicRasterAcquireRequest( bbox={"min_x": min_x, "min_y": min_y, "max_x": max_x, "max_y": max_y, "crs": "EPSG:4326"}, product_key=product_key, force_refresh=True, ) def raster_bytes(values: np.ndarray, resolution: float, *, nodata: float = -9999.0) -> bytes: with MemoryFile() as memory: with memory.open( driver="GTiff", width=values.shape[1], height=values.shape[0], count=1, dtype=str(values.dtype), crs="EPSG:31370", transform=from_origin(200_000, 210_000 + values.shape[0] * resolution, resolution, resolution), nodata=nodata, ) as output: output.write(values, 1) return memory.read() def test_registry_contains_governed_policy_products_including_forest_and_agriculture() -> None: products = ThematicRasterAcquisitionService.list_products() assert [item["key"] for item in products] == [ "space_occupation_2025", "open_space_2022", "forest_land_use_2025", "agricultural_land_use_2025", "population_density_2019", "node_value_2022", "service_level_2022", ] assert {item["theme"] for item in products} == { "space_occupation", "open_space", "forest", "agriculture", "population", "accessibility", "services", } assert {item["native_resolution_m"] for item in products} == {10.0, 100.0} assert all(item["coverage_id"].startswith(("lu:", "ni:")) for item in products) assert all(item["source_crs"] == "EPSG:31370" for item in products) assert all(item["attribution"] and item["license_note"] and item["limitation_message"] for item in products) assert next(item for item in products if item["theme"] == "forest")["included_source_values"] == [12] assert next(item for item in products if item["theme"] == "agriculture")["included_source_values"] == [13, 14] def test_request_is_bounded_allowlisted_and_uses_native_wcs_resolution() -> None: settings = Settings(_env_file=None) prepared = ThematicRasterAcquisitionService._prepared_request(payload("population_density_2019"), settings) url = ThematicRasterAcquisitionService._wcs_request_url(settings, prepared["product"], tuple(prepared["bbox_epsg31370"])) assert "VERSION=1.0.0" in url assert "COVERAGE=ni%3Ani_inw_ha_vlaa_2019" in url assert "RESX=100" in url and "RESY=100" in url assert prepared["width"] * prepared["height"] <= settings.thematic_raster_max_pixels with pytest.raises(AppError) as exc_info: ThematicRasterAcquisitionService._prepared_request(payload("arbitrary_remote_layer"), settings) assert exc_info.value.code == "THEMATIC_RASTER_PRODUCT_NOT_SUPPORTED" def test_complete_kempen_scope_fits_the_tiled_thematic_guardrails() -> None: settings = Settings(_env_file=None) request = ThematicRasterAcquireRequest( bbox={ "min_x": 4.59723873, "min_y": 51.01047967, "max_x": 5.26224853, "max_y": 51.50511313, "crs": "EPSG:4326", }, product_key="space_occupation_2025", ) prepared = ThematicRasterAcquisitionService._prepared_request(request, settings) assert prepared["width"] * prepared["height"] <= 30_000_000 assert len(ThematicRasterAcquisitionService._tile_bounds(prepared)) > 1 with pytest.raises(AppError) as exc_info: ThematicRasterAcquisitionService._prepared_request(payload(side_m=61_000.0), settings) assert exc_info.value.code == "THEMATIC_RASTER_SELECTION_TOO_LARGE" def test_coverage_scope_only_labels_named_municipality_areas_as_municipality() -> None: project_id = uuid4() municipality_id = uuid4() region_id = uuid4() db = FakeSession({ (Area, municipality_id): Area(id=municipality_id, project_id=project_id, name="Gemeente Mol"), (Area, region_id): Area(id=region_id, project_id=project_id, name="Vlaanderen"), }) assert ThematicRasterAcquisitionService._coverage_scope(db, municipality_id) == "municipality" assert ThematicRasterAcquisitionService._coverage_scope(db, region_id) == "bounded_selection" assert ThematicRasterAcquisitionService._coverage_scope(db, None) == "bounded_selection" def test_wcs_fetch_retries_an_incomplete_tile_without_accepting_partial_bytes(monkeypatch) -> None: content = b"II*\x00complete-geotiff" responses = [IncompleteResponse(content), FakeResponse(content)] attempts = 0 def opener(*_args, **_kwargs): nonlocal attempts response = responses[attempts] attempts += 1 return response monkeypatch.setattr("app.services.thematic_raster_acquisition_service.time.sleep", lambda _seconds: None) result, content_type = ThematicRasterAcquisitionService._fetch( "https://example.invalid/wcs", Settings(_env_file=None), opener, ) assert attempts == 2 assert result == content assert content_type == "image/tiff" def test_wcs_fetch_fails_closed_after_bounded_incomplete_tile_retries(monkeypatch) -> None: attempts = 0 def opener(*_args, **_kwargs): nonlocal attempts attempts += 1 return IncompleteResponse(b"II*\x00partial") monkeypatch.setattr("app.services.thematic_raster_acquisition_service.time.sleep", lambda _seconds: None) with pytest.raises(AppError) as exc_info: ThematicRasterAcquisitionService._fetch( "https://example.invalid/wcs", Settings(_env_file=None), opener, ) assert attempts == ThematicRasterAcquisitionService.WCS_FETCH_ATTEMPTS assert exc_info.value.code == "THEMATIC_RASTER_PROVIDER_UNAVAILABLE" assert exc_info.value.details["attempts"] == 3 def test_binary_and_normalized_products_fail_closed_on_invalid_values() -> None: binary = ThematicRasterAcquisitionService._product("space_occupation_2025") score = ThematicRasterAcquisitionService._product("service_level_2022") with pytest.raises(AppError, match="Binary"): ThematicRasterAcquisitionService._validate_values(np.asarray([0.0, 2.0]), binary) with pytest.raises(AppError, match="0-1"): ThematicRasterAcquisitionService._validate_values(np.asarray([0.2, 1.2]), score) def test_acquisition_clips_validates_and_delegates_persistence(monkeypatch) -> None: project_id, output_dataset_id = uuid4(), uuid4() db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}) content = raster_bytes(np.ones((100, 100), dtype="float32"), 10.0) captured: dict = {} def fake_import(_db, **kwargs): captured.update(kwargs) return SimpleNamespace(id=output_dataset_id) monkeypatch.setattr(DatasetService, "import_raster_bytes", fake_import) result = ThematicRasterAcquisitionService.acquire( db, project_id, payload(side_m=1000.0), settings=Settings(_env_file=None), opener=lambda *_args, **_kwargs: FakeResponse(content), ) assert result["output_dataset_id"] == str(output_dataset_id) assert captured["source_name"] == ThematicRasterAcquisitionService.PROVIDER assert captured["source_metadata"]["product_key"] == "space_occupation_2025" assert captured["source_metadata"]["metric_kind"] == "binary_area" assert captured["source_metadata"]["valid_pixel_count"] > 9_800 assert captured["provenance_metadata"]["acquisition"] == "explicit_bounded_tiled_wcs_coverage" assert len(captured["provenance_metadata"]["normalized_sha256"]) == 64 def test_binary_area_analysis_returns_hectares_and_share(tmp_path) -> None: project_id, dataset_id = uuid4(), uuid4() values = np.zeros((10, 10), dtype="float32") values[:, :5] = 1.0 path = tmp_path / "space.tif" path.write_bytes(raster_bytes(values, 10.0)) dataset = Dataset( id=dataset_id, project_id=project_id, name="space.tif", dataset_type="raster", source="official", source_name=ThematicRasterAcquisitionService.PROVIDER, source_metadata={"product_key": "space_occupation_2025", "coverage_id": "lu:lu_ruibes_vlaa_2025_v3"}, status="ready", storage_path=str(path), ) db = FakeSession({(Dataset, dataset_id): dataset}) result = ThematicRasterAnalysisService.analyze( db, project_id, dataset_id, ThematicRasterSelectionRequest(bbox=payload(side_m=100.0).bbox), ) metrics = {item["metric_key"]: item for item in result["summary"]["metrics"]} assert result["valid_cell_count"] == 100 assert metrics["space_occupation_area_ha"]["metric_value"] == pytest.approx(0.5) assert metrics["space_occupation_share_pct"]["metric_value"] == pytest.approx(50.0) assert "object_count" in result["unsupported_metrics"] def test_population_analysis_sums_one_hectare_density_cells_without_claiming_current_counts(tmp_path) -> None: project_id, dataset_id = uuid4(), uuid4() values = np.asarray([[10.0, 20.0], [30.0, 40.0]], dtype="float32") path = tmp_path / "population.tif" path.write_bytes(raster_bytes(values, 100.0)) dataset = Dataset( id=dataset_id, project_id=project_id, name="population.tif", dataset_type="raster", source="official", source_name=ThematicRasterAcquisitionService.PROVIDER, source_metadata={"product_key": "population_density_2019", "coverage_id": "ni:ni_inw_ha_vlaa_2019"}, status="ready", storage_path=str(path), ) db = FakeSession({(Dataset, dataset_id): dataset}) result = ThematicRasterAnalysisService.analyze( db, project_id, dataset_id, ThematicRasterSelectionRequest(bbox=payload("population_density_2019", side_m=200.0).bbox), ) metrics = {item["metric_key"]: item for item in result["summary"]["metrics"]} assert metrics["estimated_inhabitants"]["metric_value"] == pytest.approx(100.0) assert metrics["population_density_mean_per_ha"]["metric_value"] == pytest.approx(25.0) assert metrics["estimated_inhabitants"]["is_estimate"] is True assert "current_population" in result["unsupported_metrics"] def test_assistant_context_receives_persisted_thematic_metrics(tmp_path) -> None: project_id, dataset_id = uuid4(), uuid4() values = np.asarray([[10.0, 20.0], [30.0, 40.0]], dtype="float32") path = tmp_path / "assistant-population.tif" path.write_bytes(raster_bytes(values, 100.0)) project = Project(id=project_id, name="Mol", region="Mol") dataset = Dataset( id=dataset_id, project_id=project_id, name="population.tif", dataset_type="raster", source="official", source_name=ThematicRasterAcquisitionService.PROVIDER, source_metadata={"product_key": "population_density_2019", "coverage_id": "ni:ni_inw_ha_vlaa_2019"}, status="ready", storage_path=str(path), ) db = FakeSession({(Project, project_id): project, (Dataset, dataset_id): dataset}, query_result=[dataset]) context, metrics, _series, dataset_ids, _warnings, _scope = GeoAssistantService(Settings(_env_file=None))._build_context( db, project_id=project_id, payload=AssistantQueryRequest(question="Hoeveel inwoners?", bbox=payload("population_density_2019", side_m=200.0).bbox), ) assert any(metric.theme == "population" and metric.label.startswith("Geraamd aantal") for metric in metrics) assert dataset_id in dataset_ids assert context["rules"]["thematic_policy_rasters_available"] is True def test_assistant_context_skips_unrequested_expensive_themes(monkeypatch) -> None: project_id = uuid4() soil_id, agriculture_id = uuid4(), uuid4() population_id, space_id = uuid4(), uuid4() project = Project(id=project_id, name="Kempen", region="Kempen") datasets = [ Dataset( id=soil_id, project_id=project_id, name="soil.geojson", dataset_type="vector", source="official", source_name="dov", source_metadata={"theme": "soil"}, status="ready", ), Dataset( id=agriculture_id, project_id=project_id, name="agriculture.geojson", dataset_type="vector", source="official", source_name="lv", source_metadata={"theme": "agriculture"}, status="ready", ), Dataset( id=population_id, project_id=project_id, name="population.tif", dataset_type="raster", source="official", source_name=ThematicRasterAcquisitionService.PROVIDER, source_metadata={"product_key": "population_density_2019"}, status="ready", ), Dataset( id=space_id, project_id=project_id, name="space.tif", dataset_type="raster", source="official", source_name=ThematicRasterAcquisitionService.PROVIDER, source_metadata={"product_key": "space_occupation_2025"}, status="ready", ), ] db = FakeSession({(Project, project_id): project}, query_result=datasets) summarized: list = [] analyzed: list = [] def summarize(_db, *, dataset, **_kwargs): summarized.append(dataset.id) return { "metric_label": "Gekarteerde bodemoppervlakte", "metric_value": 12.5, "metric_unit": "ha", "is_estimate": False, "warning": "Historische bodemkaart", } def analyze(_db, _project_id, dataset_id, _payload, **_kwargs): analyzed.append(dataset_id) return { "theme": "population", "summary": { "metrics": [ { "metric_label": "Geraamd aantal inwoners (2019)", "metric_value": 100.0, "metric_unit": "inwoners", "is_estimate": True, } ] }, "unsupported_metrics": ["current_population"], "limitation_message": "Rasterraming", } monkeypatch.setattr(VectorFeatureService, "summarize_features_by_bbox", summarize) monkeypatch.setattr(ThematicRasterAnalysisService, "analyze", analyze) context, metrics, _series, dataset_ids, _warnings, _scope = GeoAssistantService(Settings(_env_file=None))._build_context( db, project_id=project_id, payload=AssistantQueryRequest( question="Hoeveel inwoners zijn er en welke bodemtypes komen voor?", bbox=payload("population_density_2019", side_m=200.0).bbox, ), ) assert summarized == [soil_id] assert analyzed == [population_id] assert {metric.theme for metric in metrics} == {"soil", "population"} assert set(dataset_ids) == {soil_id, population_id} assert context["scope"]["requested_themes"] == ["population", "soil"] def test_index_renderer_returns_browser_png(tmp_path) -> None: project_id, dataset_id = uuid4(), uuid4() values = np.linspace(0.1, 4.0, 100, dtype="float32").reshape((10, 10)) path = tmp_path / "node.tif" path.write_bytes(raster_bytes(values, 100.0)) dataset = Dataset( id=dataset_id, project_id=project_id, name="node.tif", dataset_type="raster", source="official", source_name=ThematicRasterAcquisitionService.PROVIDER, source_metadata={ "product_key": "node_value_2022", "coverage_id": "lu:lu_knptw_ha_2022_v3", "render_min_value": 0.1, "render_max_value": 4.0, }, status="ready", storage_path=str(path), ) db = FakeSession({(Dataset, dataset_id): dataset}) assert ThematicRasterAnalysisService.render_png(db, project_id, dataset_id).startswith(b"\x89PNG\r\n\x1a\n") def test_api_uses_canonical_envelopes(monkeypatch) -> None: project_id, dataset_id = uuid4(), uuid4() db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}) monkeypatch.setattr( ThematicRasterAcquisitionService, "acquire", lambda *_args, **_kwargs: {"output_dataset_id": str(dataset_id), "provider": ThematicRasterAcquisitionService.PROVIDER}, ) monkeypatch.setattr( ThematicRasterAnalysisService, "analyze", lambda *_args, **_kwargs: { "dataset_id": str(dataset_id), "product_key": "population_density_2019", "theme": "population", "metric_kind": "population_density", "selection_bbox": payload().bbox.model_dump(), "selected_cell_count": 10, "valid_cell_count": 10, "coverage_ratio": 1.0, "resolution_m": 100.0, "observation_year": 2019, "summary": { "metric_label": "Geraamd aantal inwoners", "metric_value": 10.0, "metric_unit": "inwoners", "aggregation_method": "sum_density_cells", "primary_metric_key": "estimated_inhabitants", "metrics": [], }, "unsupported_metrics": ["current_population"], "limitation_message": "2019 density estimate.", "generated_at": "2026-07-18T00:00:00Z", }, ) app.dependency_overrides[get_db] = lambda: db try: client = TestClient(app) products = client.get(f"/api/v1/projects/{project_id}/datasets/thematic-raster/products") acquisition = client.post( f"/api/v1/projects/{project_id}/datasets/thematic-raster/acquire", json=payload().model_dump(mode="json"), ) selection = client.post( f"/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/thematic/select", json={"bbox": payload().bbox.model_dump()}, ) finally: app.dependency_overrides.clear() assert products.status_code == 200 and set(products.json()) == {"data"} assert products.json()["data"]["total"] == 7 assert acquisition.status_code == 200 and set(acquisition.json()) == {"data"} assert acquisition.json()["data"]["job_type"] == "raster.thematic.acquire" assert selection.status_code == 200 and selection.json()["data"]["theme"] == "population" assert any(isinstance(item, Job) for item in db.added)