from __future__ import annotations import json from pathlib import Path from types import SimpleNamespace from urllib.parse import parse_qs, urlparse from uuid import uuid4 import numpy as np import pytest from fastapi.testclient import TestClient from geoalchemy2.shape import from_shape from rasterio.io import MemoryFile from rasterio.transform import from_origin from shapely.geometry import MultiPolygon, Polygon from app.core.config import Settings from app.core.errors import AppError from app.db.session import get_db from app.main import app from app.models import Area, Dataset, Job, Project from app.schemas.official_vector import OfficialVectorAcquireRequest from app.services.dataset_service import DatasetService from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService from tests.frontend_contract import read_feature ROOT = Path(__file__).resolve().parents[2] class FakeQuery: def __init__(self, result=None): self.result = result def filter(self, *_args): return self def order_by(self, *_args): return self def all(self): return self.result if isinstance(self.result, list) else [] class FakeSession: def __init__(self, rows=None, query_result=None): self.rows = rows or {} self.query_result = query_result self.added = [] def get(self, model, row_id): row = self.rows.get((model, row_id)) if row is not None: return row return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None) def add(self, row): self.added.append(row) def commit(self): return None def rollback(self): return None def refresh(self, row): return row def query(self, _model): return FakeQuery(self.query_result) class JsonResponse: def __init__(self, payload): self.content = json.dumps(payload).encode("utf-8") def __enter__(self): return self def __exit__(self, *_args): return False def read(self, size=-1): return self.content if size < 0 else self.content[:size] def request(product_key: str, *, area_id=None) -> OfficialVectorAcquireRequest: return OfficialVectorAcquireRequest( bbox={ "min_x": 5.15, "min_y": 51.18, "max_x": 5.17, "max_y": 51.20, "crs": "EPSG:4326", }, area_id=area_id, product_key=product_key, force_refresh=True, ) def polygon_feature(feature_id: str, *, properties=None) -> dict: return { "type": "Feature", "id": feature_id, "geometry": { "type": "Polygon", "coordinates": [[ [5.155, 51.185], [5.175, 51.185], [5.175, 51.195], [5.155, 51.195], [5.155, 51.185], ]], }, "properties": properties or {}, } def test_product_registries_expose_honest_forest_agriculture_nature_and_soil() -> None: raster = {item["key"]: item for item in ThematicRasterAcquisitionService.list_products()} vector = {item["key"]: item for item in OfficialVectorAcquisitionService.list_products()} assert raster["forest_land_use_2025"]["included_source_values"] == [12] assert raster["agricultural_land_use_2025"]["included_source_values"] == [13, 14] assert "geen juridische bosgrens" in raster["forest_land_use_2025"]["limitation_message"].lower() assert "geen alz-perceelaangifte" in raster["agricultural_land_use_2025"]["limitation_message"].lower() assert { "bwk_natura2000_2025", "dov_soil_types", "spw_picc_buildings", "spw_picc_roads", "spw_picc_waterways", "spw_picc_water_surfaces", "spw_flood_hazard_2021", "urbis_buildings", "urbis_cadastral_parcels", "urbis_street_axes", "urbis_land_cover_blocks", "urbis_forest_parks", "urbis_water_surfaces", } == set(vector) assert vector["bwk_natura2000_2025"]["authority_level"] == "authoritative" assert vector["dov_soil_types"]["authority_level"] == "authoritative_historical_baseline" assert "1949-1971" in vector["dov_soil_types"]["observation_label"] def test_land_use_classes_are_converted_to_binary_masks_without_nodata_cast_warning() -> None: values = np.asarray([[12.0, 13.0], [14.0, -9999.0]], dtype="float32") with MemoryFile() as source_memory: with source_memory.open( driver="GTiff", width=2, height=2, count=1, dtype="float32", crs="EPSG:31370", transform=from_origin(200_000, 210_020, 10, 10), nodata=-9999.0, ) as source: source.write(values, 1) from pyproj import Transformer to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True) scope = Polygon([ to_wgs84.transform(200_000, 210_000), to_wgs84.transform(200_020, 210_000), to_wgs84.transform(200_020, 210_020), to_wgs84.transform(200_000, 210_020), to_wgs84.transform(200_000, 210_000), ]) content, validation = ThematicRasterAcquisitionService._normalize_raster( source_memory.read(), scope, { "product": ThematicRasterAcquisitionService._product("forest_land_use_2025"), "width": 2, "height": 2, "bbox_epsg31370": [200_000, 210_000, 200_020, 210_020], }, ) with MemoryFile(content) as normalized_memory: with normalized_memory.open() as normalized: output = normalized.read(1, masked=True) assert output.compressed().tolist() == [1.0, 0.0, 0.0] assert validation["included_source_values"] == [12] assert validation["source_minimum_value"] == 12.0 assert validation["source_maximum_value"] == 14.0 def test_bwk_wfs_pagination_clips_geometry_and_preserves_semantics() -> None: product = OfficialVectorAcquisitionService._product("bwk_natura2000_2025") scope_wgs84 = Polygon([ (5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18) ]) from shapely.ops import transform from app.services.official_vector_acquisition_service import _TO_LAMBERT72 scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84) calls = [] def opener(raw_request, timeout): assert timeout == 180 calls.append(raw_request.full_url) query = parse_qs(urlparse(raw_request.full_url).query) assert query["typeNames"] == ["BWK:Bwkhab"] assert query["sortBy"] == ["UIDN"] feature = polygon_feature( "Bwkhab.1", properties={"UIDN": 42, "EVAL": "z", "HAB1": "2310", "PHAB1": 60}, ) if query.get("startIndex") == ["1"]: return JsonResponse({ "type": "FeatureCollection", "numberReturned": 0, "features": [], }) return JsonResponse({ "type": "FeatureCollection", "numberReturned": 1, "features": [feature], }) features, transfer = OfficialVectorAcquisitionService._fetch_features( product, scope_wgs84, scope_metric, "bounded_selection", Settings(_env_file=None, OFFICIAL_VECTOR_PAGE_SIZE=1), opener, ) assert len(calls) == 2 assert transfer["reference_truncated"] is False assert features[0]["id"] == "BWK:Bwkhab:42" assert features[0]["properties"]["bwk_evaluation_code"] == "z" assert features[0]["properties"]["natura2000_share_percent"] == 60 assert features[0]["properties"]["geometry_clipped_to_selection"] is True def test_bwk_rejects_a_non_https_configured_endpoint_before_network_access() -> None: product = OfficialVectorAcquisitionService._product("bwk_natura2000_2025") scope_wgs84 = Polygon([ (5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18) ]) from shapely.ops import transform from app.services.official_vector_acquisition_service import _TO_LAMBERT72 scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84) def opener(_request, timeout): del _request, timeout raise AssertionError("network access must not occur") with pytest.raises(AppError) as exc_info: OfficialVectorAcquisitionService._fetch_features( product, scope_wgs84, scope_metric, "bounded_selection", Settings(_env_file=None, BWK_WFS_URL="http://example.invalid/wfs"), opener, ) assert exc_info.value.code == "OFFICIAL_VECTOR_PROVIDER_INVALID_PAGINATION" def test_dov_wfs_uses_stable_complete_pagination_and_historical_fields() -> None: product = OfficialVectorAcquisitionService._product("dov_soil_types") scope_wgs84 = Polygon([ (5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18) ]) from shapely.ops import transform from app.services.official_vector_acquisition_service import _TO_LAMBERT72 scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84) def opener(raw_request, timeout): assert timeout == 180 query = parse_qs(urlparse(raw_request.full_url).query) assert query["typeNames"] == ["bodemkaart:bodemtypes"] assert query["sortBy"] == ["gid"] return JsonResponse({ "type": "FeatureCollection", "numberMatched": 1, "numberReturned": 1, "features": [polygon_feature( "bodemtypes.7", properties={ "gid": 7, "Bodemtype": "Zcg", "Gegeneraliseerde_legende": "Droog zand", "Drainageklasse": "Matig droog", }, )], }) features, transfer = OfficialVectorAcquisitionService._fetch_features( product, scope_wgs84, scope_metric, "bounded_selection", Settings(_env_file=None), opener, ) assert transfer["page_count"] == 1 assert transfer["candidate_feature_count"] == 1 assert features[0]["properties"]["soil_type_code"] == "Zcg" assert features[0]["properties"]["soil_generalized_legend"] == "Droog zand" assert features[0]["properties"]["survey_period"] == "1949-1971" def test_nature_acquisition_persists_only_through_dataset_service(monkeypatch) -> None: project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4() municipality = MultiPolygon([Polygon([ (5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18) ])]) db = FakeSession({ (Project, project_id): Project(id=project_id, name="Vlaanderen"), (Area, area_id): Area( id=area_id, project_id=project_id, name="Gemeente Mol", geometry=from_shape(municipality, srid=4326), ), }) captured = {} def opener(_request, timeout): del timeout return JsonResponse({ "type": "FeatureCollection", "features": [polygon_feature( "Bwkhab.1", properties={"UIDN": 42, "EVAL": "w", "HAB1": "rbbmr", "PHAB1": 100}, )], "links": [], }) def persist(_db, **kwargs): captured.update(kwargs) dataset = Dataset( id=dataset_id, project_id=project_id, area_id=area_id, name=kwargs["filename"], dataset_type="vector", source=kwargs["source"], dataset_role=kwargs["dataset_role"], source_name=kwargs["source_name"], reference_layer_name=kwargs["reference_layer_name"], observed_at=kwargs["observed_at"], source_version=kwargs["source_version"], source_metadata=kwargs["source_metadata"], provenance_metadata=kwargs["provenance_metadata"], metadata_json={"feature_count": 1}, status="ready", ) db.rows[(Dataset, dataset_id)] = dataset return SimpleNamespace(id=dataset_id) monkeypatch.setattr(DatasetService, "import_vector_bytes", persist) result = OfficialVectorAcquisitionService.acquire( db, project_id, request("bwk_natura2000_2025", area_id=area_id), settings=Settings(_env_file=None), opener=opener, ) assert result["output_dataset_id"] == str(dataset_id) assert captured["dataset_role"] == "reference" assert captured["source_name"] == "inbo_bwk_natura2000" assert captured["reference_layer_name"] == "nature_value" assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == "nature_mapped_area" assert captured["source_metadata"]["selection_metrics"][4]["is_estimate"] is True assert captured["provenance_metadata"]["reference_truncated"] is False assert json.loads(captured["content"])["features"][0]["properties"]["coverage_scope"] == "municipality" def test_official_vector_routes_and_frontend_use_canonical_backend_path(monkeypatch) -> None: project_id, dataset_id = uuid4(), uuid4() db = FakeSession({(Project, project_id): Project(id=project_id, name="Vlaanderen")}) monkeypatch.setattr( OfficialVectorAcquisitionService, "acquire", lambda *_args, **_kwargs: { "output_dataset_id": str(dataset_id), "product_key": "bwk_natura2000_2025", "feature_count": 1, }, ) app.dependency_overrides[get_db] = lambda: db try: client = TestClient(app) products_response = client.get( f"/api/v1/projects/{project_id}/datasets/official-vector/products" ) acquire_response = client.post( f"/api/v1/projects/{project_id}/datasets/official-vector/acquire", json=request("bwk_natura2000_2025").model_dump(mode="json"), ) finally: app.dependency_overrides.clear() assert products_response.status_code == 200 assert set(products_response.json()) == {"data"} assert products_response.json()["data"]["total"] == 13 assert acquire_response.status_code == 200 assert set(acquire_response.json()) == {"data"} assert acquire_response.json()["data"]["job_type"] == "vector.official.acquire" assert any(isinstance(item, Job) for item in db.added) selection_hook = read_feature("map_workspace") catalog_hook = read_feature("map_workspace") workspace = read_feature("map_workspace") assert "datasetsApi.acquireOfficialVector" in selection_hook assert "datasetsApi.listOfficialVectorProducts" in catalog_hook assert "officialMapProducts.officialVector" in workspace assert "officialMapProducts.thematic" in workspace assert "result[product.theme] = null" in workspace assert "geo.api.vlaanderen.be" not in workspace