feat: operationalize Flemish land and nature themes

backend/tests/test_sprint213_thematic_rasters.py

@@ -122,21 +122,33 @@ def raster_bytes(values: np.ndarray, resolution: float, *, nodata: float = -9999
         return memory.read()
 
 
-def test_registry_contains_five_governed_non_water_policy_products() -> None:
+def test_registry_contains_governed_policy_products_including_forest_and_agriculture() -> None:
     products = ThematicRasterAcquisitionService.list_products()
 
     assert [item["key"] for item in products] == [
         "space_occupation_2025",
         "open_space_2022",
+        "forest_land_use_2025",
+        "agricultural_land_use_2025",
         "population_density_2019",
         "node_value_2022",
         "service_level_2022",
     ]
-    assert {item["theme"] for item in products} == {"space_occupation", "open_space", "population", "accessibility", "services"}
+    assert {item["theme"] for item in products} == {
+        "space_occupation",
+        "open_space",
+        "forest",
+        "agriculture",
+        "population",
+        "accessibility",
+        "services",
+    }
     assert {item["native_resolution_m"] for item in products} == {10.0, 100.0}
     assert all(item["coverage_id"].startswith(("lu:", "ni:")) for item in products)
     assert all(item["source_crs"] == "EPSG:31370" for item in products)
     assert all(item["attribution"] and item["license_note"] and item["limitation_message"] for item in products)
+    assert next(item for item in products if item["theme"] == "forest")["included_source_values"] == [12]
+    assert next(item for item in products if item["theme"] == "agriculture")["included_source_values"] == [13, 14]
 
 
 def test_request_is_bounded_allowlisted_and_uses_native_wcs_resolution() -> None:
@@ -520,7 +532,7 @@ def test_api_uses_canonical_envelopes(monkeypatch) -> None:
         app.dependency_overrides.clear()
 
     assert products.status_code == 200 and set(products.json()) == {"data"}
-    assert products.json()["data"]["total"] == 5
+    assert products.json()["data"]["total"] == 7
     assert acquisition.status_code == 200 and set(acquisition.json()) == {"data"}
     assert acquisition.json()["data"]["job_type"] == "raster.thematic.acquire"
     assert selection.status_code == 200 and selection.json()["data"]["theme"] == "population"

backend/tests/test_sprint240_official_flemish_themes.py

@@ -0,0 +1,407 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from types import SimpleNamespace
+from urllib.parse import parse_qs, urlparse
+from uuid import uuid4
+
+import numpy as np
+import pytest
+from fastapi.testclient import TestClient
+from geoalchemy2.shape import from_shape
+from rasterio.io import MemoryFile
+from rasterio.transform import from_origin
+from shapely.geometry import MultiPolygon, Polygon
+
+from app.core.config import Settings
+from app.core.errors import AppError
+from app.db.session import get_db
+from app.main import app
+from app.models import Area, Dataset, Job, Project
+from app.schemas.official_vector import OfficialVectorAcquireRequest
+from app.services.dataset_service import DatasetService
+from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
+from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+class FakeQuery:
+    def __init__(self, result=None):
+        self.result = result
+
+    def filter(self, *_args):
+        return self
+
+    def order_by(self, *_args):
+        return self
+
+    def all(self):
+        return self.result if isinstance(self.result, list) else []
+
+
+class FakeSession:
+    def __init__(self, rows=None, query_result=None):
+        self.rows = rows or {}
+        self.query_result = query_result
+        self.added = []
+
+    def get(self, model, row_id):
+        row = self.rows.get((model, row_id))
+        if row is not None:
+            return row
+        return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
+
+    def add(self, row):
+        self.added.append(row)
+
+    def commit(self):
+        return None
+
+    def rollback(self):
+        return None
+
+    def refresh(self, row):
+        return row
+
+    def query(self, _model):
+        return FakeQuery(self.query_result)
+
+
+class JsonResponse:
+    def __init__(self, payload):
+        self.content = json.dumps(payload).encode("utf-8")
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        return False
+
+    def read(self, size=-1):
+        return self.content if size < 0 else self.content[:size]
+
+
+def request(product_key: str, *, area_id=None) -> OfficialVectorAcquireRequest:
+    return OfficialVectorAcquireRequest(
+        bbox={
+            "min_x": 5.15,
+            "min_y": 51.18,
+            "max_x": 5.17,
+            "max_y": 51.20,
+            "crs": "EPSG:4326",
+        },
+        area_id=area_id,
+        product_key=product_key,
+        force_refresh=True,
+    )
+
+
+def polygon_feature(feature_id: str, *, properties=None) -> dict:
+    return {
+        "type": "Feature",
+        "id": feature_id,
+        "geometry": {
+            "type": "Polygon",
+            "coordinates": [[
+                [5.155, 51.185],
+                [5.175, 51.185],
+                [5.175, 51.195],
+                [5.155, 51.195],
+                [5.155, 51.185],
+            ]],
+        },
+        "properties": properties or {},
+    }
+
+
+def test_product_registries_expose_honest_forest_agriculture_nature_and_soil() -> None:
+    raster = {item["key"]: item for item in ThematicRasterAcquisitionService.list_products()}
+    vector = {item["key"]: item for item in OfficialVectorAcquisitionService.list_products()}
+
+    assert raster["forest_land_use_2025"]["included_source_values"] == [12]
+    assert raster["agricultural_land_use_2025"]["included_source_values"] == [13, 14]
+    assert "geen juridische bosgrens" in raster["forest_land_use_2025"]["limitation_message"].lower()
+    assert "geen alz-perceelaangifte" in raster["agricultural_land_use_2025"]["limitation_message"].lower()
+    assert set(vector) == {"bwk_natura2000_2025", "dov_soil_types"}
+    assert vector["bwk_natura2000_2025"]["authority_level"] == "authoritative"
+    assert vector["dov_soil_types"]["authority_level"] == "authoritative_historical_baseline"
+    assert "1949-1971" in vector["dov_soil_types"]["observation_label"]
+
+
+def test_land_use_classes_are_converted_to_binary_masks_without_nodata_cast_warning() -> None:
+    values = np.asarray([[12.0, 13.0], [14.0, -9999.0]], dtype="float32")
+    with MemoryFile() as source_memory:
+        with source_memory.open(
+            driver="GTiff",
+            width=2,
+            height=2,
+            count=1,
+            dtype="float32",
+            crs="EPSG:31370",
+            transform=from_origin(200_000, 210_020, 10, 10),
+            nodata=-9999.0,
+        ) as source:
+            source.write(values, 1)
+        from pyproj import Transformer
+        to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
+        scope = Polygon([
+            to_wgs84.transform(200_000, 210_000),
+            to_wgs84.transform(200_020, 210_000),
+            to_wgs84.transform(200_020, 210_020),
+            to_wgs84.transform(200_000, 210_020),
+            to_wgs84.transform(200_000, 210_000),
+        ])
+        content, validation = ThematicRasterAcquisitionService._normalize_raster(
+            source_memory.read(),
+            scope,
+            {
+                "product": ThematicRasterAcquisitionService._product("forest_land_use_2025"),
+                "width": 2,
+                "height": 2,
+                "bbox_epsg31370": [200_000, 210_000, 200_020, 210_020],
+            },
+        )
+    with MemoryFile(content) as normalized_memory:
+        with normalized_memory.open() as normalized:
+            output = normalized.read(1, masked=True)
+
+    assert output.compressed().tolist() == [1.0, 0.0, 0.0]
+    assert validation["included_source_values"] == [12]
+    assert validation["source_minimum_value"] == 12.0
+    assert validation["source_maximum_value"] == 14.0
+
+
+def test_bwk_wfs_pagination_clips_geometry_and_preserves_semantics() -> None:
+    product = OfficialVectorAcquisitionService._product("bwk_natura2000_2025")
+    scope_wgs84 = Polygon([
+        (5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
+    ])
+    from shapely.ops import transform
+    from app.services.official_vector_acquisition_service import _TO_LAMBERT72
+
+    scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84)
+    calls = []
+
+    def opener(raw_request, timeout):
+        assert timeout == 180
+        calls.append(raw_request.full_url)
+        query = parse_qs(urlparse(raw_request.full_url).query)
+        assert query["typeNames"] == ["BWK:Bwkhab"]
+        assert query["sortBy"] == ["UIDN"]
+        feature = polygon_feature(
+            "Bwkhab.1",
+            properties={"UIDN": 42, "EVAL": "z", "HAB1": "2310", "PHAB1": 60},
+        )
+        if query.get("startIndex") == ["1"]:
+            return JsonResponse({
+                "type": "FeatureCollection",
+                "numberReturned": 0,
+                "features": [],
+            })
+        return JsonResponse({
+            "type": "FeatureCollection",
+            "numberReturned": 1,
+            "features": [feature],
+        })
+
+    features, transfer = OfficialVectorAcquisitionService._fetch_features(
+        product,
+        scope_wgs84,
+        scope_metric,
+        "bounded_selection",
+        Settings(_env_file=None, OFFICIAL_VECTOR_PAGE_SIZE=1),
+        opener,
+    )
+
+    assert len(calls) == 2
+    assert transfer["reference_truncated"] is False
+    assert features[0]["id"] == "BWK:Bwkhab:42"
+    assert features[0]["properties"]["bwk_evaluation_code"] == "z"
+    assert features[0]["properties"]["natura2000_share_percent"] == 60
+    assert features[0]["properties"]["geometry_clipped_to_selection"] is True
+
+
+def test_bwk_rejects_a_non_https_configured_endpoint_before_network_access() -> None:
+    product = OfficialVectorAcquisitionService._product("bwk_natura2000_2025")
+    scope_wgs84 = Polygon([
+        (5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
+    ])
+    from shapely.ops import transform
+    from app.services.official_vector_acquisition_service import _TO_LAMBERT72
+
+    scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84)
+
+    def opener(_request, timeout):
+        del _request, timeout
+        raise AssertionError("network access must not occur")
+
+    with pytest.raises(AppError) as exc_info:
+        OfficialVectorAcquisitionService._fetch_features(
+            product,
+            scope_wgs84,
+            scope_metric,
+            "bounded_selection",
+            Settings(_env_file=None, BWK_WFS_URL="http://example.invalid/wfs"),
+            opener,
+        )
+
+    assert exc_info.value.code == "OFFICIAL_VECTOR_PROVIDER_INVALID_PAGINATION"
+
+
+def test_dov_wfs_uses_stable_complete_pagination_and_historical_fields() -> None:
+    product = OfficialVectorAcquisitionService._product("dov_soil_types")
+    scope_wgs84 = Polygon([
+        (5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
+    ])
+    from shapely.ops import transform
+    from app.services.official_vector_acquisition_service import _TO_LAMBERT72
+
+    scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84)
+
+    def opener(raw_request, timeout):
+        assert timeout == 180
+        query = parse_qs(urlparse(raw_request.full_url).query)
+        assert query["typeNames"] == ["bodemkaart:bodemtypes"]
+        assert query["sortBy"] == ["gid"]
+        return JsonResponse({
+            "type": "FeatureCollection",
+            "numberMatched": 1,
+            "numberReturned": 1,
+            "features": [polygon_feature(
+                "bodemtypes.7",
+                properties={
+                    "gid": 7,
+                    "Bodemtype": "Zcg",
+                    "Gegeneraliseerde_legende": "Droog zand",
+                    "Drainageklasse": "Matig droog",
+                },
+            )],
+        })
+
+    features, transfer = OfficialVectorAcquisitionService._fetch_features(
+        product,
+        scope_wgs84,
+        scope_metric,
+        "bounded_selection",
+        Settings(_env_file=None),
+        opener,
+    )
+
+    assert transfer["page_count"] == 1
+    assert transfer["candidate_feature_count"] == 1
+    assert features[0]["properties"]["soil_type_code"] == "Zcg"
+    assert features[0]["properties"]["soil_generalized_legend"] == "Droog zand"
+    assert features[0]["properties"]["survey_period"] == "1949-1971"
+
+
+def test_nature_acquisition_persists_only_through_dataset_service(monkeypatch) -> None:
+    project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4()
+    municipality = MultiPolygon([Polygon([
+        (5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
+    ])])
+    db = FakeSession({
+        (Project, project_id): Project(id=project_id, name="Vlaanderen"),
+        (Area, area_id): Area(
+            id=area_id,
+            project_id=project_id,
+            name="Gemeente Mol",
+            geometry=from_shape(municipality, srid=4326),
+        ),
+    })
+    captured = {}
+
+    def opener(_request, timeout):
+        del timeout
+        return JsonResponse({
+            "type": "FeatureCollection",
+            "features": [polygon_feature(
+                "Bwkhab.1",
+                properties={"UIDN": 42, "EVAL": "w", "HAB1": "rbbmr", "PHAB1": 100},
+            )],
+            "links": [],
+        })
+
+    def persist(_db, **kwargs):
+        captured.update(kwargs)
+        dataset = Dataset(
+            id=dataset_id,
+            project_id=project_id,
+            area_id=area_id,
+            name=kwargs["filename"],
+            dataset_type="vector",
+            source=kwargs["source"],
+            dataset_role=kwargs["dataset_role"],
+            source_name=kwargs["source_name"],
+            reference_layer_name=kwargs["reference_layer_name"],
+            observed_at=kwargs["observed_at"],
+            source_version=kwargs["source_version"],
+            source_metadata=kwargs["source_metadata"],
+            provenance_metadata=kwargs["provenance_metadata"],
+            metadata_json={"feature_count": 1},
+            status="ready",
+        )
+        db.rows[(Dataset, dataset_id)] = dataset
+        return SimpleNamespace(id=dataset_id)
+
+    monkeypatch.setattr(DatasetService, "import_vector_bytes", persist)
+    result = OfficialVectorAcquisitionService.acquire(
+        db,
+        project_id,
+        request("bwk_natura2000_2025", area_id=area_id),
+        settings=Settings(_env_file=None),
+        opener=opener,
+    )
+
+    assert result["output_dataset_id"] == str(dataset_id)
+    assert captured["dataset_role"] == "reference"
+    assert captured["source_name"] == "inbo_bwk_natura2000"
+    assert captured["reference_layer_name"] == "nature_value"
+    assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == "nature_mapped_area"
+    assert captured["source_metadata"]["selection_metrics"][4]["is_estimate"] is True
+    assert captured["provenance_metadata"]["reference_truncated"] is False
+    assert json.loads(captured["content"])["features"][0]["properties"]["coverage_scope"] == "municipality"
+
+
+def test_official_vector_routes_and_frontend_use_canonical_backend_path(monkeypatch) -> None:
+    project_id, dataset_id = uuid4(), uuid4()
+    db = FakeSession({(Project, project_id): Project(id=project_id, name="Vlaanderen")})
+    monkeypatch.setattr(
+        OfficialVectorAcquisitionService,
+        "acquire",
+        lambda *_args, **_kwargs: {
+            "output_dataset_id": str(dataset_id),
+            "product_key": "bwk_natura2000_2025",
+            "feature_count": 1,
+        },
+    )
+    app.dependency_overrides[get_db] = lambda: db
+    try:
+        client = TestClient(app)
+        products_response = client.get(
+            f"/api/v1/projects/{project_id}/datasets/official-vector/products"
+        )
+        acquire_response = client.post(
+            f"/api/v1/projects/{project_id}/datasets/official-vector/acquire",
+            json=request("bwk_natura2000_2025").model_dump(mode="json"),
+        )
+    finally:
+        app.dependency_overrides.clear()
+
+    assert products_response.status_code == 200
+    assert set(products_response.json()) == {"data"}
+    assert products_response.json()["data"]["total"] == 2
+    assert acquire_response.status_code == 200
+    assert set(acquire_response.json()) == {"data"}
+    assert acquire_response.json()["data"]["job_type"] == "vector.official.acquire"
+    assert any(isinstance(item, Job) for item in db.added)
+
+    selection_hook = (ROOT / "frontend/src/hooks/useMapThemeSelectionInsights.ts").read_text(encoding="utf-8")
+    catalog_hook = (ROOT / "frontend/src/hooks/useOfficialMapProducts.ts").read_text(encoding="utf-8")
+    workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
+    assert "datasetsApi.acquireOfficialVector" in selection_hook
+    assert "datasetsApi.listOfficialVectorProducts" in catalog_hook
+    assert "officialMapProducts.officialVector" in workspace
+    assert "geo.api.vlaanderen.be" not in workspace