feat: add cross-domain Mol data profile

2026-07-16 03:06:36 +02:00
parent 4aa0d6da24
commit 035ec3b233
39 changed files with 3049 additions and 22 deletions
@@ -0,0 +1,339 @@
+from __future__ import annotations
+
+from pathlib import Path
+from types import SimpleNamespace
+from uuid import uuid4
+
+import numpy as np
+import pytest
+import rasterio
+from fastapi.testclient import TestClient
+from pyproj import Transformer
+from rasterio.io import MemoryFile
+from rasterio.transform import from_origin
+
+from app.core.config import Settings
+from app.core.errors import AppError
+from app.db.session import get_db
+from app.main import app
+from app.models import Dataset, Job, Project
+from app.schemas.thematic_raster import ThematicRasterAcquireRequest, ThematicRasterSelectionRequest
+from app.schemas.assistant import AssistantQueryRequest
+from app.services.geo_assistant_service import GeoAssistantService
+from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
+from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
+from app.services.dataset_service import DatasetService
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+class FakeQuery:
+    def __init__(self, result=None):
+        self.result = result
+
+    def filter(self, *_args):
+        return self
+
+    def order_by(self, *_args):
+        return self
+
+    def first(self):
+        return self.result
+
+    def all(self):
+        return self.result if isinstance(self.result, list) else []
+
+
+class FakeSession:
+    def __init__(self, rows=None, query_result=None):
+        self.rows = rows or {}
+        self.query_result = query_result
+        self.added = []
+
+    def get(self, model, row_id):
+        row = self.rows.get((model, row_id))
+        if row is not None:
+            return row
+        return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
+
+    def add(self, row):
+        self.added.append(row)
+
+    def commit(self):
+        return None
+
+    def rollback(self):
+        return None
+
+    def refresh(self, row):
+        return row
+
+    def query(self, _model):
+        return FakeQuery(self.query_result)
+
+
+class FakeResponse:
+    def __init__(self, content: bytes):
+        self.content = content
+        self.headers = {"Content-Type": "image/tiff", "Content-Length": str(len(content))}
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        return None
+
+    def read(self, limit: int):
+        return self.content[:limit]
+
+
+def payload(product_key: str = "space_occupation_2025", *, side_m: float = 1000.0) -> ThematicRasterAcquireRequest:
+    transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
+    min_x, min_y = transformer.transform(200_000, 210_000)
+    max_x, max_y = transformer.transform(200_000 + side_m, 210_000 + side_m)
+    return ThematicRasterAcquireRequest(
+        bbox={"min_x": min_x, "min_y": min_y, "max_x": max_x, "max_y": max_y, "crs": "EPSG:4326"},
+        product_key=product_key,
+        force_refresh=True,
+    )
+
+
+def raster_bytes(values: np.ndarray, resolution: float, *, nodata: float = -9999.0) -> bytes:
+    with MemoryFile() as memory:
+        with memory.open(
+            driver="GTiff",
+            width=values.shape[1],
+            height=values.shape[0],
+            count=1,
+            dtype=str(values.dtype),
+            crs="EPSG:31370",
+            transform=from_origin(200_000, 210_000 + values.shape[0] * resolution, resolution, resolution),
+            nodata=nodata,
+        ) as output:
+            output.write(values, 1)
+        return memory.read()
+
+
+def test_registry_contains_five_governed_non_water_policy_products() -> None:
+    products = ThematicRasterAcquisitionService.list_products()
+
+    assert [item["key"] for item in products] == [
+        "space_occupation_2025",
+        "open_space_2022",
+        "population_density_2019",
+        "node_value_2022",
+        "service_level_2022",
+    ]
+    assert {item["theme"] for item in products} == {"space_occupation", "open_space", "population", "accessibility", "services"}
+    assert {item["native_resolution_m"] for item in products} == {10.0, 100.0}
+    assert all(item["coverage_id"].startswith(("lu:", "ni:")) for item in products)
+    assert all(item["source_crs"] == "EPSG:31370" for item in products)
+    assert all(item["attribution"] and item["license_note"] and item["limitation_message"] for item in products)
+
+
+def test_request_is_bounded_allowlisted_and_uses_native_wcs_resolution() -> None:
+    settings = Settings(_env_file=None)
+    prepared = ThematicRasterAcquisitionService._prepared_request(payload("population_density_2019"), settings)
+    url = ThematicRasterAcquisitionService._wcs_request_url(settings, prepared["product"], tuple(prepared["bbox_epsg31370"]))
+
+    assert "VERSION=1.0.0" in url
+    assert "COVERAGE=ni%3Ani_inw_ha_vlaa_2019" in url
+    assert "RESX=100" in url and "RESY=100" in url
+    assert prepared["width"] * prepared["height"] <= settings.thematic_raster_max_pixels
+
+    with pytest.raises(AppError) as exc_info:
+        ThematicRasterAcquisitionService._prepared_request(payload("arbitrary_remote_layer"), settings)
+    assert exc_info.value.code == "THEMATIC_RASTER_PRODUCT_NOT_SUPPORTED"
+
+
+def test_binary_and_normalized_products_fail_closed_on_invalid_values() -> None:
+    binary = ThematicRasterAcquisitionService._product("space_occupation_2025")
+    score = ThematicRasterAcquisitionService._product("service_level_2022")
+
+    with pytest.raises(AppError, match="Binary"):
+        ThematicRasterAcquisitionService._validate_values(np.asarray([0.0, 2.0]), binary)
+    with pytest.raises(AppError, match="0-1"):
+        ThematicRasterAcquisitionService._validate_values(np.asarray([0.2, 1.2]), score)
+
+
+def test_acquisition_clips_validates_and_delegates_persistence(monkeypatch) -> None:
+    project_id, output_dataset_id = uuid4(), uuid4()
+    db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
+    content = raster_bytes(np.ones((100, 100), dtype="float32"), 10.0)
+    captured: dict = {}
+
+    def fake_import(_db, **kwargs):
+        captured.update(kwargs)
+        return SimpleNamespace(id=output_dataset_id)
+
+    monkeypatch.setattr(DatasetService, "import_raster_bytes", fake_import)
+    result = ThematicRasterAcquisitionService.acquire(
+        db,
+        project_id,
+        payload(side_m=1000.0),
+        settings=Settings(_env_file=None),
+        opener=lambda *_args, **_kwargs: FakeResponse(content),
+    )
+
+    assert result["output_dataset_id"] == str(output_dataset_id)
+    assert captured["source_name"] == ThematicRasterAcquisitionService.PROVIDER
+    assert captured["source_metadata"]["product_key"] == "space_occupation_2025"
+    assert captured["source_metadata"]["metric_kind"] == "binary_area"
+    assert captured["source_metadata"]["valid_pixel_count"] > 9_800
+    assert captured["provenance_metadata"]["acquisition"] == "explicit_bounded_tiled_wcs_coverage"
+    assert len(captured["provenance_metadata"]["normalized_sha256"]) == 64
+
+
+def test_binary_area_analysis_returns_hectares_and_share(tmp_path) -> None:
+    project_id, dataset_id = uuid4(), uuid4()
+    values = np.zeros((10, 10), dtype="float32")
+    values[:, :5] = 1.0
+    path = tmp_path / "space.tif"
+    path.write_bytes(raster_bytes(values, 10.0))
+    dataset = Dataset(
+        id=dataset_id,
+        project_id=project_id,
+        name="space.tif",
+        dataset_type="raster",
+        source="official",
+        source_name=ThematicRasterAcquisitionService.PROVIDER,
+        source_metadata={"product_key": "space_occupation_2025", "coverage_id": "lu:lu_ruibes_vlaa_2025_v3"},
+        status="ready",
+        storage_path=str(path),
+    )
+    db = FakeSession({(Dataset, dataset_id): dataset})
+    result = ThematicRasterAnalysisService.analyze(
+        db,
+        project_id,
+        dataset_id,
+        ThematicRasterSelectionRequest(bbox=payload(side_m=100.0).bbox),
+    )
+    metrics = {item["metric_key"]: item for item in result["summary"]["metrics"]}
+
+    assert result["valid_cell_count"] == 100
+    assert metrics["space_occupation_area_ha"]["metric_value"] == pytest.approx(0.5)
+    assert metrics["space_occupation_share_pct"]["metric_value"] == pytest.approx(50.0)
+    assert "object_count" in result["unsupported_metrics"]
+
+
+def test_population_analysis_sums_one_hectare_density_cells_without_claiming_current_counts(tmp_path) -> None:
+    project_id, dataset_id = uuid4(), uuid4()
+    values = np.asarray([[10.0, 20.0], [30.0, 40.0]], dtype="float32")
+    path = tmp_path / "population.tif"
+    path.write_bytes(raster_bytes(values, 100.0))
+    dataset = Dataset(
+        id=dataset_id,
+        project_id=project_id,
+        name="population.tif",
+        dataset_type="raster",
+        source="official",
+        source_name=ThematicRasterAcquisitionService.PROVIDER,
+        source_metadata={"product_key": "population_density_2019", "coverage_id": "ni:ni_inw_ha_vlaa_2019"},
+        status="ready",
+        storage_path=str(path),
+    )
+    db = FakeSession({(Dataset, dataset_id): dataset})
+    result = ThematicRasterAnalysisService.analyze(
+        db,
+        project_id,
+        dataset_id,
+        ThematicRasterSelectionRequest(bbox=payload("population_density_2019", side_m=200.0).bbox),
+    )
+    metrics = {item["metric_key"]: item for item in result["summary"]["metrics"]}
+
+    assert metrics["estimated_inhabitants"]["metric_value"] == pytest.approx(100.0)
+    assert metrics["population_density_mean_per_ha"]["metric_value"] == pytest.approx(25.0)
+    assert metrics["estimated_inhabitants"]["is_estimate"] is True
+    assert "current_population" in result["unsupported_metrics"]
+
+
+def test_assistant_context_receives_persisted_thematic_metrics(tmp_path) -> None:
+    project_id, dataset_id = uuid4(), uuid4()
+    values = np.asarray([[10.0, 20.0], [30.0, 40.0]], dtype="float32")
+    path = tmp_path / "assistant-population.tif"
+    path.write_bytes(raster_bytes(values, 100.0))
+    project = Project(id=project_id, name="Mol", region="Mol")
+    dataset = Dataset(
+        id=dataset_id,
+        project_id=project_id,
+        name="population.tif",
+        dataset_type="raster",
+        source="official",
+        source_name=ThematicRasterAcquisitionService.PROVIDER,
+        source_metadata={"product_key": "population_density_2019", "coverage_id": "ni:ni_inw_ha_vlaa_2019"},
+        status="ready",
+        storage_path=str(path),
+    )
+    db = FakeSession({(Project, project_id): project, (Dataset, dataset_id): dataset}, query_result=[dataset])
+    context, metrics, _series, dataset_ids, _warnings, _scope = GeoAssistantService(Settings(_env_file=None))._build_context(
+        db,
+        project_id=project_id,
+        payload=AssistantQueryRequest(question="Hoeveel inwoners?", bbox=payload("population_density_2019", side_m=200.0).bbox),
+    )
+
+    assert any(metric.theme == "population" and metric.label.startswith("Geraamd aantal") for metric in metrics)
+    assert dataset_id in dataset_ids
+    assert context["rules"]["thematic_policy_rasters_available"] is True
+
+
+def test_index_renderer_returns_browser_png(tmp_path) -> None:
+    project_id, dataset_id = uuid4(), uuid4()
+    values = np.linspace(0.1, 4.0, 100, dtype="float32").reshape((10, 10))
+    path = tmp_path / "node.tif"
+    path.write_bytes(raster_bytes(values, 100.0))
+    dataset = Dataset(
+        id=dataset_id,
+        project_id=project_id,
+        name="node.tif",
+        dataset_type="raster",
+        source="official",
+        source_name=ThematicRasterAcquisitionService.PROVIDER,
+        source_metadata={
+            "product_key": "node_value_2022",
+            "coverage_id": "lu:lu_knptw_ha_2022_v3",
+            "render_min_value": 0.1,
+            "render_max_value": 4.0,
+        },
+        status="ready",
+        storage_path=str(path),
+    )
+    db = FakeSession({(Dataset, dataset_id): dataset})
+
+    assert ThematicRasterAnalysisService.render_png(db, project_id, dataset_id).startswith(b"\x89PNG\r\n\x1a\n")
+
+
+def test_api_uses_canonical_envelopes(monkeypatch) -> None:
+    project_id, dataset_id = uuid4(), uuid4()
+    db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
+    monkeypatch.setattr(
+        ThematicRasterAcquisitionService,
+        "acquire",
+        lambda *_args, **_kwargs: {"output_dataset_id": str(dataset_id), "provider": ThematicRasterAcquisitionService.PROVIDER},
+    )
+    monkeypatch.setattr(
+        ThematicRasterAnalysisService,
+        "analyze",
+        lambda *_args, **_kwargs: {"dataset_id": str(dataset_id), "theme": "population", "summary": {"metric_value": 10.0}},
+    )
+    app.dependency_overrides[get_db] = lambda: db
+    try:
+        client = TestClient(app)
+        products = client.get(f"/api/v1/projects/{project_id}/datasets/thematic-raster/products")
+        acquisition = client.post(
+            f"/api/v1/projects/{project_id}/datasets/thematic-raster/acquire",
+            json=payload().model_dump(mode="json"),
+        )
+        selection = client.post(
+            f"/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/thematic/select",
+            json={"bbox": payload().bbox.model_dump()},
+        )
+    finally:
+        app.dependency_overrides.clear()
+
+    assert products.status_code == 200 and set(products.json()) == {"data"}
+    assert products.json()["data"]["total"] == 5
+    assert acquisition.status_code == 200 and set(acquisition.json()) == {"data"}
+    assert acquisition.json()["data"]["job_type"] == "raster.thematic.acquire"
+    assert selection.status_code == 200 and selection.json()["data"]["theme"] == "population"
+    assert any(isinstance(item, Job) for item in db.added)
@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+from shapely.geometry import box, mapping, shape
+from shapely.ops import transform as transform_geometry
+
+from app.models import Dataset
+from app.services.vector_feature_service import VectorFeatureService
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def load_operator():
+    path = ROOT / "scripts" / "provision_mol_soil_map.py"
+    spec = importlib.util.spec_from_file_location("dov_soil_map_operator", path)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+class FakeResponse:
+    def __init__(self, payload: dict, url: str):
+        self._payload = payload
+        self.url = url
+        self.content = b'{"type":"FeatureCollection"}'
+
+    def raise_for_status(self) -> None:
+        return None
+
+    def json(self) -> dict:
+        return self._payload
+
+
+class FakeSession:
+    def __init__(self, pages: list[dict]):
+        self.pages = pages
+        self.calls: list[dict] = []
+
+    def get(self, _url: str, *, params: dict, timeout: int):
+        self.calls.append({"params": dict(params), "timeout": timeout})
+        return FakeResponse(self.pages[len(self.calls) - 1], f"https://example.test/page/{len(self.calls)}")
+
+
+def soil_feature(module, feature_id: str = "bodemtypes.1") -> dict:
+    return {
+        "type": "Feature",
+        "id": feature_id,
+        "geometry": mapping(box(5.0, 51.0, 5.02, 51.02)),
+        "properties": {
+            "gid": 1,
+            "id_kaartvlak": 10,
+            "Bodemtype": "Zeg",
+            "Unibodemtype": "Zeg",
+            "Bodemserie": "Zeg",
+            "Beknopte_omschrijving_bodemserie": "Natte zandbodem",
+            "Gegeneraliseerde_legende": "Nat zand",
+            "Textuurklasse_code": "Z",
+            "Textuurklasse": "zand",
+            "Drainageklasse_code": "e",
+            "Drainageklasse": "nat",
+            "Profielontwikkelingsgroep_code": "g",
+            "Profielontwikkelingsgroep": "humus B horizont",
+            "Eenduidige_legende_titel": "bodemserie Zeg",
+        },
+    }
+
+
+def test_wfs_pagination_is_bounded_complete_and_deterministic() -> None:
+    module = load_operator()
+    feature = soil_feature(module)
+    pages = [
+        {
+            "type": "FeatureCollection",
+            "numberMatched": 3,
+            "numberReturned": 2,
+            "features": [feature, {**feature, "id": "bodemtypes.2"}],
+        },
+        {
+            "type": "FeatureCollection",
+            "numberMatched": 3,
+            "numberReturned": 1,
+            "features": [{**feature, "id": "bodemtypes.3"}],
+        },
+    ]
+    session = FakeSession(pages)
+
+    result = list(
+        module.iter_wfs_pages(
+            session,
+            (196000.0, 205000.0, 211000.0, 224000.0),
+            page_limit=2,
+            timeout=30,
+        )
+    )
+
+    assert len(result) == 2
+    assert [call["params"]["startIndex"] for call in session.calls] == ["0", "2"]
+    assert all(call["params"]["typeNames"] == "bodemkaart:bodemtypes" for call in session.calls)
+    assert all(call["params"]["bbox"].endswith("EPSG:31370") for call in session.calls)
+    assert all(call["params"]["sortBy"] == "gid" for call in session.calls)
+
+
+def test_soil_feature_is_exactly_clipped_and_keeps_governed_properties() -> None:
+    module = load_operator()
+    boundary_wgs84 = box(5.005, 51.005, 5.015, 51.015)
+    boundary_lambert72 = transform_geometry(module.TO_LAMBERT72.transform, boundary_wgs84)
+
+    normalized, was_clipped = module.normalize_feature(soil_feature(module), boundary_lambert72)
+
+    assert normalized is not None and was_clipped is True
+    persisted_geometry = shape(normalized["geometry"])
+    assert persisted_geometry.within(boundary_wgs84.buffer(1e-7))
+    properties = normalized["properties"]
+    assert properties["source_name"] == "dov_soil_map"
+    assert properties["soil_texture_class"] == "zand"
+    assert properties["soil_drainage_class"] == "nat"
+    assert properties["survey_period"] == "1949-1971"
+    assert properties["clipped_area_ha"] > 0
+    assert "may differ today" in properties["historical_drainage_limitation"]
+
+
+def test_soil_map_uses_existing_semantic_selection_architecture() -> None:
+    dataset = Dataset(
+        id=uuid4(),
+        project_id=uuid4(),
+        name="dov_soil_map_mol.geojson",
+        dataset_type="vector",
+        source="operator_official_import",
+        source_name="dov_soil_map",
+        reference_layer_name="soil",
+        source_metadata={
+            "theme": "soil",
+            "selection_aggregation": {
+                "method": "intersection_area",
+                "label": "Bodemkaartoppervlakte",
+                "unit": "ha",
+            },
+        },
+        status="ready",
+    )
+
+    assert VectorFeatureService._dataset_theme(dataset) == "soil"
+    assert VectorFeatureService.supports_selection_summary(dataset) is True
+    assert VectorFeatureService.can_use_full_area_fast_path(dataset, None) is False
+
+
+def test_soil_operator_contract_has_no_direct_persistence_and_is_packaged() -> None:
+    operator = (ROOT / "scripts" / "provision_mol_soil_map.py").read_text(encoding="utf-8")
+    dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
+    map_workspace = (ROOT / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8")
+
+    assert "/datasets/upload" in operator
+    assert "vector_features" in operator
+    assert "does not write directly" in " ".join(operator.split())
+    assert "SessionLocal" not in operator and "INSERT INTO" not in operator
+    assert "COPY scripts/provision_mol_soil_map.py" in dockerfile
+    assert "py_compile scripts/provision_mol_soil_map.py" in readiness
+    assert "id: 'soil'" in map_workspace
+    assert "dataset.source_name === 'dov_soil_map'" in map_workspace
+
+
+def test_incomplete_wfs_pagination_fails_closed() -> None:
+    module = load_operator()
+    session = FakeSession(
+        [
+            {
+                "type": "FeatureCollection",
+                "numberMatched": 2,
+                "numberReturned": 0,
+                "features": [],
+            }
+        ]
+    )
+
+    with pytest.raises(RuntimeError, match="returned 0 of 2"):
+        list(module.iter_wfs_pages(session, (0.0, 0.0, 1.0, 1.0), page_limit=100, timeout=30))