feat: operationalize Flemish land and nature themes
This commit is contained in:
@@ -1803,3 +1803,24 @@ readiness state such as TLS or endpoint failure. Runtime controls are
|
||||
`MDK_BATHYMETRY_PROBE_ENABLED`, `MDK_BATHYMETRY_WCS_URL`,
|
||||
`MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS` and
|
||||
`MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB`. TLS verification cannot be disabled.
|
||||
|
||||
## Governed forest, agriculture, nature and soil acquisition
|
||||
|
||||
The thematic raster registry includes forest and agricultural land-use masks
|
||||
derived from Landgebruik Vlaanderen 2025 classes 12 and 13/14. They use the
|
||||
existing thematic acquisition and selection routes.
|
||||
|
||||
Two polygon products are exposed through
|
||||
`/datasets/official-vector/products` and
|
||||
`/datasets/official-vector/acquire`: INBO BWK/Natura 2000 2025 and the DOV
|
||||
digital soil map. Both require an EPSG:4326 rectangle, optionally intersect it
|
||||
with a persisted Area, clip in EPSG:31370 and persist through
|
||||
`DatasetService.import_vector_bytes`.
|
||||
|
||||
Runtime controls are `OFFICIAL_VECTOR_ENABLED`, `BWK_WFS_URL`,
|
||||
`DOV_SOIL_WFS_URL`, `OFFICIAL_VECTOR_MIN_SIDE_M`,
|
||||
`OFFICIAL_VECTOR_MAX_SIDE_M`, `OFFICIAL_VECTOR_PAGE_SIZE`,
|
||||
`OFFICIAL_VECTOR_MAX_PAGES`, `OFFICIAL_VECTOR_MAX_FEATURES`,
|
||||
`OFFICIAL_VECTOR_TIMEOUT_SECONDS`, `OFFICIAL_VECTOR_MAX_RESPONSE_MB`,
|
||||
`OFFICIAL_VECTOR_MAX_TOTAL_RESPONSE_MB` and
|
||||
`OFFICIAL_VECTOR_CACHE_TTL_HOURS`.
|
||||
|
||||
@@ -32,6 +32,7 @@ from app.schemas import (
|
||||
ThematicRasterAcquireRequest,
|
||||
ThematicRasterSelectionRequest,
|
||||
GrbAcquireRequest,
|
||||
OfficialVectorAcquireRequest,
|
||||
VectorBBoxResponse,
|
||||
VectorBufferRequest,
|
||||
VectorClipRequest,
|
||||
@@ -51,6 +52,7 @@ from app.services.source_freshness_service import SourceFreshnessService
|
||||
from app.services.source_catalog_probe_service import SourceCatalogProbeService
|
||||
from app.services.grb_refresh_plan_service import GrbRefreshPlanService
|
||||
from app.services.grb_acquisition_service import GrbAcquisitionService
|
||||
from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
|
||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||
from app.services.terrain_analysis_service import TerrainAnalysisService
|
||||
@@ -219,6 +221,30 @@ def list_grb_products(project_id: UUID, db: Session = Depends(get_db)):
|
||||
return envelope({"items": items, "total": len(items)})
|
||||
|
||||
|
||||
@router.post("/datasets/official-vector/acquire", response_model=dict)
|
||||
def acquire_bounded_official_vector(
|
||||
project_id: UUID,
|
||||
payload: OfficialVectorAcquireRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
job = JobService.run_sync_job(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
job_type="vector.official.acquire",
|
||||
parameters=payload.model_dump(mode="json"),
|
||||
operation=lambda: OfficialVectorAcquisitionService.acquire(db, project_id, payload),
|
||||
)
|
||||
return envelope(job)
|
||||
|
||||
|
||||
@router.get("/datasets/official-vector/products", response_model=dict)
|
||||
def list_official_vector_products(project_id: UUID, db: Session = Depends(get_db)):
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
items = OfficialVectorAcquisitionService.list_products()
|
||||
return envelope({"items": items, "total": len(items)})
|
||||
|
||||
|
||||
@router.post("/datasets/flood-hazard/acquire", response_model=dict)
|
||||
def acquire_bounded_flood_hazard(
|
||||
project_id: UUID,
|
||||
|
||||
@@ -87,6 +87,66 @@ class Settings(BaseSettings):
|
||||
validation_alias="GRB_MAX_TOTAL_RESPONSE_MB",
|
||||
)
|
||||
grb_cache_ttl_hours: int = Field(default=24, ge=0, le=8760, validation_alias="GRB_CACHE_TTL_HOURS")
|
||||
official_vector_enabled: bool = Field(default=True, validation_alias="OFFICIAL_VECTOR_ENABLED")
|
||||
bwk_wfs_url: str = Field(
|
||||
default="https://geo.api.vlaanderen.be/BWK/wfs",
|
||||
validation_alias="BWK_WFS_URL",
|
||||
)
|
||||
dov_soil_wfs_url: str = Field(
|
||||
default="https://www.dov.vlaanderen.be/geoserver/wfs",
|
||||
validation_alias="DOV_SOIL_WFS_URL",
|
||||
)
|
||||
official_vector_min_side_m: float = Field(
|
||||
default=10.0,
|
||||
gt=0,
|
||||
validation_alias="OFFICIAL_VECTOR_MIN_SIDE_M",
|
||||
)
|
||||
official_vector_max_side_m: float = Field(
|
||||
default=20_000.0,
|
||||
gt=0,
|
||||
validation_alias="OFFICIAL_VECTOR_MAX_SIDE_M",
|
||||
)
|
||||
official_vector_page_size: int = Field(
|
||||
default=1000,
|
||||
ge=1,
|
||||
le=2000,
|
||||
validation_alias="OFFICIAL_VECTOR_PAGE_SIZE",
|
||||
)
|
||||
official_vector_max_pages: int = Field(
|
||||
default=200,
|
||||
ge=1,
|
||||
le=1000,
|
||||
validation_alias="OFFICIAL_VECTOR_MAX_PAGES",
|
||||
)
|
||||
official_vector_max_features: int = Field(
|
||||
default=100_000,
|
||||
ge=1,
|
||||
validation_alias="OFFICIAL_VECTOR_MAX_FEATURES",
|
||||
)
|
||||
official_vector_timeout_seconds: int = Field(
|
||||
default=180,
|
||||
ge=1,
|
||||
le=600,
|
||||
validation_alias="OFFICIAL_VECTOR_TIMEOUT_SECONDS",
|
||||
)
|
||||
official_vector_max_response_mb: int = Field(
|
||||
default=20,
|
||||
ge=1,
|
||||
le=100,
|
||||
validation_alias="OFFICIAL_VECTOR_MAX_RESPONSE_MB",
|
||||
)
|
||||
official_vector_max_total_response_mb: int = Field(
|
||||
default=256,
|
||||
ge=1,
|
||||
le=2048,
|
||||
validation_alias="OFFICIAL_VECTOR_MAX_TOTAL_RESPONSE_MB",
|
||||
)
|
||||
official_vector_cache_ttl_hours: int = Field(
|
||||
default=24,
|
||||
ge=0,
|
||||
le=8760,
|
||||
validation_alias="OFFICIAL_VECTOR_CACHE_TTL_HOURS",
|
||||
)
|
||||
dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED")
|
||||
dhmv_wcs_url: str = Field(
|
||||
default="https://geo.api.vlaanderen.be/DHMV/wcs",
|
||||
|
||||
@@ -18,6 +18,11 @@ from .source_catalog import (
|
||||
)
|
||||
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
|
||||
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
|
||||
from .official_vector import (
|
||||
OfficialVectorAcquireRequest,
|
||||
OfficialVectorAcquisitionResult,
|
||||
OfficialVectorProductRead,
|
||||
)
|
||||
from .detection import (
|
||||
DetectionListResponse,
|
||||
DetectionModelCapability,
|
||||
@@ -165,6 +170,9 @@ __all__ = [
|
||||
"GrbAcquireRequest",
|
||||
"GrbAcquisitionResult",
|
||||
"GrbProductRead",
|
||||
"OfficialVectorAcquireRequest",
|
||||
"OfficialVectorAcquisitionResult",
|
||||
"OfficialVectorProductRead",
|
||||
"DetectionListResponse",
|
||||
"DetectionModelCapability",
|
||||
"DetectionModelsResponse",
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class OfficialVectorAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
product_key: str
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class OfficialVectorProductRead(BaseModel):
|
||||
key: str
|
||||
display_name: str
|
||||
theme: str
|
||||
provider: str
|
||||
source_name: str
|
||||
reference_layer_name: str
|
||||
service_type: str
|
||||
collection: str
|
||||
geometry_types: list[str]
|
||||
source_crs: str
|
||||
source_version: str
|
||||
observation_label: str
|
||||
authority_level: str
|
||||
catalog_url: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class OfficialVectorAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
product_key: str
|
||||
display_name: str
|
||||
theme: str
|
||||
provider: str
|
||||
source_name: str
|
||||
reference_layer_name: str
|
||||
service_type: str
|
||||
collection: str
|
||||
feature_count: int
|
||||
candidate_feature_count: int
|
||||
page_count: int
|
||||
bbox_epsg4326: list[float]
|
||||
source_version: str
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
@@ -30,6 +30,7 @@ class ThematicRasterProductRead(BaseModel):
|
||||
license_note: str
|
||||
legend_min_label: str
|
||||
legend_max_label: str
|
||||
included_source_values: list[int]
|
||||
limitation_message: str
|
||||
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -45,6 +45,7 @@ class ThematicRasterProduct:
|
||||
legend_min_label: str
|
||||
legend_max_label: str
|
||||
limitation_message: str
|
||||
included_source_values: tuple[int, ...] = ()
|
||||
|
||||
|
||||
class ThematicRasterAcquisitionService:
|
||||
@@ -100,6 +101,44 @@ class ThematicRasterAcquisitionService:
|
||||
"synoniem met natuur, bos, publieke toegankelijkheid of planologische bestemming."
|
||||
),
|
||||
),
|
||||
ThematicRasterProduct(
|
||||
key="forest_land_use_2025",
|
||||
display_name="Bos volgens Landgebruik Vlaanderen 2025",
|
||||
theme="forest",
|
||||
metric_kind="binary_area",
|
||||
coverage_id="lu:lu_landgebruik_vlaa_2025_v3",
|
||||
native_resolution_m=10.0,
|
||||
source_value_unit="class_0_1",
|
||||
observation_year=2025,
|
||||
source_version="Toestand 2025 versie 3",
|
||||
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2025",
|
||||
legend_min_label="Geen bosklasse",
|
||||
legend_max_label="Bos",
|
||||
limitation_message=(
|
||||
"10 m-afleiding van bronklasse 12 (bos) uit Landgebruik Vlaanderen 2025. De oppervlakte is "
|
||||
"resolutiegebonden en vormt geen juridische bosgrens, boomtelling, kroonbedekking of houtvolume."
|
||||
),
|
||||
included_source_values=(12,),
|
||||
),
|
||||
ThematicRasterProduct(
|
||||
key="agricultural_land_use_2025",
|
||||
display_name="Akker en landbouwgrasland 2025",
|
||||
theme="agriculture",
|
||||
metric_kind="binary_area",
|
||||
coverage_id="lu:lu_landgebruik_vlaa_2025_v3",
|
||||
native_resolution_m=10.0,
|
||||
source_value_unit="class_0_1",
|
||||
observation_year=2025,
|
||||
source_version="Toestand 2025 versie 3",
|
||||
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/landgebruik-vlaanderen-toestand-2025",
|
||||
legend_min_label="Ander landgebruik",
|
||||
legend_max_label="Akker of landbouwgrasland",
|
||||
limitation_message=(
|
||||
"10 m-afleiding van bronklassen 13 (akker) en 14 (grasland in landbouwgebruik). Dit is werkelijk "
|
||||
"landgebruik en geen ALZ-perceelaangifte, eigendomsgrens, teeltregister of juridische bestemming."
|
||||
),
|
||||
included_source_values=(13, 14),
|
||||
),
|
||||
ThematicRasterProduct(
|
||||
key="population_density_2019",
|
||||
display_name="Inwonersdichtheid per hectare 2019",
|
||||
@@ -176,6 +215,7 @@ class ThematicRasterAcquisitionService:
|
||||
license_note=ThematicRasterAcquisitionService.LICENSE_NOTE,
|
||||
legend_min_label=product.legend_min_label,
|
||||
legend_max_label=product.legend_max_label,
|
||||
included_source_values=list(product.included_source_values),
|
||||
limitation_message=product.limitation_message,
|
||||
).model_dump()
|
||||
for product in ThematicRasterAcquisitionService._products().values()
|
||||
@@ -461,7 +501,29 @@ class ThematicRasterAcquisitionService:
|
||||
invalid = np.ma.getmaskarray(band) | ~np.isfinite(raw)
|
||||
if source.nodata is not None:
|
||||
invalid |= np.isclose(raw, float(source.nodata))
|
||||
normalized = np.ma.array(raw, mask=invalid)
|
||||
source_values = np.ma.array(raw, mask=invalid).compressed().astype("float64")
|
||||
if product.included_source_values:
|
||||
rounded = np.rint(source_values)
|
||||
if not np.allclose(source_values, rounded, atol=0.0001):
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_INVALID_VALUES",
|
||||
message="Categorical land-use coverage contains non-integer source classes",
|
||||
status_code=502,
|
||||
)
|
||||
if source_values.size and (
|
||||
float(source_values.min()) < 0
|
||||
or float(source_values.max()) > 255
|
||||
):
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_INVALID_VALUES",
|
||||
message="Categorical land-use coverage contains source classes outside the governed range",
|
||||
status_code=502,
|
||||
)
|
||||
source_classes = np.where(invalid, 0, np.rint(raw)).astype("int16")
|
||||
binary = np.isin(source_classes, product.included_source_values).astype("float32")
|
||||
normalized = np.ma.array(binary, mask=invalid)
|
||||
else:
|
||||
normalized = np.ma.array(raw, mask=invalid)
|
||||
values = normalized.compressed().astype("float64")
|
||||
ThematicRasterAcquisitionService._validate_values(values, product)
|
||||
profile = source.profile.copy()
|
||||
@@ -482,6 +544,9 @@ class ThematicRasterAcquisitionService:
|
||||
"maximum_value": float(values.max()),
|
||||
"p02_value": float(np.percentile(values, 2)),
|
||||
"p98_value": float(np.percentile(values, 98)),
|
||||
"included_source_values": list(product.included_source_values),
|
||||
"source_minimum_value": float(source_values.min()),
|
||||
"source_maximum_value": float(source_values.max()),
|
||||
}
|
||||
except AppError:
|
||||
raise
|
||||
@@ -563,6 +628,7 @@ class ThematicRasterAcquisitionService:
|
||||
"analysis_resolution_m": product.native_resolution_m,
|
||||
"source_crs": ThematicRasterAcquisitionService.SOURCE_CRS,
|
||||
"source_value_unit": product.source_value_unit,
|
||||
"included_source_values": list(product.included_source_values),
|
||||
"observation_year": product.observation_year,
|
||||
"observation_date_precision": "year",
|
||||
"valid_pixel_count": validation["valid_pixel_count"],
|
||||
|
||||
@@ -68,6 +68,10 @@ class ThematicRasterAnalysisService:
|
||||
@staticmethod
|
||||
def _unsupported_metrics(product: ThematicRasterProduct) -> list[str]:
|
||||
if product.metric_kind == "binary_area":
|
||||
if product.theme == "forest":
|
||||
return ["tree_count", "canopy_cover", "timber_volume", "legal_forest_boundary"]
|
||||
if product.theme == "agriculture":
|
||||
return ["declared_parcel_area", "crop_declaration", "ownership", "cadastral_area"]
|
||||
return ["object_count", "parcel_area", "current_land_use"]
|
||||
if product.metric_kind == "population_density":
|
||||
return ["current_population", "household_count", "address_level_population"]
|
||||
@@ -152,7 +156,12 @@ class ThematicRasterAnalysisService:
|
||||
positive_count = int(np.count_nonzero(values >= 0.5))
|
||||
positive_area_ha = positive_count * cell_area_m2 / 10_000.0
|
||||
positive_share = positive_count / max(1, valid_cell_count) * 100.0
|
||||
label = "Ruimtebeslag" if product.theme == "space_occupation" else "Open ruimte"
|
||||
label = {
|
||||
"space_occupation": "Ruimtebeslag",
|
||||
"open_space": "Open ruimte",
|
||||
"forest": "Bos",
|
||||
"agriculture": "Akker en landbouwgrasland",
|
||||
}[product.theme]
|
||||
metrics = [
|
||||
metric(f"{product.theme}_area_ha", f"{label} in selectie", positive_area_ha, "ha", "positive_source_cells_times_cell_area"),
|
||||
metric(f"{product.theme}_share_pct", f"Aandeel {label.lower()}", positive_share, "%", "positive_source_cells_divided_by_valid_selected_cells"),
|
||||
@@ -216,6 +225,8 @@ class ThematicRasterAnalysisService:
|
||||
palettes = {
|
||||
"space_occupation": np.asarray([[251, 231, 211], [190, 62, 51]], dtype="float64"),
|
||||
"open_space": np.asarray([[221, 238, 219], [38, 122, 70]], dtype="float64"),
|
||||
"forest": np.asarray([[223, 237, 226], [43, 117, 72]], dtype="float64"),
|
||||
"agriculture": np.asarray([[245, 237, 204], [166, 122, 35]], dtype="float64"),
|
||||
"population": np.asarray([[238, 231, 246], [103, 58, 151]], dtype="float64"),
|
||||
"accessibility": np.asarray([[233, 241, 244], [15, 118, 110]], dtype="float64"),
|
||||
"services": np.asarray([[255, 244, 191], [182, 109, 22]], dtype="float64"),
|
||||
|
||||
@@ -122,21 +122,33 @@ def raster_bytes(values: np.ndarray, resolution: float, *, nodata: float = -9999
|
||||
return memory.read()
|
||||
|
||||
|
||||
def test_registry_contains_five_governed_non_water_policy_products() -> None:
|
||||
def test_registry_contains_governed_policy_products_including_forest_and_agriculture() -> None:
|
||||
products = ThematicRasterAcquisitionService.list_products()
|
||||
|
||||
assert [item["key"] for item in products] == [
|
||||
"space_occupation_2025",
|
||||
"open_space_2022",
|
||||
"forest_land_use_2025",
|
||||
"agricultural_land_use_2025",
|
||||
"population_density_2019",
|
||||
"node_value_2022",
|
||||
"service_level_2022",
|
||||
]
|
||||
assert {item["theme"] for item in products} == {"space_occupation", "open_space", "population", "accessibility", "services"}
|
||||
assert {item["theme"] for item in products} == {
|
||||
"space_occupation",
|
||||
"open_space",
|
||||
"forest",
|
||||
"agriculture",
|
||||
"population",
|
||||
"accessibility",
|
||||
"services",
|
||||
}
|
||||
assert {item["native_resolution_m"] for item in products} == {10.0, 100.0}
|
||||
assert all(item["coverage_id"].startswith(("lu:", "ni:")) for item in products)
|
||||
assert all(item["source_crs"] == "EPSG:31370" for item in products)
|
||||
assert all(item["attribution"] and item["license_note"] and item["limitation_message"] for item in products)
|
||||
assert next(item for item in products if item["theme"] == "forest")["included_source_values"] == [12]
|
||||
assert next(item for item in products if item["theme"] == "agriculture")["included_source_values"] == [13, 14]
|
||||
|
||||
|
||||
def test_request_is_bounded_allowlisted_and_uses_native_wcs_resolution() -> None:
|
||||
@@ -520,7 +532,7 @@ def test_api_uses_canonical_envelopes(monkeypatch) -> None:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert products.status_code == 200 and set(products.json()) == {"data"}
|
||||
assert products.json()["data"]["total"] == 5
|
||||
assert products.json()["data"]["total"] == 7
|
||||
assert acquisition.status_code == 200 and set(acquisition.json()) == {"data"}
|
||||
assert acquisition.json()["data"]["job_type"] == "raster.thematic.acquire"
|
||||
assert selection.status_code == 200 and selection.json()["data"]["theme"] == "population"
|
||||
|
||||
@@ -0,0 +1,407 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from geoalchemy2.shape import from_shape
|
||||
from rasterio.io import MemoryFile
|
||||
from rasterio.transform import from_origin
|
||||
from shapely.geometry import MultiPolygon, Polygon
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, Job, Project
|
||||
from app.schemas.official_vector import OfficialVectorAcquireRequest
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
|
||||
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, result=None):
|
||||
self.result = result
|
||||
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
|
||||
def order_by(self, *_args):
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return self.result if isinstance(self.result, list) else []
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, rows=None, query_result=None):
|
||||
self.rows = rows or {}
|
||||
self.query_result = query_result
|
||||
self.added = []
|
||||
|
||||
def get(self, model, row_id):
|
||||
row = self.rows.get((model, row_id))
|
||||
if row is not None:
|
||||
return row
|
||||
return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
|
||||
|
||||
def add(self, row):
|
||||
self.added.append(row)
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
def rollback(self):
|
||||
return None
|
||||
|
||||
def refresh(self, row):
|
||||
return row
|
||||
|
||||
def query(self, _model):
|
||||
return FakeQuery(self.query_result)
|
||||
|
||||
|
||||
class JsonResponse:
|
||||
def __init__(self, payload):
|
||||
self.content = json.dumps(payload).encode("utf-8")
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self, size=-1):
|
||||
return self.content if size < 0 else self.content[:size]
|
||||
|
||||
|
||||
def request(product_key: str, *, area_id=None) -> OfficialVectorAcquireRequest:
|
||||
return OfficialVectorAcquireRequest(
|
||||
bbox={
|
||||
"min_x": 5.15,
|
||||
"min_y": 51.18,
|
||||
"max_x": 5.17,
|
||||
"max_y": 51.20,
|
||||
"crs": "EPSG:4326",
|
||||
},
|
||||
area_id=area_id,
|
||||
product_key=product_key,
|
||||
force_refresh=True,
|
||||
)
|
||||
|
||||
|
||||
def polygon_feature(feature_id: str, *, properties=None) -> dict:
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": feature_id,
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[
|
||||
[5.155, 51.185],
|
||||
[5.175, 51.185],
|
||||
[5.175, 51.195],
|
||||
[5.155, 51.195],
|
||||
[5.155, 51.185],
|
||||
]],
|
||||
},
|
||||
"properties": properties or {},
|
||||
}
|
||||
|
||||
|
||||
def test_product_registries_expose_honest_forest_agriculture_nature_and_soil() -> None:
|
||||
raster = {item["key"]: item for item in ThematicRasterAcquisitionService.list_products()}
|
||||
vector = {item["key"]: item for item in OfficialVectorAcquisitionService.list_products()}
|
||||
|
||||
assert raster["forest_land_use_2025"]["included_source_values"] == [12]
|
||||
assert raster["agricultural_land_use_2025"]["included_source_values"] == [13, 14]
|
||||
assert "geen juridische bosgrens" in raster["forest_land_use_2025"]["limitation_message"].lower()
|
||||
assert "geen alz-perceelaangifte" in raster["agricultural_land_use_2025"]["limitation_message"].lower()
|
||||
assert set(vector) == {"bwk_natura2000_2025", "dov_soil_types"}
|
||||
assert vector["bwk_natura2000_2025"]["authority_level"] == "authoritative"
|
||||
assert vector["dov_soil_types"]["authority_level"] == "authoritative_historical_baseline"
|
||||
assert "1949-1971" in vector["dov_soil_types"]["observation_label"]
|
||||
|
||||
|
||||
def test_land_use_classes_are_converted_to_binary_masks_without_nodata_cast_warning() -> None:
|
||||
values = np.asarray([[12.0, 13.0], [14.0, -9999.0]], dtype="float32")
|
||||
with MemoryFile() as source_memory:
|
||||
with source_memory.open(
|
||||
driver="GTiff",
|
||||
width=2,
|
||||
height=2,
|
||||
count=1,
|
||||
dtype="float32",
|
||||
crs="EPSG:31370",
|
||||
transform=from_origin(200_000, 210_020, 10, 10),
|
||||
nodata=-9999.0,
|
||||
) as source:
|
||||
source.write(values, 1)
|
||||
from pyproj import Transformer
|
||||
to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
||||
scope = Polygon([
|
||||
to_wgs84.transform(200_000, 210_000),
|
||||
to_wgs84.transform(200_020, 210_000),
|
||||
to_wgs84.transform(200_020, 210_020),
|
||||
to_wgs84.transform(200_000, 210_020),
|
||||
to_wgs84.transform(200_000, 210_000),
|
||||
])
|
||||
content, validation = ThematicRasterAcquisitionService._normalize_raster(
|
||||
source_memory.read(),
|
||||
scope,
|
||||
{
|
||||
"product": ThematicRasterAcquisitionService._product("forest_land_use_2025"),
|
||||
"width": 2,
|
||||
"height": 2,
|
||||
"bbox_epsg31370": [200_000, 210_000, 200_020, 210_020],
|
||||
},
|
||||
)
|
||||
with MemoryFile(content) as normalized_memory:
|
||||
with normalized_memory.open() as normalized:
|
||||
output = normalized.read(1, masked=True)
|
||||
|
||||
assert output.compressed().tolist() == [1.0, 0.0, 0.0]
|
||||
assert validation["included_source_values"] == [12]
|
||||
assert validation["source_minimum_value"] == 12.0
|
||||
assert validation["source_maximum_value"] == 14.0
|
||||
|
||||
|
||||
def test_bwk_wfs_pagination_clips_geometry_and_preserves_semantics() -> None:
|
||||
product = OfficialVectorAcquisitionService._product("bwk_natura2000_2025")
|
||||
scope_wgs84 = Polygon([
|
||||
(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
|
||||
])
|
||||
from shapely.ops import transform
|
||||
from app.services.official_vector_acquisition_service import _TO_LAMBERT72
|
||||
|
||||
scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84)
|
||||
calls = []
|
||||
|
||||
def opener(raw_request, timeout):
|
||||
assert timeout == 180
|
||||
calls.append(raw_request.full_url)
|
||||
query = parse_qs(urlparse(raw_request.full_url).query)
|
||||
assert query["typeNames"] == ["BWK:Bwkhab"]
|
||||
assert query["sortBy"] == ["UIDN"]
|
||||
feature = polygon_feature(
|
||||
"Bwkhab.1",
|
||||
properties={"UIDN": 42, "EVAL": "z", "HAB1": "2310", "PHAB1": 60},
|
||||
)
|
||||
if query.get("startIndex") == ["1"]:
|
||||
return JsonResponse({
|
||||
"type": "FeatureCollection",
|
||||
"numberReturned": 0,
|
||||
"features": [],
|
||||
})
|
||||
return JsonResponse({
|
||||
"type": "FeatureCollection",
|
||||
"numberReturned": 1,
|
||||
"features": [feature],
|
||||
})
|
||||
|
||||
features, transfer = OfficialVectorAcquisitionService._fetch_features(
|
||||
product,
|
||||
scope_wgs84,
|
||||
scope_metric,
|
||||
"bounded_selection",
|
||||
Settings(_env_file=None, OFFICIAL_VECTOR_PAGE_SIZE=1),
|
||||
opener,
|
||||
)
|
||||
|
||||
assert len(calls) == 2
|
||||
assert transfer["reference_truncated"] is False
|
||||
assert features[0]["id"] == "BWK:Bwkhab:42"
|
||||
assert features[0]["properties"]["bwk_evaluation_code"] == "z"
|
||||
assert features[0]["properties"]["natura2000_share_percent"] == 60
|
||||
assert features[0]["properties"]["geometry_clipped_to_selection"] is True
|
||||
|
||||
|
||||
def test_bwk_rejects_a_non_https_configured_endpoint_before_network_access() -> None:
|
||||
product = OfficialVectorAcquisitionService._product("bwk_natura2000_2025")
|
||||
scope_wgs84 = Polygon([
|
||||
(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
|
||||
])
|
||||
from shapely.ops import transform
|
||||
from app.services.official_vector_acquisition_service import _TO_LAMBERT72
|
||||
|
||||
scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84)
|
||||
|
||||
def opener(_request, timeout):
|
||||
del _request, timeout
|
||||
raise AssertionError("network access must not occur")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
OfficialVectorAcquisitionService._fetch_features(
|
||||
product,
|
||||
scope_wgs84,
|
||||
scope_metric,
|
||||
"bounded_selection",
|
||||
Settings(_env_file=None, BWK_WFS_URL="http://example.invalid/wfs"),
|
||||
opener,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "OFFICIAL_VECTOR_PROVIDER_INVALID_PAGINATION"
|
||||
|
||||
|
||||
def test_dov_wfs_uses_stable_complete_pagination_and_historical_fields() -> None:
|
||||
product = OfficialVectorAcquisitionService._product("dov_soil_types")
|
||||
scope_wgs84 = Polygon([
|
||||
(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
|
||||
])
|
||||
from shapely.ops import transform
|
||||
from app.services.official_vector_acquisition_service import _TO_LAMBERT72
|
||||
|
||||
scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84)
|
||||
|
||||
def opener(raw_request, timeout):
|
||||
assert timeout == 180
|
||||
query = parse_qs(urlparse(raw_request.full_url).query)
|
||||
assert query["typeNames"] == ["bodemkaart:bodemtypes"]
|
||||
assert query["sortBy"] == ["gid"]
|
||||
return JsonResponse({
|
||||
"type": "FeatureCollection",
|
||||
"numberMatched": 1,
|
||||
"numberReturned": 1,
|
||||
"features": [polygon_feature(
|
||||
"bodemtypes.7",
|
||||
properties={
|
||||
"gid": 7,
|
||||
"Bodemtype": "Zcg",
|
||||
"Gegeneraliseerde_legende": "Droog zand",
|
||||
"Drainageklasse": "Matig droog",
|
||||
},
|
||||
)],
|
||||
})
|
||||
|
||||
features, transfer = OfficialVectorAcquisitionService._fetch_features(
|
||||
product,
|
||||
scope_wgs84,
|
||||
scope_metric,
|
||||
"bounded_selection",
|
||||
Settings(_env_file=None),
|
||||
opener,
|
||||
)
|
||||
|
||||
assert transfer["page_count"] == 1
|
||||
assert transfer["candidate_feature_count"] == 1
|
||||
assert features[0]["properties"]["soil_type_code"] == "Zcg"
|
||||
assert features[0]["properties"]["soil_generalized_legend"] == "Droog zand"
|
||||
assert features[0]["properties"]["survey_period"] == "1949-1971"
|
||||
|
||||
|
||||
def test_nature_acquisition_persists_only_through_dataset_service(monkeypatch) -> None:
|
||||
project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4()
|
||||
municipality = MultiPolygon([Polygon([
|
||||
(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
|
||||
])])
|
||||
db = FakeSession({
|
||||
(Project, project_id): Project(id=project_id, name="Vlaanderen"),
|
||||
(Area, area_id): Area(
|
||||
id=area_id,
|
||||
project_id=project_id,
|
||||
name="Gemeente Mol",
|
||||
geometry=from_shape(municipality, srid=4326),
|
||||
),
|
||||
})
|
||||
captured = {}
|
||||
|
||||
def opener(_request, timeout):
|
||||
del timeout
|
||||
return JsonResponse({
|
||||
"type": "FeatureCollection",
|
||||
"features": [polygon_feature(
|
||||
"Bwkhab.1",
|
||||
properties={"UIDN": 42, "EVAL": "w", "HAB1": "rbbmr", "PHAB1": 100},
|
||||
)],
|
||||
"links": [],
|
||||
})
|
||||
|
||||
def persist(_db, **kwargs):
|
||||
captured.update(kwargs)
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
name=kwargs["filename"],
|
||||
dataset_type="vector",
|
||||
source=kwargs["source"],
|
||||
dataset_role=kwargs["dataset_role"],
|
||||
source_name=kwargs["source_name"],
|
||||
reference_layer_name=kwargs["reference_layer_name"],
|
||||
observed_at=kwargs["observed_at"],
|
||||
source_version=kwargs["source_version"],
|
||||
source_metadata=kwargs["source_metadata"],
|
||||
provenance_metadata=kwargs["provenance_metadata"],
|
||||
metadata_json={"feature_count": 1},
|
||||
status="ready",
|
||||
)
|
||||
db.rows[(Dataset, dataset_id)] = dataset
|
||||
return SimpleNamespace(id=dataset_id)
|
||||
|
||||
monkeypatch.setattr(DatasetService, "import_vector_bytes", persist)
|
||||
result = OfficialVectorAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
request("bwk_natura2000_2025", area_id=area_id),
|
||||
settings=Settings(_env_file=None),
|
||||
opener=opener,
|
||||
)
|
||||
|
||||
assert result["output_dataset_id"] == str(dataset_id)
|
||||
assert captured["dataset_role"] == "reference"
|
||||
assert captured["source_name"] == "inbo_bwk_natura2000"
|
||||
assert captured["reference_layer_name"] == "nature_value"
|
||||
assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == "nature_mapped_area"
|
||||
assert captured["source_metadata"]["selection_metrics"][4]["is_estimate"] is True
|
||||
assert captured["provenance_metadata"]["reference_truncated"] is False
|
||||
assert json.loads(captured["content"])["features"][0]["properties"]["coverage_scope"] == "municipality"
|
||||
|
||||
|
||||
def test_official_vector_routes_and_frontend_use_canonical_backend_path(monkeypatch) -> None:
|
||||
project_id, dataset_id = uuid4(), uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Vlaanderen")})
|
||||
monkeypatch.setattr(
|
||||
OfficialVectorAcquisitionService,
|
||||
"acquire",
|
||||
lambda *_args, **_kwargs: {
|
||||
"output_dataset_id": str(dataset_id),
|
||||
"product_key": "bwk_natura2000_2025",
|
||||
"feature_count": 1,
|
||||
},
|
||||
)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
try:
|
||||
client = TestClient(app)
|
||||
products_response = client.get(
|
||||
f"/api/v1/projects/{project_id}/datasets/official-vector/products"
|
||||
)
|
||||
acquire_response = client.post(
|
||||
f"/api/v1/projects/{project_id}/datasets/official-vector/acquire",
|
||||
json=request("bwk_natura2000_2025").model_dump(mode="json"),
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert products_response.status_code == 200
|
||||
assert set(products_response.json()) == {"data"}
|
||||
assert products_response.json()["data"]["total"] == 2
|
||||
assert acquire_response.status_code == 200
|
||||
assert set(acquire_response.json()) == {"data"}
|
||||
assert acquire_response.json()["data"]["job_type"] == "vector.official.acquire"
|
||||
assert any(isinstance(item, Job) for item in db.added)
|
||||
|
||||
selection_hook = (ROOT / "frontend/src/hooks/useMapThemeSelectionInsights.ts").read_text(encoding="utf-8")
|
||||
catalog_hook = (ROOT / "frontend/src/hooks/useOfficialMapProducts.ts").read_text(encoding="utf-8")
|
||||
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||
assert "datasetsApi.acquireOfficialVector" in selection_hook
|
||||
assert "datasetsApi.listOfficialVectorProducts" in catalog_hook
|
||||
assert "officialMapProducts.officialVector" in workspace
|
||||
assert "geo.api.vlaanderen.be" not in workspace
|
||||
Reference in New Issue
Block a user