Federate official Belgium data sources
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-19 01:34:39 +02:00
parent 33bcd0f3bd
commit 50897c3473
37 changed files with 2179 additions and 186 deletions
+11 -7
View File
@@ -1854,21 +1854,25 @@ readiness state such as TLS or endpoint failure. Runtime controls are
`MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS` and
`MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB`. TLS verification cannot be disabled.
## Governed forest, agriculture, nature and soil acquisition
## Governed regional official-vector acquisition
The thematic raster registry includes forest and agricultural land-use masks
derived from Landgebruik Vlaanderen 2025 classes 12 and 13/14. They use the
existing thematic acquisition and selection routes.
Two polygon products are exposed through
Eight fixed products are exposed through
`/datasets/official-vector/products` and
`/datasets/official-vector/acquire`: INBO BWK/Natura 2000 2025 and the DOV
digital soil map. Both require an EPSG:4326 rectangle, optionally intersect it
with a persisted Area, clip in EPSG:31370 and persist through
`DatasetService.import_vector_bytes`.
`/datasets/official-vector/acquire`: INBO BWK/Natura 2000 2025 and DOV soil
for Flanders; PICC buildings, roads, hydrographic axes and surfaces for
Wallonia; and UrbIS buildings and cadastral parcels for Brussels. All require
an EPSG:4326 rectangle, clip in a provider-appropriate metric CRS and persist
through `DatasetService.import_vector_bytes`. SPW/PICC and UrbIS additionally
require a persisted exact regional coverage Area and never write directly to
`vector_features`.
Runtime controls are `OFFICIAL_VECTOR_ENABLED`, `BWK_WFS_URL`,
`DOV_SOIL_WFS_URL`, `OFFICIAL_VECTOR_MIN_SIDE_M`,
`DOV_SOIL_WFS_URL`, `SPW_PICC_ENABLED`, `SPW_PICC_MAPSERVER_URL`,
`URBIS_ENABLED`, `URBIS_WFS_URL`, `OFFICIAL_VECTOR_MIN_SIDE_M`,
`OFFICIAL_VECTOR_MAX_SIDE_M`, `OFFICIAL_VECTOR_PAGE_SIZE`,
`OFFICIAL_VECTOR_MAX_PAGES`, `OFFICIAL_VECTOR_MAX_FEATURES`,
`OFFICIAL_VECTOR_TIMEOUT_SECONDS`, `OFFICIAL_VECTOR_MAX_RESPONSE_MB`,
+13
View File
@@ -152,6 +152,19 @@ class Settings(BaseSettings):
le=8760,
validation_alias="OFFICIAL_VECTOR_CACHE_TTL_HOURS",
)
spw_picc_enabled: bool = Field(default=True, validation_alias="SPW_PICC_ENABLED")
spw_picc_mapserver_url: str = Field(
default=(
"https://geoservices.wallonie.be/arcgis/rest/services/"
"TOPOGRAPHIE/PICC_VDIFF/MapServer"
),
validation_alias="SPW_PICC_MAPSERVER_URL",
)
urbis_enabled: bool = Field(default=True, validation_alias="URBIS_ENABLED")
urbis_wfs_url: str = Field(
default="https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows",
validation_alias="URBIS_WFS_URL",
)
dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED")
dhmv_wcs_url: str = Field(
default="https://geo.api.vlaanderen.be/DHMV/wcs",
+1
View File
@@ -32,6 +32,7 @@ class OfficialVectorProductRead(BaseModel):
attribution: str
license_note: str
limitation_message: str
coverage_zones: list[str]
class OfficialVectorAcquisitionResult(BaseModel):
@@ -79,6 +79,7 @@ class _SourceDefinition:
contract: CoverageSourceContract
materialized_layer_names: tuple[str, ...] = ()
materialized_source_names: tuple[str, ...] = ()
operational_themes: tuple[str, ...] = ()
def _contract(
@@ -98,6 +99,7 @@ def _contract(
limitation_message: str,
materialized_layer_names: tuple[str, ...] = (),
materialized_source_names: tuple[str, ...] = (),
operational_themes: tuple[str, ...] = (),
) -> _SourceDefinition:
return _SourceDefinition(
contract=CoverageSourceContract(
@@ -117,6 +119,7 @@ def _contract(
),
materialized_layer_names=materialized_layer_names,
materialized_source_names=materialized_source_names or (source_name,),
operational_themes=operational_themes,
)
@@ -156,12 +159,17 @@ SOURCE_DEFINITIONS = (
themes=("admin", "population"),
native_layers=("statistical_sectors", "population_statistics"),
geometry_types=("Polygon", "MultiPolygon", "Tabular"),
acquisition_mode="catalog_only",
integration_status="not_configured",
acquisition_mode="operator_archive",
integration_status="operational",
source_url="https://statbel.fgov.be/en/open-data",
attribution="Statbel",
license_note="Consult the license of the selected Statbel release.",
limitation_message="The catalog is audited, but no national bounded acquisition adapter is configured yet.",
limitation_message=(
"National editions require the governed plan-stage-review-apply operator; "
"population in partially selected sectors is area-weighted."
),
materialized_layer_names=("population",),
operational_themes=("population",),
),
_contract(
source_name="digitaal_vlaanderen",
@@ -219,12 +227,17 @@ SOURCE_DEFINITIONS = (
),
native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"),
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
acquisition_mode="catalog_only",
integration_status="not_configured",
acquisition_mode="bounded_api",
integration_status="operational",
source_url="https://geoportail.wallonie.be/catalogue",
attribution="Service public de Wallonie",
license_note="Consult the license of each Geoportail Wallonie product.",
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.",
limitation_message=(
"Bounded PICC buildings, road axes and hydrography are operational; "
"other Walloon themes remain unavailable until separately governed."
),
materialized_source_names=("spw_picc",),
operational_themes=("buildings", "roads", "surface_water"),
),
_contract(
source_name="urbis",
@@ -234,12 +247,17 @@ SOURCE_DEFINITIONS = (
themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"),
native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"),
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
acquisition_mode="catalog_only",
integration_status="not_configured",
acquisition_mode="bounded_api",
integration_status="operational",
source_url="https://datastore.brussels",
attribution="Brussels UrbIS",
license_note="Consult the license of the selected UrbIS dataset.",
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.",
limitation_message=(
"Bounded UrbIS buildings and cadastral parcels are operational; "
"other Brussels themes remain unavailable until separately governed."
),
materialized_source_names=("urbis",),
operational_themes=("buildings", "parcels"),
),
_contract(
source_name="rbins_marine_reporting_units",
@@ -330,6 +348,18 @@ FLANDERS_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = {
"flood_climate": {"vmm_flood_hazard": ()},
}
REGIONAL_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = {
"spw_geoportail": {
"buildings": ("buildings",),
"roads": ("roads",),
"surface_water": ("water",),
},
"urbis": {
"buildings": ("buildings",),
"parcels": ("parcels",),
},
}
class CoverageRegistryService:
@staticmethod
@@ -403,6 +433,11 @@ class CoverageRegistryService:
if dataset.source_name not in theme_sources:
continue
layer_names = theme_sources[dataset.source_name]
elif definition.contract.source_name in REGIONAL_THEME_DATASETS:
theme_layers = REGIONAL_THEME_DATASETS[definition.contract.source_name].get(theme)
if theme_layers is None:
continue
layer_names = theme_layers
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or []
if isinstance(coverage_zones, str):
@@ -448,10 +483,17 @@ class CoverageRegistryService:
materialized.extend(matches)
if matches:
source_statuses.append("operational")
elif definition.contract.integration_status == "operational":
elif (
definition.contract.integration_status == "operational"
and (not definition.operational_themes or theme in definition.operational_themes)
):
source_statuses.append("partial")
else:
source_statuses.append(definition.contract.integration_status)
source_statuses.append(
"not_configured"
if definition.contract.integration_status == "operational"
else definition.contract.integration_status
)
limitations.append(definition.contract.limitation_message)
best_status = max(source_statuses, key=STATUS_RANK.__getitem__)
@@ -15,7 +15,7 @@ from uuid import UUID
from geoalchemy2.shape import to_shape
from pyproj import Transformer
from shapely.geometry import MultiPolygon, Polygon, box, mapping, shape
from shapely.geometry import LineString, MultiLineString, MultiPolygon, Polygon, box, mapping, shape
from shapely.ops import transform, unary_union
from shapely.validation import make_valid
@@ -60,11 +60,17 @@ class OfficialVectorProduct:
license_note: str
limitation_message: str
source: str
observed_at: datetime
observed_at: datetime | None
valid_from: datetime | None
valid_to: datetime | None
primary_metric: dict[str, Any]
selection_metrics: tuple[dict[str, Any], ...]
geometry_types: tuple[str, ...] = ("Polygon", "MultiPolygon")
coverage_zones: tuple[str, ...] = ("flanders",)
endpoint_kind: str = "wfs"
response_crs: str = "EPSG:4326"
identity_field: str | None = None
requires_coverage_area: bool = False
class OfficialVectorAcquisitionService:
@@ -178,6 +184,7 @@ class OfficialVectorAcquisitionService:
"warning_only_when_estimate": False,
},
),
endpoint_kind="bwk_wfs",
),
OfficialVectorProduct(
key="dov_soil_types",
@@ -252,6 +259,307 @@ class OfficialVectorAcquisitionService:
"filter_values": ["Antropogeen"],
},
),
endpoint_kind="dov_wfs",
),
OfficialVectorProduct(
key="spw_picc_buildings",
display_name="PICC building footprints",
theme="buildings",
provider="Service public de Wallonie",
source_name="spw_picc",
reference_layer_name="buildings",
service_type="ArcGIS REST",
collection="11",
source_crs="EPSG:3812",
source_version="2026-07-11",
observation_label="Weekly updated PICC snapshot",
authority_level="authoritative",
catalog_url=(
"https://geoportail.wallonie.be/catalogue/"
"b795de68-726c-4bdf-a62a-a42686aa5b6f.html"
),
attribution="Service public de Wallonie (SPW) - PICC",
license_note="CC BY 4.0; cite SPW PICC and identify modifications.",
limitation_message=(
"Topographic building footprints from PICC; these are not cadastral parcels "
"or legal building registrations."
),
source="SPW PICC ArcGIS REST",
observed_at=None,
valid_from=None,
valid_to=None,
primary_metric={
"metric_key": "building_footprint_area",
"method": "intersection_area",
"label": "Bebouwde voetafdruk",
"unit": "ha",
"geometry_dimension": 2,
"is_estimate": False,
},
selection_metrics=(
{
"metric_key": "building_count",
"method": "feature_count",
"label": "Gebouwen",
"unit": "objecten",
"geometry_dimension": 2,
},
),
coverage_zones=("wallonia",),
endpoint_kind="spw_arcgis",
identity_field="GEOREF_ID",
requires_coverage_area=True,
),
OfficialVectorProduct(
key="spw_picc_roads",
display_name="PICC road axes",
theme="roads",
provider="Service public de Wallonie",
source_name="spw_picc",
reference_layer_name="roads",
service_type="ArcGIS REST",
collection="21",
source_crs="EPSG:3812",
source_version="2026-07-11",
observation_label="Weekly updated PICC snapshot",
authority_level="authoritative",
catalog_url=(
"https://geoportail.wallonie.be/catalogue/"
"b795de68-726c-4bdf-a62a-a42686aa5b6f.html"
),
attribution="Service public de Wallonie (SPW) - PICC",
license_note="CC BY 4.0; cite SPW PICC and identify modifications.",
limitation_message=(
"PICC road axes describe topographic road geometry and are not a routing "
"network or a traffic measurement."
),
source="SPW PICC ArcGIS REST",
observed_at=None,
valid_from=None,
valid_to=None,
primary_metric={
"metric_key": "road_length",
"method": "intersection_length",
"label": "Wegaslengte",
"unit": "km",
"geometry_dimension": 1,
"is_estimate": False,
},
selection_metrics=(
{
"metric_key": "road_segment_count",
"method": "feature_count",
"label": "Wegsegmenten",
"unit": "objecten",
"geometry_dimension": 1,
},
),
geometry_types=("LineString", "MultiLineString"),
coverage_zones=("wallonia",),
endpoint_kind="spw_arcgis",
identity_field="GEOREF_ID",
requires_coverage_area=True,
),
OfficialVectorProduct(
key="spw_picc_waterways",
display_name="PICC hydrographic axes",
theme="water",
provider="Service public de Wallonie",
source_name="spw_picc",
reference_layer_name="water",
service_type="ArcGIS REST",
collection="28",
source_crs="EPSG:3812",
source_version="2026-07-11",
observation_label="Weekly updated PICC snapshot",
authority_level="authoritative",
catalog_url=(
"https://geoportail.wallonie.be/catalogue/"
"b795de68-726c-4bdf-a62a-a42686aa5b6f.html"
),
attribution="Service public de Wallonie (SPW) - PICC",
license_note="CC BY 4.0; cite SPW PICC and identify modifications.",
limitation_message=(
"Hydrographic axes describe mapped centre lines. They do not provide depth, "
"discharge or water volume."
),
source="SPW PICC ArcGIS REST",
observed_at=None,
valid_from=None,
valid_to=None,
primary_metric={
"metric_key": "waterway_length",
"method": "intersection_length",
"label": "Waterlooplengte",
"unit": "km",
"geometry_dimension": 1,
"is_estimate": False,
},
selection_metrics=(
{
"metric_key": "waterway_segment_count",
"method": "feature_count",
"label": "Waterloopsegmenten",
"unit": "objecten",
"geometry_dimension": 1,
},
),
geometry_types=("LineString", "MultiLineString"),
coverage_zones=("wallonia",),
endpoint_kind="spw_arcgis",
identity_field="GEOREF_ID",
requires_coverage_area=True,
),
OfficialVectorProduct(
key="spw_picc_water_surfaces",
display_name="PICC hydrographic surfaces",
theme="water",
provider="Service public de Wallonie",
source_name="spw_picc",
reference_layer_name="water",
service_type="ArcGIS REST",
collection="30",
source_crs="EPSG:3812",
source_version="2026-07-11",
observation_label="Weekly updated PICC snapshot",
authority_level="authoritative",
catalog_url=(
"https://geoportail.wallonie.be/catalogue/"
"b795de68-726c-4bdf-a62a-a42686aa5b6f.html"
),
attribution="Service public de Wallonie (SPW) - PICC",
license_note="CC BY 4.0; cite SPW PICC and identify modifications.",
limitation_message=(
"Mapped hydrographic surface area is not water volume and does not imply "
"a measured water level."
),
source="SPW PICC ArcGIS REST",
observed_at=None,
valid_from=None,
valid_to=None,
primary_metric={
"metric_key": "water_surface_area",
"method": "intersection_area",
"label": "Wateroppervlakte",
"unit": "ha",
"geometry_dimension": 2,
"is_estimate": False,
},
selection_metrics=(
{
"metric_key": "water_surface_count",
"method": "feature_count",
"label": "Wateroppervlakken",
"unit": "objecten",
"geometry_dimension": 2,
},
),
coverage_zones=("wallonia",),
endpoint_kind="spw_arcgis",
identity_field="GEOREF_ID",
requires_coverage_area=True,
),
OfficialVectorProduct(
key="urbis_buildings",
display_name="UrbIS buildings",
theme="buildings",
provider="Paradigm Brussels",
source_name="urbis",
reference_layer_name="buildings",
service_type="WFS 2.0",
collection="urbisvector:Buildings",
source_crs="EPSG:31370",
source_version="2026-06-06",
observation_label="UrbIS revision 6 June 2026",
authority_level="authoritative",
catalog_url=(
"https://datastore.brussels/web/data/dataset/"
"2cf42541-1813-11ef-8a81-00090ffe0001"
),
attribution="Paradigm Brussels - UrbIS",
license_note="Buildings are published under CC0.",
limitation_message=(
"UrbIS building geometry is a regional topographic reference and is not "
"a legal cadastral registration."
),
source="UrbIS WFS",
observed_at=datetime(2026, 6, 6, tzinfo=UTC),
valid_from=None,
valid_to=None,
primary_metric={
"metric_key": "building_footprint_area",
"method": "intersection_area",
"label": "Bebouwde voetafdruk",
"unit": "ha",
"geometry_dimension": 2,
"is_estimate": False,
},
selection_metrics=(
{
"metric_key": "building_count",
"method": "feature_count",
"label": "Gebouwen",
"unit": "objecten",
"geometry_dimension": 2,
},
),
coverage_zones=("brussels",),
endpoint_kind="urbis_wfs",
response_crs="EPSG:31370",
identity_field="INSPIRE_ID",
requires_coverage_area=True,
),
OfficialVectorProduct(
key="urbis_cadastral_parcels",
display_name="UrbIS cadastral parcels",
theme="parcels",
provider="Paradigm Brussels / FPS Finance",
source_name="urbis",
reference_layer_name="parcels",
service_type="WFS 2.0",
collection="urbisvector:CadastralParcels",
source_crs="EPSG:31370",
source_version="2026-06-06",
observation_label="UrbIS revision 6 June 2026",
authority_level="authoritative",
catalog_url=(
"https://datastore.brussels/web/data/dataset/"
"2cf42541-1813-11ef-8a81-00090ffe0001"
),
attribution="Paradigm Brussels and FPS Finance - cadastral parcel plan",
license_note=(
"The FPS Finance open-data cadastral plan licence applies to cadastral parcels."
),
limitation_message=(
"Cadastral parcel geometry is reference data. GeoIntel does not infer "
"ownership, rights or legal boundaries beyond the published source."
),
source="UrbIS WFS",
observed_at=datetime(2026, 1, 1, tzinfo=UTC),
valid_from=None,
valid_to=None,
primary_metric={
"metric_key": "parcel_area",
"method": "intersection_area",
"label": "Perceeloppervlakte",
"unit": "ha",
"geometry_dimension": 2,
"is_estimate": False,
},
selection_metrics=(
{
"metric_key": "parcel_count",
"method": "feature_count",
"label": "Percelen",
"unit": "objecten",
"geometry_dimension": 2,
},
),
coverage_zones=("brussels",),
endpoint_kind="urbis_wfs",
response_crs="EPSG:31370",
identity_field="INSPIRE_ID",
requires_coverage_area=True,
),
)
return {product.key: product for product in products}
@@ -268,7 +576,7 @@ class OfficialVectorAcquisitionService:
reference_layer_name=product.reference_layer_name,
service_type=product.service_type,
collection=product.collection,
geometry_types=["Polygon", "MultiPolygon"],
geometry_types=list(product.geometry_types),
source_crs=product.source_crs,
source_version=product.source_version,
observation_label=product.observation_label,
@@ -277,6 +585,7 @@ class OfficialVectorAcquisitionService:
attribution=product.attribution,
license_note=product.license_note,
limitation_message=product.limitation_message,
coverage_zones=list(product.coverage_zones),
).model_dump()
for product in OfficialVectorAcquisitionService._products().values()
]
@@ -320,12 +629,44 @@ class OfficialVectorAcquisitionService:
result = make_valid(result)
return result if not result.is_empty and result.is_valid else None
@staticmethod
def _dimensional(geometry: Any, dimension: int) -> Any | None:
if geometry is None or geometry.is_empty:
return None
if not geometry.is_valid:
geometry = make_valid(geometry)
parts: list[Any] = []
def collect(item: Any) -> None:
if item is None or item.is_empty:
return
if dimension == 2 and isinstance(item, Polygon):
parts.append(item)
elif dimension == 2 and isinstance(item, MultiPolygon):
parts.extend(part for part in item.geoms if not part.is_empty)
elif dimension == 1 and isinstance(item, LineString):
parts.append(item)
elif dimension == 1 and isinstance(item, MultiLineString):
parts.extend(part for part in item.geoms if not part.is_empty)
elif hasattr(item, "geoms"):
for part in item.geoms:
collect(part)
collect(geometry)
if not parts:
return None
result = unary_union(parts)
if not result.is_valid:
result = make_valid(result)
return result if not result.is_empty and result.is_valid else None
@staticmethod
def _validate_scope(
db,
project_id: UUID,
payload: OfficialVectorAcquireRequest,
settings: Settings,
product: OfficialVectorProduct,
) -> tuple[Any, Any, list[float], list[float]]:
if not settings.official_vector_enabled:
raise AppError(
@@ -333,6 +674,18 @@ class OfficialVectorAcquisitionService:
message="Bounded official vector acquisition is disabled",
status_code=503,
)
if product.endpoint_kind == "spw_arcgis" and not settings.spw_picc_enabled:
raise AppError(
code="SPW_PICC_NOT_CONFIGURED",
message="Bounded SPW PICC acquisition is disabled",
status_code=503,
)
if product.endpoint_kind == "urbis_wfs" and not settings.urbis_enabled:
raise AppError(
code="URBIS_NOT_CONFIGURED",
message="Bounded UrbIS acquisition is disabled",
status_code=503,
)
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
if payload.bbox.crs.upper() != "EPSG:4326":
@@ -375,6 +728,7 @@ class OfficialVectorAcquisitionService:
status_code=422,
)
scope_wgs84 = box(*values)
area = None
if payload.area_id:
area = db.get(Area, payload.area_id)
if area is None:
@@ -394,6 +748,53 @@ class OfficialVectorAcquisitionService:
message="The selection does not intersect the selected area",
status_code=400,
)
if product.requires_coverage_area:
coverage_area_names = {
"wallonia": "Wallonia",
"brussels": "Brussels-Capital Region",
}
required_names = [
coverage_area_names[zone]
for zone in product.coverage_zones
if zone in coverage_area_names
]
coverage_rows = (
[area]
if area is not None and area.name in required_names
else db.query(Area)
.filter(
Area.project_id == project_id,
Area.name.in_(required_names),
)
.all()
)
coverage_geometries = [
to_shape(item.geometry)
for item in coverage_rows
if item is not None and item.geometry is not None
]
coverage_geometry = (
OfficialVectorAcquisitionService._polygonal(unary_union(coverage_geometries))
if coverage_geometries
else None
)
if coverage_geometry is None:
raise AppError(
code="OFFICIAL_VECTOR_COVERAGE_NOT_READY",
message="The official regional coverage boundary is not persisted in this project",
details={"required_areas": required_names},
status_code=409,
)
scope_wgs84 = OfficialVectorAcquisitionService._polygonal(
scope_wgs84.intersection(coverage_geometry)
)
if scope_wgs84 is None:
raise AppError(
code="OFFICIAL_VECTOR_OUTSIDE_COVERAGE",
message="The selection does not intersect the official product coverage",
details={"coverage_zones": list(product.coverage_zones)},
status_code=422,
)
scope_metric = OfficialVectorAcquisitionService._polygonal(
transform(_TO_LAMBERT72.transform, scope_wgs84)
)
@@ -425,6 +826,21 @@ class OfficialVectorAcquisitionService:
request,
timeout=settings.official_vector_timeout_seconds,
) as response:
content_type = ""
if hasattr(response, "getheader"):
content_type = str(response.getheader("Content-Type") or "")
elif hasattr(response, "headers"):
content_type = str(response.headers.get("Content-Type") or "")
if content_type and not any(
allowed in content_type.lower()
for allowed in ("application/json", "application/geo+json")
):
raise AppError(
code="OFFICIAL_VECTOR_PROVIDER_INVALID_CONTENT_TYPE",
message="The official vector provider returned an unsupported content type",
details={"content_type": content_type},
status_code=502,
)
limit = settings.official_vector_max_response_mb * 1024 * 1024
content = response.read(limit + 1)
except HTTPError as exc:
@@ -502,6 +918,137 @@ class OfficialVectorAcquisitionService:
)
return f"{settings.dov_soil_wfs_url.rstrip('?')}?{query}"
@staticmethod
def _spw_url(
settings: Settings,
product: OfficialVectorProduct,
bbox_values: tuple[float, ...],
start_index: int,
) -> str:
query = urlencode(
{
"where": "1=1",
"geometry": ",".join(f"{value:.8f}" for value in bbox_values),
"geometryType": "esriGeometryEnvelope",
"inSR": "4326",
"outSR": "4326",
"spatialRel": "esriSpatialRelIntersects",
"outFields": "*",
"returnGeometry": "true",
"returnZ": "false",
"returnM": "false",
"resultOffset": start_index,
"resultRecordCount": min(settings.official_vector_page_size, 2000),
"orderByFields": "OBJECTID",
"f": "geojson",
}
)
base = settings.spw_picc_mapserver_url.rstrip("/")
return f"{base}/{product.collection}/query?{query}"
@staticmethod
def _urbis_url(
settings: Settings,
product: OfficialVectorProduct,
metric_bbox: tuple[float, ...],
start_index: int,
) -> str:
query = urlencode(
{
"service": "WFS",
"version": "2.0.0",
"request": "GetFeature",
"typeNames": product.collection,
"srsName": "EPSG:31370",
"bbox": ",".join(f"{value:.3f}" for value in metric_bbox) + ",EPSG:31370",
"count": settings.official_vector_page_size,
"startIndex": start_index,
"sortBy": product.identity_field or "INSPIRE_ID",
"outputFormat": "application/json",
}
)
return f"{settings.urbis_wfs_url.rstrip('?')}?{query}"
@staticmethod
def _page_url(
product: OfficialVectorProduct,
settings: Settings,
scope_wgs84: Any,
scope_metric: Any,
start_index: int,
) -> str:
if product.endpoint_kind == "bwk_wfs":
return OfficialVectorAcquisitionService._nature_url(
settings,
tuple(scope_wgs84.bounds),
start_index,
)
if product.endpoint_kind == "dov_wfs":
return OfficialVectorAcquisitionService._soil_url(
settings,
tuple(scope_metric.bounds),
start_index,
)
if product.endpoint_kind == "spw_arcgis":
return OfficialVectorAcquisitionService._spw_url(
settings,
product,
tuple(scope_wgs84.bounds),
start_index,
)
if product.endpoint_kind == "urbis_wfs":
return OfficialVectorAcquisitionService._urbis_url(
settings,
product,
tuple(scope_metric.bounds),
start_index,
)
raise AppError(
code="OFFICIAL_VECTOR_PRODUCT_NOT_SUPPORTED",
message="The official vector product has no governed acquisition adapter",
status_code=422,
)
@staticmethod
def _validate_page_url(
url: str,
product: OfficialVectorProduct,
settings: Settings,
) -> None:
parsed = urlparse(url)
configured_url = {
"bwk_wfs": settings.bwk_wfs_url,
"dov_wfs": settings.dov_soil_wfs_url,
"spw_arcgis": settings.spw_picc_mapserver_url,
"urbis_wfs": settings.urbis_wfs_url,
}.get(product.endpoint_kind)
if configured_url is None:
raise AppError(
code="OFFICIAL_VECTOR_PRODUCT_NOT_SUPPORTED",
message="The official vector product has no configured endpoint",
status_code=422,
)
base = urlparse(configured_url)
expected_path = (
f"{base.path.rstrip('/')}/{product.collection}/query"
if product.endpoint_kind == "spw_arcgis"
else base.path
)
if (
parsed.scheme != "https"
or base.scheme != "https"
or parsed.netloc.casefold() != base.netloc.casefold()
or parsed.path != expected_path
or parsed.username
or parsed.password
or parsed.fragment
):
raise AppError(
code="OFFICIAL_VECTOR_PROVIDER_INVALID_PAGINATION",
message="The official vector request escaped the governed HTTPS endpoint",
status_code=502,
)
@staticmethod
def _habitat_breakdown(properties: dict[str, Any]) -> tuple[list[dict[str, Any]], float, float, float]:
entries: list[dict[str, Any]] = []
@@ -527,6 +1074,92 @@ class OfficialVectorAcquisitionService:
uncertain_share = 100.0
return entries, min(100.0, natura_share), min(100.0, regional_share), min(100.0, uncertain_share)
@staticmethod
def _normalize_regional_feature(
product: OfficialVectorProduct,
feature: dict[str, Any],
scope_metric: Any,
coverage_scope: str,
) -> dict[str, Any] | None:
dimension = 2 if any("Polygon" in item for item in product.geometry_types) else 1
try:
source_geometry = shape(feature.get("geometry"))
except Exception as exc:
raise AppError(
code="OFFICIAL_VECTOR_PROVIDER_INVALID_GEOMETRY",
message=f"{product.display_name} returned invalid geometry",
status_code=502,
) from exc
if product.response_crs == "EPSG:31370":
source_metric = OfficialVectorAcquisitionService._dimensional(
source_geometry,
dimension,
)
else:
source_wgs84 = OfficialVectorAcquisitionService._dimensional(
source_geometry,
dimension,
)
source_metric = (
OfficialVectorAcquisitionService._dimensional(
transform(_TO_LAMBERT72.transform, source_wgs84),
dimension,
)
if source_wgs84 is not None
else None
)
if source_metric is None or not source_metric.intersects(scope_metric):
return None
clipped_metric = OfficialVectorAcquisitionService._dimensional(
source_metric.intersection(scope_metric),
dimension,
)
if clipped_metric is None:
return None
clipped_wgs84 = OfficialVectorAcquisitionService._dimensional(
transform(_TO_WGS84.transform, clipped_metric),
dimension,
)
if clipped_wgs84 is None:
return None
raw = dict(feature.get("properties") or {})
identity = (
raw.get(product.identity_field or "")
or feature.get("id")
or raw.get("OBJECTID")
)
if identity in (None, ""):
raise AppError(
code="OFFICIAL_VECTOR_PROVIDER_INVALID_RESPONSE",
message=f"{product.display_name} returned a feature without an official identity",
status_code=502,
)
feature_id = f"{product.collection}:{identity}"
properties = {
**raw,
"source_name": product.source_name,
"source_collection": product.collection,
"source_feature_id": feature_id,
"reference_layer_name": product.reference_layer_name,
"theme": product.theme,
"authority_level": product.authority_level,
"coverage_scope": coverage_scope,
"coverage_zones": list(product.coverage_zones),
"source_version": product.source_version,
"attribution": product.attribution,
"geometry_clipped_to_selection": not scope_metric.covers(source_metric),
}
if dimension == 2:
properties["clipped_area_ha"] = round(float(clipped_metric.area) / 10_000.0, 8)
else:
properties["clipped_length_km"] = round(float(clipped_metric.length) / 1_000.0, 8)
return {
"type": "Feature",
"id": feature_id,
"geometry": mapping(clipped_wgs84),
"properties": properties,
}
@staticmethod
def _normalize_feature(
product: OfficialVectorProduct,
@@ -534,6 +1167,13 @@ class OfficialVectorAcquisitionService:
scope_metric: Any,
coverage_scope: str,
) -> dict[str, Any] | None:
if product.endpoint_kind in {"spw_arcgis", "urbis_wfs"}:
return OfficialVectorAcquisitionService._normalize_regional_feature(
product,
feature,
scope_metric,
coverage_scope,
)
try:
source_wgs84 = OfficialVectorAcquisitionService._polygonal(shape(feature.get("geometry")))
except Exception as exc:
@@ -667,32 +1307,21 @@ class OfficialVectorAcquisitionService:
response_hashes: list[str] = []
total_bytes = candidate_count = 0
expected_total: int | None = None
next_url = (
OfficialVectorAcquisitionService._nature_url(settings, tuple(scope_wgs84.bounds), 0)
if product.theme == "nature_value"
else OfficialVectorAcquisitionService._soil_url(settings, tuple(scope_metric.bounds), 0)
next_url = OfficialVectorAcquisitionService._page_url(
product,
settings,
scope_wgs84,
scope_metric,
0,
)
start_index = 0
seen_pages: set[str] = set()
while next_url:
parsed = urlparse(next_url)
configured_url = (
settings.bwk_wfs_url
if product.theme == "nature_value"
else settings.dov_soil_wfs_url
OfficialVectorAcquisitionService._validate_page_url(
next_url,
product,
settings,
)
base = urlparse(configured_url)
if (
parsed.scheme != "https"
or base.scheme != "https"
or parsed.netloc.casefold() != base.netloc.casefold()
or parsed.path != base.path
):
raise AppError(
code="OFFICIAL_VECTOR_PROVIDER_INVALID_PAGINATION",
message="The official WFS request escaped the governed endpoint",
status_code=502,
)
if next_url in seen_pages:
raise AppError(
code="OFFICIAL_VECTOR_PROVIDER_PAGINATION_LOOP",
@@ -777,7 +1406,26 @@ class OfficialVectorAcquisitionService:
status_code=502,
)
start_index += returned_count
if returned_count == 0 or (
arcgis_has_more = payload.get("exceededTransferLimit") is True
if product.endpoint_kind == "spw_arcgis":
if arcgis_has_more and returned_count == 0:
raise AppError(
code="OFFICIAL_VECTOR_PROVIDER_INCOMPLETE_RESPONSE",
message="The SPW provider reported more records but returned an empty page",
status_code=502,
)
next_url = (
OfficialVectorAcquisitionService._page_url(
product,
settings,
scope_wgs84,
scope_metric,
start_index,
)
if arcgis_has_more
else None
)
elif returned_count == 0 or (
expected_total is not None and start_index >= expected_total
) or (expected_total is None and returned_count < settings.official_vector_page_size):
if expected_total is not None and start_index != expected_total:
@@ -788,13 +1436,13 @@ class OfficialVectorAcquisitionService:
status_code=502,
)
next_url = None
elif product.theme == "nature_value":
next_url = OfficialVectorAcquisitionService._nature_url(
settings, tuple(scope_wgs84.bounds), start_index
)
else:
next_url = OfficialVectorAcquisitionService._soil_url(
settings, tuple(scope_metric.bounds), start_index
next_url = OfficialVectorAcquisitionService._page_url(
product,
settings,
scope_wgs84,
scope_metric,
start_index,
)
return retained, {
"candidate_feature_count": candidate_count,
@@ -884,7 +1532,7 @@ class OfficialVectorAcquisitionService:
product = OfficialVectorAcquisitionService._product(payload.product_key)
scope_wgs84, scope_metric, bbox_values, metric_bounds = (
OfficialVectorAcquisitionService._validate_scope(
db, project_id, payload, resolved_settings
db, project_id, payload, resolved_settings, product
)
)
request_identity = {
@@ -906,9 +1554,13 @@ class OfficialVectorAcquisitionService:
)
area = db.get(Area, payload.area_id) if payload.area_id else None
coverage_scope = (
"municipality"
if area is not None and area.name.strip().lower().startswith("gemeente ")
else "bounded_selection"
product.coverage_zones[0]
if product.requires_coverage_area
else (
"municipality"
if area is not None and area.name.strip().lower().startswith("gemeente ")
else "bounded_selection"
)
)
features, transfer = OfficialVectorAcquisitionService._fetch_features(
product,
@@ -942,6 +1594,7 @@ class OfficialVectorAcquisitionService:
"theme": product.theme,
"layer_type": product.reference_layer_name,
"coverage_scope": coverage_scope,
"coverage_zones": list(product.coverage_zones),
"geometry_clipped_to_area": payload.area_id is not None,
"geometry_clipped_to_selection": True,
"bbox_epsg4326": bbox_values,
@@ -994,7 +1647,7 @@ class OfficialVectorAcquisitionService:
dataset_role="reference",
reference_layer_name=product.reference_layer_name,
temporal_series_key=f"{product.source_name}:{product.key}:{request_hash[:24]}",
observed_at=product.observed_at,
observed_at=product.observed_at or acquired_at,
valid_from=product.valid_from,
valid_to=product.valid_to,
temporal_granularity="period" if product.valid_from else "snapshot",
@@ -45,6 +45,19 @@ PRECLIPPED_MUNICIPALITY_PARTITION_OPERATOR_TOOLS = {
SEMANTIC_SELECTION_METRICS: dict[str, tuple[dict[str, Any], ...]] = {
"administrative": (
{
"metric_key": "covered_area",
"method": "intersection_area",
"label": "Bestuurlijk ingedeelde oppervlakte",
"unit": "ha",
"geometry_dimension": 2,
"warning": (
"Dit is de doorsnede met één bestuurlijk schaalniveau uit de gekozen NGI-laag; "
"het is geen kadastrale of juridische grensopmeting."
),
},
),
"buildings": (
{
"metric_key": "footprint_area",
@@ -113,9 +126,14 @@ SEMANTIC_SELECTION_METRICS: dict[str, tuple[dict[str, Any], ...]] = {
"warning": "Historische bodemkartering op schaal 1:20.000; actuele lokale bodem- en drainagetoestand kan afwijken.",
},
),
# Maritieme plan- en rapportagezones kunnen elkaar overlappen. Een
# opgetelde oppervlakte zou daarom geen unieke zeeoppervlakte voorstellen.
"maritime_planning": (),
"marine_environment": (),
}
SEMANTIC_COUNT_LABELS = {
"administrative": "Bestuursgebieden",
"buildings": "Gebouwen",
"population": "Statistische sectoren",
"forest": "Bosvlakken",
@@ -125,6 +143,8 @@ SEMANTIC_COUNT_LABELS = {
"nature_value": "BWK-kaartvlakken",
"agriculture": "Landbouwgebruikspercelen",
"soil": "Bodemkaartvlakken",
"maritime_planning": "Maritieme planobjecten",
"marine_environment": "Mariene rapportagezones",
}
# Sprint 205 initially normalized two official comma-separated ALZ group labels
@@ -157,6 +177,12 @@ class VectorFeatureService:
source_metadata.get("layer_type"),
)
aliases = {
"belgium_land_boundary": "administrative",
"belgium_regions": "administrative",
"belgium_provinces": "administrative",
"belgium_municipalities": "administrative",
"marine_spatial_plan_2026": "maritime_planning",
"marine_legal_scopes": "marine_environment",
"building": "buildings",
"bebouwing": "buildings",
"population": "population",
@@ -250,6 +250,33 @@ def test_runtime_sets_writable_ultralytics_config_directory() -> None:
assert "YOLO_CONFIG_DIR=/app/storage/ultralytics" in unraid_env
def test_regional_official_vector_sources_are_configurable_in_every_runtime() -> None:
compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
unraid_compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(
encoding="utf-8"
)
env_example = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(
encoding="utf-8"
)
template = (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(
encoding="utf-8"
)
for key in (
"SPW_PICC_ENABLED",
"SPW_PICC_MAPSERVER_URL",
"URBIS_ENABLED",
"URBIS_WFS_URL",
):
assert key in compose
assert key in unraid_compose
assert f'{key}="${{{key}:-' in run_script
assert f'-e {key}="${key}"' in run_script
assert f"{key}=" in env_example
assert f'Target="{key}"' in template
def test_docker_build_contexts_exclude_vendor_build_and_cache_outputs() -> None:
required_patterns = {
"node_modules",
@@ -0,0 +1,308 @@
from __future__ import annotations
import json
from types import SimpleNamespace
from urllib.parse import parse_qs, urlparse
from uuid import uuid4
import pytest
from geoalchemy2.shape import from_shape
from pyproj import Transformer
from shapely.geometry import MultiPolygon, Polygon
from app.core.config import Settings
from app.core.errors import AppError
from app.models import Area, Dataset, Project
from app.schemas.official_vector import OfficialVectorAcquireRequest
from app.services.dataset_service import DatasetService
from app.services.official_vector_acquisition_service import (
OfficialVectorAcquisitionService,
_TO_LAMBERT72,
)
class FakeQuery:
def __init__(self, result=None):
self.result = result
def filter(self, *_args):
return self
def order_by(self, *_args):
return self
def all(self):
return self.result if isinstance(self.result, list) else []
class FakeSession:
def __init__(self, rows=None, query_result=None):
self.rows = rows or {}
self.query_result = query_result
def get(self, model, row_id):
return self.rows.get((model, row_id))
def query(self, _model):
return FakeQuery(self.query_result)
class JsonResponse:
def __init__(self, payload, content_type="application/geo+json"):
self.content = json.dumps(payload).encode("utf-8")
self.content_type = content_type
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, size=-1):
return self.content if size < 0 else self.content[:size]
def getheader(self, name):
return self.content_type if name.lower() == "content-type" else None
def request(product_key: str, bbox: tuple[float, float, float, float], area_id=None):
return OfficialVectorAcquireRequest(
bbox={
"min_x": bbox[0],
"min_y": bbox[1],
"max_x": bbox[2],
"max_y": bbox[3],
"crs": "EPSG:4326",
},
area_id=area_id,
product_key=product_key,
force_refresh=True,
)
def area(project_id, name: str, bounds: tuple[float, float, float, float]):
min_x, min_y, max_x, max_y = bounds
geometry = MultiPolygon(
[
Polygon(
[
(min_x, min_y),
(max_x, min_y),
(max_x, max_y),
(min_x, max_y),
(min_x, min_y),
]
)
]
)
return Area(
id=uuid4(),
project_id=project_id,
name=name,
geometry=from_shape(geometry, srid=4326),
)
def test_regional_product_registry_is_explicit_and_source_specific() -> None:
products = {
item["key"]: item
for item in OfficialVectorAcquisitionService.list_products()
}
assert products["spw_picc_buildings"]["coverage_zones"] == ["wallonia"]
assert products["spw_picc_roads"]["geometry_types"] == [
"LineString",
"MultiLineString",
]
assert products["spw_picc_waterways"]["collection"] == "28"
assert products["spw_picc_water_surfaces"]["collection"] == "30"
assert products["urbis_buildings"]["coverage_zones"] == ["brussels"]
assert products["urbis_buildings"]["license_note"] == "Buildings are published under CC0."
assert "FPS Finance" in products["urbis_cadastral_parcels"]["license_note"]
def test_spw_arcgis_paging_is_bounded_stable_and_clipped() -> None:
product = OfficialVectorAcquisitionService._product("spw_picc_buildings")
scope = Polygon(
[(4.55, 50.58), (4.56, 50.58), (4.56, 50.59), (4.55, 50.59), (4.55, 50.58)]
)
scope_metric = Polygon(
[
_TO_LAMBERT72.transform(x, y)
for x, y in scope.exterior.coords
]
)
offsets = []
def feature(object_id: int, min_x: float):
return {
"type": "Feature",
"id": object_id,
"geometry": {
"type": "Polygon",
"coordinates": [[
[min_x, 50.581],
[min_x + 0.002, 50.581],
[min_x + 0.002, 50.583],
[min_x, 50.583],
[min_x, 50.581],
]],
},
"properties": {"OBJECTID": object_id, "GEOREF_ID": f"wallonia-{object_id}"},
}
def opener(raw_request, timeout):
assert timeout == 180
query = parse_qs(urlparse(raw_request.full_url).query)
assert query["orderByFields"] == ["OBJECTID"]
assert query["f"] == ["geojson"]
offset = int(query["resultOffset"][0])
offsets.append(offset)
return JsonResponse(
{
"type": "FeatureCollection",
"features": [feature(offset + 1, 4.551 + offset * 0.0001)],
"exceededTransferLimit": offset == 0,
}
)
features, transfer = OfficialVectorAcquisitionService._fetch_features(
product,
scope,
scope_metric,
"wallonia",
Settings(_env_file=None, OFFICIAL_VECTOR_PAGE_SIZE=1),
opener,
)
assert offsets == [0, 1]
assert transfer["page_count"] == 2
assert transfer["reference_truncated"] is False
assert {item["properties"]["source_feature_id"] for item in features} == {
"11:wallonia-1",
"11:wallonia-2",
}
assert all(item["properties"]["coverage_scope"] == "wallonia" for item in features)
assert all(item["properties"]["clipped_area_ha"] > 0 for item in features)
def test_regional_products_require_the_persisted_authoritative_coverage_area() -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
with pytest.raises(AppError) as exc_info:
OfficialVectorAcquisitionService.acquire(
db,
project_id,
request("spw_picc_buildings", (4.55, 50.58, 4.56, 50.59)),
settings=Settings(_env_file=None),
)
assert exc_info.value.code == "OFFICIAL_VECTOR_COVERAGE_NOT_READY"
def test_urbis_wfs_transforms_lambert72_and_persists_through_dataset_service(
monkeypatch,
) -> None:
project_id, dataset_id = uuid4(), uuid4()
brussels = area(project_id, "Brussels-Capital Region", (4.25, 50.75, 4.5, 50.95))
db = FakeSession(
{
(Project, project_id): Project(id=project_id, name="Belgium"),
(Area, brussels.id): brussels,
},
query_result=[brussels],
)
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
min_x, min_y = to_lambert.transform(4.35, 50.84)
max_x, max_y = to_lambert.transform(4.351, 50.841)
captured = {}
def opener(raw_request, timeout):
assert timeout == 180
query = parse_qs(urlparse(raw_request.full_url).query)
assert query["typeNames"] == ["urbisvector:Buildings"]
assert query["srsName"] == ["EPSG:31370"]
assert query["sortBy"] == ["INSPIRE_ID"]
return JsonResponse(
{
"type": "FeatureCollection",
"numberMatched": 1,
"numberReturned": 1,
"features": [
{
"type": "Feature",
"id": "Buildings.1",
"geometry": {
"type": "MultiPolygon",
"coordinates": [[[
[min_x, min_y],
[max_x, min_y],
[max_x, max_y],
[min_x, max_y],
[min_x, min_y],
]]],
},
"properties": {
"INSPIRE_ID": "https://databrussels.be/id/building/1",
"AREA": 75,
},
}
],
},
"application/json",
)
def persist(_db, **kwargs):
captured.update(kwargs)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
area_id=brussels.id,
name=kwargs["filename"],
dataset_type="vector",
source=kwargs["source"],
dataset_role=kwargs["dataset_role"],
source_name=kwargs["source_name"],
reference_layer_name=kwargs["reference_layer_name"],
temporal_series_key=kwargs["temporal_series_key"],
observed_at=kwargs["observed_at"],
source_version=kwargs["source_version"],
source_metadata=kwargs["source_metadata"],
provenance_metadata=kwargs["provenance_metadata"],
metadata_json={"feature_count": 1},
status="ready",
)
db.rows[(Dataset, dataset_id)] = dataset
return SimpleNamespace(id=dataset_id)
monkeypatch.setattr(DatasetService, "import_vector_bytes", persist)
result = OfficialVectorAcquisitionService.acquire(
db,
project_id,
request(
"urbis_buildings",
(4.349, 50.839, 4.352, 50.842),
area_id=brussels.id,
),
settings=Settings(_env_file=None),
opener=opener,
)
assert result["output_dataset_id"] == str(dataset_id)
assert captured["source_name"] == "urbis"
assert captured["reference_layer_name"] == "buildings"
assert captured["source_metadata"]["coverage_zones"] == ["brussels"]
assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == (
"building_footprint_area"
)
collection = json.loads(captured["content"])
geometry = collection["features"][0]["geometry"]
assert geometry["type"] in {"Polygon", "MultiPolygon"}
first_coordinate = (
geometry["coordinates"][0][0][0]
if geometry["type"] == "MultiPolygon"
else geometry["coordinates"][0][0]
)
assert 4.34999 <= first_coordinate[0] <= 4.35101
assert 50.83999 <= first_coordinate[1] <= 50.84101
@@ -12,6 +12,7 @@ from app.main import app
from app.models import Area, Dataset, Project
from app.schemas.coverage import CoverageBBox
from app.services.coverage_registry_service import CoverageRegistryService, THEMES, ZONES
from app.services.vector_feature_service import VectorFeatureService
class FakeQuery:
@@ -72,6 +73,44 @@ def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider
assert response.json()["data"]["themes"] == list(THEMES)
def test_national_and_maritime_reference_layers_are_selection_analyzable() -> None:
cases = (
("ngi_adminvector", "belgium_municipalities", "administrative"),
("rbins_marine_reporting_units", "marine_legal_scopes", "marine_environment"),
("rbins_msp_2026", "marine_spatial_plan_2026", "maritime_planning"),
)
for source_name, layer_name, expected_theme in cases:
dataset = Dataset(
id=uuid4(),
project_id=uuid4(),
name=f"{layer_name}.geojson",
dataset_type="vector",
source="operator_official_import",
source_name=source_name,
reference_layer_name=layer_name,
source_metadata={"authority_level": "authoritative"},
status="ready",
)
assert VectorFeatureService._dataset_theme(dataset) == expected_theme
assert VectorFeatureService.supports_selection_summary(dataset) is True
def test_national_scope_operator_assigns_explicit_map_themes() -> None:
root = Path(__file__).resolve().parents[2]
operator = (root / "scripts" / "provision_belgium_north_sea_scope.py").read_text(encoding="utf-8")
map_workspace = (root / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(
encoding="utf-8"
)
assert '"belgium_municipalities": "administrative"' in operator
assert '"marine_legal_scopes": "marine_environment"' in operator
assert '"marine_spatial_plan_2026": "maritime_planning"' in operator
assert "id: 'administrative'" in map_workspace
assert "id: 'maritime_planning'" in map_workspace
assert "id: 'marine_environment'" in map_workspace
def test_coverage_resolver_only_reports_operational_for_materialized_ready_dataset() -> None:
project_id = uuid4()
project = SimpleNamespace(id=project_id)
@@ -55,16 +55,22 @@ def test_population_operator_filters_to_the_approved_scope() -> None:
module = load_script("provision_mol_population_history.py")
regional = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]
mol = module.GEOGRAPHIC_SCOPES["mol"]
belgium = module.GEOGRAPHIC_SCOPES["belgium"]
regional_rows = module.population_rows(population_archive(), regional)
mol_rows = module.population_rows(population_archive(), mol)
national_rows = module.population_rows(population_archive(), belgium)
assert set(regional_rows) == {"13025A00-", "13008A00-"}
assert regional_rows["13008A00-"]["municipality"] == "Geel"
assert regional_rows["13008A00-"]["nis_code"] == "13008"
assert set(mol_rows) == {"13025A00-"}
assert set(national_rows) == {"13025A00-", "13008A00-", "11002A00-"}
assert national_rows["11002A00-"]["municipality"] == "Antwerpen"
assert belgium.all_municipalities is True
assert module.series_key(regional) == "statbel:population-statistical-sector:kempen-transport-region"
assert module.series_key(mol) == "statbel:population-statistical-sector:mol"
assert module.series_key(belgium) == "statbel:population-statistical-sector:belgium"
def test_population_operator_resolves_the_persisted_scope_boundary(tmp_path: Path) -> None:
@@ -90,6 +96,48 @@ def test_population_operator_resolves_the_persisted_scope_boundary(tmp_path: Pat
assert module.resolve_boundary_path(args, scope) == boundary
def test_population_operator_resolves_checksum_verified_belgium_boundary(tmp_path: Path) -> None:
module = load_script("provision_mol_population_history.py")
scope = module.GEOGRAPHIC_SCOPES["belgium"]
scope_dir = tmp_path / "belgium-north-sea"
scope_dir.mkdir(parents=True)
boundary = scope_dir / "belgium_land_boundary.geojson"
boundary.write_text(
json.dumps(
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {
"type": "Polygon",
"coordinates": [[[2.5, 49.5], [6.4, 49.5], [6.4, 51.5], [2.5, 51.5], [2.5, 49.5]]],
},
"properties": {},
}
],
}
),
encoding="utf-8",
)
(scope_dir / "manifest.json").write_text(
json.dumps(
{
"scope": "belgium-and-belgian-north-sea",
"artifacts": {
"belgium_land_boundary": {
"sha256": module.sha256_path(boundary),
}
},
}
),
encoding="utf-8",
)
args = argparse.Namespace(boundary_path=None, scope_output_root=tmp_path)
assert module.resolve_boundary_path(args, scope) == boundary
def test_regional_coordinator_builds_explicit_population_and_landuse_commands(tmp_path: Path) -> None:
module = load_script("provision_regional_timeseries.py")
scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]
@@ -205,6 +205,42 @@ def test_preflight_reconciles_spatial_and_unlocated_population(tmp_path: Path) -
assert len(manifest["schemas"]["geometry_schema_sha256"]) == 64
def test_preflight_supports_the_complete_national_scope() -> None:
result = PREFLIGHT.validate_statbel_release(
year=2025,
layout="new",
population_content=population_archive(),
population_url=POPULATION_URL,
geometry_content=geometry_archive(),
geometry_url=GEOMETRY_URL,
scope=PREFLIGHT.GEOGRAPHIC_SCOPES["belgium"],
)
accounting = result.manifest["scope_accounting"]
assert accounting["scope_key"] == "belgium"
assert accounting["member_count"] == 2
assert accounting["member_nis_codes"] == ["13008", "13025"]
assert accounting["spatial_population_total"] == 240
assert accounting["unlocated_population_total"] == 3
assert accounting["accounted_population_total"] == 243
def test_national_preflight_rejects_an_unscoped_baseline(tmp_path: Path) -> None:
with pytest.raises(PREFLIGHT.StatbelPreflightError) as exc_info:
PREFLIGHT.validate_statbel_release(
year=2025,
layout="new",
population_content=population_archive(),
population_url=POPULATION_URL,
geometry_content=geometry_archive(),
geometry_url=GEOMETRY_URL,
scope=PREFLIGHT.GEOGRAPHIC_SCOPES["belgium"],
baseline_snapshot=baseline_snapshot(tmp_path / "baseline.geojson"),
)
assert exc_info.value.code == "STATBEL_BASELINE_SCOPE_MISMATCH"
def test_preflight_reports_bounded_topology_repairs(tmp_path: Path) -> None:
result = validate(tmp_path, geometry_content=geometry_archive(repairable_invalid=True))
@@ -125,7 +125,16 @@ def test_product_registries_expose_honest_forest_agriculture_nature_and_soil() -
assert raster["agricultural_land_use_2025"]["included_source_values"] == [13, 14]
assert "geen juridische bosgrens" in raster["forest_land_use_2025"]["limitation_message"].lower()
assert "geen alz-perceelaangifte" in raster["agricultural_land_use_2025"]["limitation_message"].lower()
assert set(vector) == {"bwk_natura2000_2025", "dov_soil_types"}
assert {
"bwk_natura2000_2025",
"dov_soil_types",
"spw_picc_buildings",
"spw_picc_roads",
"spw_picc_waterways",
"spw_picc_water_surfaces",
"urbis_buildings",
"urbis_cadastral_parcels",
} == set(vector)
assert vector["bwk_natura2000_2025"]["authority_level"] == "authoritative"
assert vector["dov_soil_types"]["authority_level"] == "authoritative_historical_baseline"
assert "1949-1971" in vector["dov_soil_types"]["observation_label"]
@@ -392,7 +401,7 @@ def test_official_vector_routes_and_frontend_use_canonical_backend_path(monkeypa
assert products_response.status_code == 200
assert set(products_response.json()) == {"data"}
assert products_response.json()["data"]["total"] == 2
assert products_response.json()["data"]["total"] == 8
assert acquire_response.status_code == 200
assert set(acquire_response.json()) == {"data"}
assert acquire_response.json()["data"]["job_type"] == "vector.official.acquire"