Federate official Belgium data sources
This commit is contained in:
Codex
@@ -79,6 +79,7 @@ class _SourceDefinition:
|
||||
contract: CoverageSourceContract
|
||||
materialized_layer_names: tuple[str, ...] = ()
|
||||
materialized_source_names: tuple[str, ...] = ()
|
||||
operational_themes: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def _contract(
|
||||
@@ -98,6 +99,7 @@ def _contract(
|
||||
limitation_message: str,
|
||||
materialized_layer_names: tuple[str, ...] = (),
|
||||
materialized_source_names: tuple[str, ...] = (),
|
||||
operational_themes: tuple[str, ...] = (),
|
||||
) -> _SourceDefinition:
|
||||
return _SourceDefinition(
|
||||
contract=CoverageSourceContract(
|
||||
@@ -117,6 +119,7 @@ def _contract(
|
||||
),
|
||||
materialized_layer_names=materialized_layer_names,
|
||||
materialized_source_names=materialized_source_names or (source_name,),
|
||||
operational_themes=operational_themes,
|
||||
)
|
||||
|
||||
|
||||
@@ -156,12 +159,17 @@ SOURCE_DEFINITIONS = (
|
||||
themes=("admin", "population"),
|
||||
native_layers=("statistical_sectors", "population_statistics"),
|
||||
geometry_types=("Polygon", "MultiPolygon", "Tabular"),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
acquisition_mode="operator_archive",
|
||||
integration_status="operational",
|
||||
source_url="https://statbel.fgov.be/en/open-data",
|
||||
attribution="Statbel",
|
||||
license_note="Consult the license of the selected Statbel release.",
|
||||
limitation_message="The catalog is audited, but no national bounded acquisition adapter is configured yet.",
|
||||
limitation_message=(
|
||||
"National editions require the governed plan-stage-review-apply operator; "
|
||||
"population in partially selected sectors is area-weighted."
|
||||
),
|
||||
materialized_layer_names=("population",),
|
||||
operational_themes=("population",),
|
||||
),
|
||||
_contract(
|
||||
source_name="digitaal_vlaanderen",
|
||||
@@ -219,12 +227,17 @@ SOURCE_DEFINITIONS = (
|
||||
),
|
||||
native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
acquisition_mode="bounded_api",
|
||||
integration_status="operational",
|
||||
source_url="https://geoportail.wallonie.be/catalogue",
|
||||
attribution="Service public de Wallonie",
|
||||
license_note="Consult the license of each Geoportail Wallonie product.",
|
||||
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.",
|
||||
limitation_message=(
|
||||
"Bounded PICC buildings, road axes and hydrography are operational; "
|
||||
"other Walloon themes remain unavailable until separately governed."
|
||||
),
|
||||
materialized_source_names=("spw_picc",),
|
||||
operational_themes=("buildings", "roads", "surface_water"),
|
||||
),
|
||||
_contract(
|
||||
source_name="urbis",
|
||||
@@ -234,12 +247,17 @@ SOURCE_DEFINITIONS = (
|
||||
themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"),
|
||||
native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
acquisition_mode="bounded_api",
|
||||
integration_status="operational",
|
||||
source_url="https://datastore.brussels",
|
||||
attribution="Brussels UrbIS",
|
||||
license_note="Consult the license of the selected UrbIS dataset.",
|
||||
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.",
|
||||
limitation_message=(
|
||||
"Bounded UrbIS buildings and cadastral parcels are operational; "
|
||||
"other Brussels themes remain unavailable until separately governed."
|
||||
),
|
||||
materialized_source_names=("urbis",),
|
||||
operational_themes=("buildings", "parcels"),
|
||||
),
|
||||
_contract(
|
||||
source_name="rbins_marine_reporting_units",
|
||||
@@ -330,6 +348,18 @@ FLANDERS_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = {
|
||||
"flood_climate": {"vmm_flood_hazard": ()},
|
||||
}
|
||||
|
||||
REGIONAL_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = {
|
||||
"spw_geoportail": {
|
||||
"buildings": ("buildings",),
|
||||
"roads": ("roads",),
|
||||
"surface_water": ("water",),
|
||||
},
|
||||
"urbis": {
|
||||
"buildings": ("buildings",),
|
||||
"parcels": ("parcels",),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class CoverageRegistryService:
|
||||
@staticmethod
|
||||
@@ -403,6 +433,11 @@ class CoverageRegistryService:
|
||||
if dataset.source_name not in theme_sources:
|
||||
continue
|
||||
layer_names = theme_sources[dataset.source_name]
|
||||
elif definition.contract.source_name in REGIONAL_THEME_DATASETS:
|
||||
theme_layers = REGIONAL_THEME_DATASETS[definition.contract.source_name].get(theme)
|
||||
if theme_layers is None:
|
||||
continue
|
||||
layer_names = theme_layers
|
||||
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||
coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or []
|
||||
if isinstance(coverage_zones, str):
|
||||
@@ -448,10 +483,17 @@ class CoverageRegistryService:
|
||||
materialized.extend(matches)
|
||||
if matches:
|
||||
source_statuses.append("operational")
|
||||
elif definition.contract.integration_status == "operational":
|
||||
elif (
|
||||
definition.contract.integration_status == "operational"
|
||||
and (not definition.operational_themes or theme in definition.operational_themes)
|
||||
):
|
||||
source_statuses.append("partial")
|
||||
else:
|
||||
source_statuses.append(definition.contract.integration_status)
|
||||
source_statuses.append(
|
||||
"not_configured"
|
||||
if definition.contract.integration_status == "operational"
|
||||
else definition.contract.integration_status
|
||||
)
|
||||
limitations.append(definition.contract.limitation_message)
|
||||
|
||||
best_status = max(source_statuses, key=STATUS_RANK.__getitem__)
|
||||
|
||||
@@ -15,7 +15,7 @@ from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import MultiPolygon, Polygon, box, mapping, shape
|
||||
from shapely.geometry import LineString, MultiLineString, MultiPolygon, Polygon, box, mapping, shape
|
||||
from shapely.ops import transform, unary_union
|
||||
from shapely.validation import make_valid
|
||||
|
||||
@@ -60,11 +60,17 @@ class OfficialVectorProduct:
|
||||
license_note: str
|
||||
limitation_message: str
|
||||
source: str
|
||||
observed_at: datetime
|
||||
observed_at: datetime | None
|
||||
valid_from: datetime | None
|
||||
valid_to: datetime | None
|
||||
primary_metric: dict[str, Any]
|
||||
selection_metrics: tuple[dict[str, Any], ...]
|
||||
geometry_types: tuple[str, ...] = ("Polygon", "MultiPolygon")
|
||||
coverage_zones: tuple[str, ...] = ("flanders",)
|
||||
endpoint_kind: str = "wfs"
|
||||
response_crs: str = "EPSG:4326"
|
||||
identity_field: str | None = None
|
||||
requires_coverage_area: bool = False
|
||||
|
||||
|
||||
class OfficialVectorAcquisitionService:
|
||||
@@ -178,6 +184,7 @@ class OfficialVectorAcquisitionService:
|
||||
"warning_only_when_estimate": False,
|
||||
},
|
||||
),
|
||||
endpoint_kind="bwk_wfs",
|
||||
),
|
||||
OfficialVectorProduct(
|
||||
key="dov_soil_types",
|
||||
@@ -252,6 +259,307 @@ class OfficialVectorAcquisitionService:
|
||||
"filter_values": ["Antropogeen"],
|
||||
},
|
||||
),
|
||||
endpoint_kind="dov_wfs",
|
||||
),
|
||||
OfficialVectorProduct(
|
||||
key="spw_picc_buildings",
|
||||
display_name="PICC building footprints",
|
||||
theme="buildings",
|
||||
provider="Service public de Wallonie",
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="buildings",
|
||||
service_type="ArcGIS REST",
|
||||
collection="11",
|
||||
source_crs="EPSG:3812",
|
||||
source_version="2026-07-11",
|
||||
observation_label="Weekly updated PICC snapshot",
|
||||
authority_level="authoritative",
|
||||
catalog_url=(
|
||||
"https://geoportail.wallonie.be/catalogue/"
|
||||
"b795de68-726c-4bdf-a62a-a42686aa5b6f.html"
|
||||
),
|
||||
attribution="Service public de Wallonie (SPW) - PICC",
|
||||
license_note="CC BY 4.0; cite SPW PICC and identify modifications.",
|
||||
limitation_message=(
|
||||
"Topographic building footprints from PICC; these are not cadastral parcels "
|
||||
"or legal building registrations."
|
||||
),
|
||||
source="SPW PICC ArcGIS REST",
|
||||
observed_at=None,
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
primary_metric={
|
||||
"metric_key": "building_footprint_area",
|
||||
"method": "intersection_area",
|
||||
"label": "Bebouwde voetafdruk",
|
||||
"unit": "ha",
|
||||
"geometry_dimension": 2,
|
||||
"is_estimate": False,
|
||||
},
|
||||
selection_metrics=(
|
||||
{
|
||||
"metric_key": "building_count",
|
||||
"method": "feature_count",
|
||||
"label": "Gebouwen",
|
||||
"unit": "objecten",
|
||||
"geometry_dimension": 2,
|
||||
},
|
||||
),
|
||||
coverage_zones=("wallonia",),
|
||||
endpoint_kind="spw_arcgis",
|
||||
identity_field="GEOREF_ID",
|
||||
requires_coverage_area=True,
|
||||
),
|
||||
OfficialVectorProduct(
|
||||
key="spw_picc_roads",
|
||||
display_name="PICC road axes",
|
||||
theme="roads",
|
||||
provider="Service public de Wallonie",
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="roads",
|
||||
service_type="ArcGIS REST",
|
||||
collection="21",
|
||||
source_crs="EPSG:3812",
|
||||
source_version="2026-07-11",
|
||||
observation_label="Weekly updated PICC snapshot",
|
||||
authority_level="authoritative",
|
||||
catalog_url=(
|
||||
"https://geoportail.wallonie.be/catalogue/"
|
||||
"b795de68-726c-4bdf-a62a-a42686aa5b6f.html"
|
||||
),
|
||||
attribution="Service public de Wallonie (SPW) - PICC",
|
||||
license_note="CC BY 4.0; cite SPW PICC and identify modifications.",
|
||||
limitation_message=(
|
||||
"PICC road axes describe topographic road geometry and are not a routing "
|
||||
"network or a traffic measurement."
|
||||
),
|
||||
source="SPW PICC ArcGIS REST",
|
||||
observed_at=None,
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
primary_metric={
|
||||
"metric_key": "road_length",
|
||||
"method": "intersection_length",
|
||||
"label": "Wegaslengte",
|
||||
"unit": "km",
|
||||
"geometry_dimension": 1,
|
||||
"is_estimate": False,
|
||||
},
|
||||
selection_metrics=(
|
||||
{
|
||||
"metric_key": "road_segment_count",
|
||||
"method": "feature_count",
|
||||
"label": "Wegsegmenten",
|
||||
"unit": "objecten",
|
||||
"geometry_dimension": 1,
|
||||
},
|
||||
),
|
||||
geometry_types=("LineString", "MultiLineString"),
|
||||
coverage_zones=("wallonia",),
|
||||
endpoint_kind="spw_arcgis",
|
||||
identity_field="GEOREF_ID",
|
||||
requires_coverage_area=True,
|
||||
),
|
||||
OfficialVectorProduct(
|
||||
key="spw_picc_waterways",
|
||||
display_name="PICC hydrographic axes",
|
||||
theme="water",
|
||||
provider="Service public de Wallonie",
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="water",
|
||||
service_type="ArcGIS REST",
|
||||
collection="28",
|
||||
source_crs="EPSG:3812",
|
||||
source_version="2026-07-11",
|
||||
observation_label="Weekly updated PICC snapshot",
|
||||
authority_level="authoritative",
|
||||
catalog_url=(
|
||||
"https://geoportail.wallonie.be/catalogue/"
|
||||
"b795de68-726c-4bdf-a62a-a42686aa5b6f.html"
|
||||
),
|
||||
attribution="Service public de Wallonie (SPW) - PICC",
|
||||
license_note="CC BY 4.0; cite SPW PICC and identify modifications.",
|
||||
limitation_message=(
|
||||
"Hydrographic axes describe mapped centre lines. They do not provide depth, "
|
||||
"discharge or water volume."
|
||||
),
|
||||
source="SPW PICC ArcGIS REST",
|
||||
observed_at=None,
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
primary_metric={
|
||||
"metric_key": "waterway_length",
|
||||
"method": "intersection_length",
|
||||
"label": "Waterlooplengte",
|
||||
"unit": "km",
|
||||
"geometry_dimension": 1,
|
||||
"is_estimate": False,
|
||||
},
|
||||
selection_metrics=(
|
||||
{
|
||||
"metric_key": "waterway_segment_count",
|
||||
"method": "feature_count",
|
||||
"label": "Waterloopsegmenten",
|
||||
"unit": "objecten",
|
||||
"geometry_dimension": 1,
|
||||
},
|
||||
),
|
||||
geometry_types=("LineString", "MultiLineString"),
|
||||
coverage_zones=("wallonia",),
|
||||
endpoint_kind="spw_arcgis",
|
||||
identity_field="GEOREF_ID",
|
||||
requires_coverage_area=True,
|
||||
),
|
||||
OfficialVectorProduct(
|
||||
key="spw_picc_water_surfaces",
|
||||
display_name="PICC hydrographic surfaces",
|
||||
theme="water",
|
||||
provider="Service public de Wallonie",
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="water",
|
||||
service_type="ArcGIS REST",
|
||||
collection="30",
|
||||
source_crs="EPSG:3812",
|
||||
source_version="2026-07-11",
|
||||
observation_label="Weekly updated PICC snapshot",
|
||||
authority_level="authoritative",
|
||||
catalog_url=(
|
||||
"https://geoportail.wallonie.be/catalogue/"
|
||||
"b795de68-726c-4bdf-a62a-a42686aa5b6f.html"
|
||||
),
|
||||
attribution="Service public de Wallonie (SPW) - PICC",
|
||||
license_note="CC BY 4.0; cite SPW PICC and identify modifications.",
|
||||
limitation_message=(
|
||||
"Mapped hydrographic surface area is not water volume and does not imply "
|
||||
"a measured water level."
|
||||
),
|
||||
source="SPW PICC ArcGIS REST",
|
||||
observed_at=None,
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
primary_metric={
|
||||
"metric_key": "water_surface_area",
|
||||
"method": "intersection_area",
|
||||
"label": "Wateroppervlakte",
|
||||
"unit": "ha",
|
||||
"geometry_dimension": 2,
|
||||
"is_estimate": False,
|
||||
},
|
||||
selection_metrics=(
|
||||
{
|
||||
"metric_key": "water_surface_count",
|
||||
"method": "feature_count",
|
||||
"label": "Wateroppervlakken",
|
||||
"unit": "objecten",
|
||||
"geometry_dimension": 2,
|
||||
},
|
||||
),
|
||||
coverage_zones=("wallonia",),
|
||||
endpoint_kind="spw_arcgis",
|
||||
identity_field="GEOREF_ID",
|
||||
requires_coverage_area=True,
|
||||
),
|
||||
OfficialVectorProduct(
|
||||
key="urbis_buildings",
|
||||
display_name="UrbIS buildings",
|
||||
theme="buildings",
|
||||
provider="Paradigm Brussels",
|
||||
source_name="urbis",
|
||||
reference_layer_name="buildings",
|
||||
service_type="WFS 2.0",
|
||||
collection="urbisvector:Buildings",
|
||||
source_crs="EPSG:31370",
|
||||
source_version="2026-06-06",
|
||||
observation_label="UrbIS revision 6 June 2026",
|
||||
authority_level="authoritative",
|
||||
catalog_url=(
|
||||
"https://datastore.brussels/web/data/dataset/"
|
||||
"2cf42541-1813-11ef-8a81-00090ffe0001"
|
||||
),
|
||||
attribution="Paradigm Brussels - UrbIS",
|
||||
license_note="Buildings are published under CC0.",
|
||||
limitation_message=(
|
||||
"UrbIS building geometry is a regional topographic reference and is not "
|
||||
"a legal cadastral registration."
|
||||
),
|
||||
source="UrbIS WFS",
|
||||
observed_at=datetime(2026, 6, 6, tzinfo=UTC),
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
primary_metric={
|
||||
"metric_key": "building_footprint_area",
|
||||
"method": "intersection_area",
|
||||
"label": "Bebouwde voetafdruk",
|
||||
"unit": "ha",
|
||||
"geometry_dimension": 2,
|
||||
"is_estimate": False,
|
||||
},
|
||||
selection_metrics=(
|
||||
{
|
||||
"metric_key": "building_count",
|
||||
"method": "feature_count",
|
||||
"label": "Gebouwen",
|
||||
"unit": "objecten",
|
||||
"geometry_dimension": 2,
|
||||
},
|
||||
),
|
||||
coverage_zones=("brussels",),
|
||||
endpoint_kind="urbis_wfs",
|
||||
response_crs="EPSG:31370",
|
||||
identity_field="INSPIRE_ID",
|
||||
requires_coverage_area=True,
|
||||
),
|
||||
OfficialVectorProduct(
|
||||
key="urbis_cadastral_parcels",
|
||||
display_name="UrbIS cadastral parcels",
|
||||
theme="parcels",
|
||||
provider="Paradigm Brussels / FPS Finance",
|
||||
source_name="urbis",
|
||||
reference_layer_name="parcels",
|
||||
service_type="WFS 2.0",
|
||||
collection="urbisvector:CadastralParcels",
|
||||
source_crs="EPSG:31370",
|
||||
source_version="2026-06-06",
|
||||
observation_label="UrbIS revision 6 June 2026",
|
||||
authority_level="authoritative",
|
||||
catalog_url=(
|
||||
"https://datastore.brussels/web/data/dataset/"
|
||||
"2cf42541-1813-11ef-8a81-00090ffe0001"
|
||||
),
|
||||
attribution="Paradigm Brussels and FPS Finance - cadastral parcel plan",
|
||||
license_note=(
|
||||
"The FPS Finance open-data cadastral plan licence applies to cadastral parcels."
|
||||
),
|
||||
limitation_message=(
|
||||
"Cadastral parcel geometry is reference data. GeoIntel does not infer "
|
||||
"ownership, rights or legal boundaries beyond the published source."
|
||||
),
|
||||
source="UrbIS WFS",
|
||||
observed_at=datetime(2026, 1, 1, tzinfo=UTC),
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
primary_metric={
|
||||
"metric_key": "parcel_area",
|
||||
"method": "intersection_area",
|
||||
"label": "Perceeloppervlakte",
|
||||
"unit": "ha",
|
||||
"geometry_dimension": 2,
|
||||
"is_estimate": False,
|
||||
},
|
||||
selection_metrics=(
|
||||
{
|
||||
"metric_key": "parcel_count",
|
||||
"method": "feature_count",
|
||||
"label": "Percelen",
|
||||
"unit": "objecten",
|
||||
"geometry_dimension": 2,
|
||||
},
|
||||
),
|
||||
coverage_zones=("brussels",),
|
||||
endpoint_kind="urbis_wfs",
|
||||
response_crs="EPSG:31370",
|
||||
identity_field="INSPIRE_ID",
|
||||
requires_coverage_area=True,
|
||||
),
|
||||
)
|
||||
return {product.key: product for product in products}
|
||||
@@ -268,7 +576,7 @@ class OfficialVectorAcquisitionService:
|
||||
reference_layer_name=product.reference_layer_name,
|
||||
service_type=product.service_type,
|
||||
collection=product.collection,
|
||||
geometry_types=["Polygon", "MultiPolygon"],
|
||||
geometry_types=list(product.geometry_types),
|
||||
source_crs=product.source_crs,
|
||||
source_version=product.source_version,
|
||||
observation_label=product.observation_label,
|
||||
@@ -277,6 +585,7 @@ class OfficialVectorAcquisitionService:
|
||||
attribution=product.attribution,
|
||||
license_note=product.license_note,
|
||||
limitation_message=product.limitation_message,
|
||||
coverage_zones=list(product.coverage_zones),
|
||||
).model_dump()
|
||||
for product in OfficialVectorAcquisitionService._products().values()
|
||||
]
|
||||
@@ -320,12 +629,44 @@ class OfficialVectorAcquisitionService:
|
||||
result = make_valid(result)
|
||||
return result if not result.is_empty and result.is_valid else None
|
||||
|
||||
@staticmethod
|
||||
def _dimensional(geometry: Any, dimension: int) -> Any | None:
|
||||
if geometry is None or geometry.is_empty:
|
||||
return None
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
parts: list[Any] = []
|
||||
|
||||
def collect(item: Any) -> None:
|
||||
if item is None or item.is_empty:
|
||||
return
|
||||
if dimension == 2 and isinstance(item, Polygon):
|
||||
parts.append(item)
|
||||
elif dimension == 2 and isinstance(item, MultiPolygon):
|
||||
parts.extend(part for part in item.geoms if not part.is_empty)
|
||||
elif dimension == 1 and isinstance(item, LineString):
|
||||
parts.append(item)
|
||||
elif dimension == 1 and isinstance(item, MultiLineString):
|
||||
parts.extend(part for part in item.geoms if not part.is_empty)
|
||||
elif hasattr(item, "geoms"):
|
||||
for part in item.geoms:
|
||||
collect(part)
|
||||
|
||||
collect(geometry)
|
||||
if not parts:
|
||||
return None
|
||||
result = unary_union(parts)
|
||||
if not result.is_valid:
|
||||
result = make_valid(result)
|
||||
return result if not result.is_empty and result.is_valid else None
|
||||
|
||||
@staticmethod
|
||||
def _validate_scope(
|
||||
db,
|
||||
project_id: UUID,
|
||||
payload: OfficialVectorAcquireRequest,
|
||||
settings: Settings,
|
||||
product: OfficialVectorProduct,
|
||||
) -> tuple[Any, Any, list[float], list[float]]:
|
||||
if not settings.official_vector_enabled:
|
||||
raise AppError(
|
||||
@@ -333,6 +674,18 @@ class OfficialVectorAcquisitionService:
|
||||
message="Bounded official vector acquisition is disabled",
|
||||
status_code=503,
|
||||
)
|
||||
if product.endpoint_kind == "spw_arcgis" and not settings.spw_picc_enabled:
|
||||
raise AppError(
|
||||
code="SPW_PICC_NOT_CONFIGURED",
|
||||
message="Bounded SPW PICC acquisition is disabled",
|
||||
status_code=503,
|
||||
)
|
||||
if product.endpoint_kind == "urbis_wfs" and not settings.urbis_enabled:
|
||||
raise AppError(
|
||||
code="URBIS_NOT_CONFIGURED",
|
||||
message="Bounded UrbIS acquisition is disabled",
|
||||
status_code=503,
|
||||
)
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
if payload.bbox.crs.upper() != "EPSG:4326":
|
||||
@@ -375,6 +728,7 @@ class OfficialVectorAcquisitionService:
|
||||
status_code=422,
|
||||
)
|
||||
scope_wgs84 = box(*values)
|
||||
area = None
|
||||
if payload.area_id:
|
||||
area = db.get(Area, payload.area_id)
|
||||
if area is None:
|
||||
@@ -394,6 +748,53 @@ class OfficialVectorAcquisitionService:
|
||||
message="The selection does not intersect the selected area",
|
||||
status_code=400,
|
||||
)
|
||||
if product.requires_coverage_area:
|
||||
coverage_area_names = {
|
||||
"wallonia": "Wallonia",
|
||||
"brussels": "Brussels-Capital Region",
|
||||
}
|
||||
required_names = [
|
||||
coverage_area_names[zone]
|
||||
for zone in product.coverage_zones
|
||||
if zone in coverage_area_names
|
||||
]
|
||||
coverage_rows = (
|
||||
[area]
|
||||
if area is not None and area.name in required_names
|
||||
else db.query(Area)
|
||||
.filter(
|
||||
Area.project_id == project_id,
|
||||
Area.name.in_(required_names),
|
||||
)
|
||||
.all()
|
||||
)
|
||||
coverage_geometries = [
|
||||
to_shape(item.geometry)
|
||||
for item in coverage_rows
|
||||
if item is not None and item.geometry is not None
|
||||
]
|
||||
coverage_geometry = (
|
||||
OfficialVectorAcquisitionService._polygonal(unary_union(coverage_geometries))
|
||||
if coverage_geometries
|
||||
else None
|
||||
)
|
||||
if coverage_geometry is None:
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_COVERAGE_NOT_READY",
|
||||
message="The official regional coverage boundary is not persisted in this project",
|
||||
details={"required_areas": required_names},
|
||||
status_code=409,
|
||||
)
|
||||
scope_wgs84 = OfficialVectorAcquisitionService._polygonal(
|
||||
scope_wgs84.intersection(coverage_geometry)
|
||||
)
|
||||
if scope_wgs84 is None:
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_OUTSIDE_COVERAGE",
|
||||
message="The selection does not intersect the official product coverage",
|
||||
details={"coverage_zones": list(product.coverage_zones)},
|
||||
status_code=422,
|
||||
)
|
||||
scope_metric = OfficialVectorAcquisitionService._polygonal(
|
||||
transform(_TO_LAMBERT72.transform, scope_wgs84)
|
||||
)
|
||||
@@ -425,6 +826,21 @@ class OfficialVectorAcquisitionService:
|
||||
request,
|
||||
timeout=settings.official_vector_timeout_seconds,
|
||||
) as response:
|
||||
content_type = ""
|
||||
if hasattr(response, "getheader"):
|
||||
content_type = str(response.getheader("Content-Type") or "")
|
||||
elif hasattr(response, "headers"):
|
||||
content_type = str(response.headers.get("Content-Type") or "")
|
||||
if content_type and not any(
|
||||
allowed in content_type.lower()
|
||||
for allowed in ("application/json", "application/geo+json")
|
||||
):
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_PROVIDER_INVALID_CONTENT_TYPE",
|
||||
message="The official vector provider returned an unsupported content type",
|
||||
details={"content_type": content_type},
|
||||
status_code=502,
|
||||
)
|
||||
limit = settings.official_vector_max_response_mb * 1024 * 1024
|
||||
content = response.read(limit + 1)
|
||||
except HTTPError as exc:
|
||||
@@ -502,6 +918,137 @@ class OfficialVectorAcquisitionService:
|
||||
)
|
||||
return f"{settings.dov_soil_wfs_url.rstrip('?')}?{query}"
|
||||
|
||||
@staticmethod
|
||||
def _spw_url(
|
||||
settings: Settings,
|
||||
product: OfficialVectorProduct,
|
||||
bbox_values: tuple[float, ...],
|
||||
start_index: int,
|
||||
) -> str:
|
||||
query = urlencode(
|
||||
{
|
||||
"where": "1=1",
|
||||
"geometry": ",".join(f"{value:.8f}" for value in bbox_values),
|
||||
"geometryType": "esriGeometryEnvelope",
|
||||
"inSR": "4326",
|
||||
"outSR": "4326",
|
||||
"spatialRel": "esriSpatialRelIntersects",
|
||||
"outFields": "*",
|
||||
"returnGeometry": "true",
|
||||
"returnZ": "false",
|
||||
"returnM": "false",
|
||||
"resultOffset": start_index,
|
||||
"resultRecordCount": min(settings.official_vector_page_size, 2000),
|
||||
"orderByFields": "OBJECTID",
|
||||
"f": "geojson",
|
||||
}
|
||||
)
|
||||
base = settings.spw_picc_mapserver_url.rstrip("/")
|
||||
return f"{base}/{product.collection}/query?{query}"
|
||||
|
||||
@staticmethod
|
||||
def _urbis_url(
|
||||
settings: Settings,
|
||||
product: OfficialVectorProduct,
|
||||
metric_bbox: tuple[float, ...],
|
||||
start_index: int,
|
||||
) -> str:
|
||||
query = urlencode(
|
||||
{
|
||||
"service": "WFS",
|
||||
"version": "2.0.0",
|
||||
"request": "GetFeature",
|
||||
"typeNames": product.collection,
|
||||
"srsName": "EPSG:31370",
|
||||
"bbox": ",".join(f"{value:.3f}" for value in metric_bbox) + ",EPSG:31370",
|
||||
"count": settings.official_vector_page_size,
|
||||
"startIndex": start_index,
|
||||
"sortBy": product.identity_field or "INSPIRE_ID",
|
||||
"outputFormat": "application/json",
|
||||
}
|
||||
)
|
||||
return f"{settings.urbis_wfs_url.rstrip('?')}?{query}"
|
||||
|
||||
@staticmethod
|
||||
def _page_url(
|
||||
product: OfficialVectorProduct,
|
||||
settings: Settings,
|
||||
scope_wgs84: Any,
|
||||
scope_metric: Any,
|
||||
start_index: int,
|
||||
) -> str:
|
||||
if product.endpoint_kind == "bwk_wfs":
|
||||
return OfficialVectorAcquisitionService._nature_url(
|
||||
settings,
|
||||
tuple(scope_wgs84.bounds),
|
||||
start_index,
|
||||
)
|
||||
if product.endpoint_kind == "dov_wfs":
|
||||
return OfficialVectorAcquisitionService._soil_url(
|
||||
settings,
|
||||
tuple(scope_metric.bounds),
|
||||
start_index,
|
||||
)
|
||||
if product.endpoint_kind == "spw_arcgis":
|
||||
return OfficialVectorAcquisitionService._spw_url(
|
||||
settings,
|
||||
product,
|
||||
tuple(scope_wgs84.bounds),
|
||||
start_index,
|
||||
)
|
||||
if product.endpoint_kind == "urbis_wfs":
|
||||
return OfficialVectorAcquisitionService._urbis_url(
|
||||
settings,
|
||||
product,
|
||||
tuple(scope_metric.bounds),
|
||||
start_index,
|
||||
)
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_PRODUCT_NOT_SUPPORTED",
|
||||
message="The official vector product has no governed acquisition adapter",
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_page_url(
|
||||
url: str,
|
||||
product: OfficialVectorProduct,
|
||||
settings: Settings,
|
||||
) -> None:
|
||||
parsed = urlparse(url)
|
||||
configured_url = {
|
||||
"bwk_wfs": settings.bwk_wfs_url,
|
||||
"dov_wfs": settings.dov_soil_wfs_url,
|
||||
"spw_arcgis": settings.spw_picc_mapserver_url,
|
||||
"urbis_wfs": settings.urbis_wfs_url,
|
||||
}.get(product.endpoint_kind)
|
||||
if configured_url is None:
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_PRODUCT_NOT_SUPPORTED",
|
||||
message="The official vector product has no configured endpoint",
|
||||
status_code=422,
|
||||
)
|
||||
base = urlparse(configured_url)
|
||||
expected_path = (
|
||||
f"{base.path.rstrip('/')}/{product.collection}/query"
|
||||
if product.endpoint_kind == "spw_arcgis"
|
||||
else base.path
|
||||
)
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or base.scheme != "https"
|
||||
or parsed.netloc.casefold() != base.netloc.casefold()
|
||||
or parsed.path != expected_path
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.fragment
|
||||
):
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_PROVIDER_INVALID_PAGINATION",
|
||||
message="The official vector request escaped the governed HTTPS endpoint",
|
||||
status_code=502,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _habitat_breakdown(properties: dict[str, Any]) -> tuple[list[dict[str, Any]], float, float, float]:
|
||||
entries: list[dict[str, Any]] = []
|
||||
@@ -527,6 +1074,92 @@ class OfficialVectorAcquisitionService:
|
||||
uncertain_share = 100.0
|
||||
return entries, min(100.0, natura_share), min(100.0, regional_share), min(100.0, uncertain_share)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_regional_feature(
|
||||
product: OfficialVectorProduct,
|
||||
feature: dict[str, Any],
|
||||
scope_metric: Any,
|
||||
coverage_scope: str,
|
||||
) -> dict[str, Any] | None:
|
||||
dimension = 2 if any("Polygon" in item for item in product.geometry_types) else 1
|
||||
try:
|
||||
source_geometry = shape(feature.get("geometry"))
|
||||
except Exception as exc:
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_PROVIDER_INVALID_GEOMETRY",
|
||||
message=f"{product.display_name} returned invalid geometry",
|
||||
status_code=502,
|
||||
) from exc
|
||||
if product.response_crs == "EPSG:31370":
|
||||
source_metric = OfficialVectorAcquisitionService._dimensional(
|
||||
source_geometry,
|
||||
dimension,
|
||||
)
|
||||
else:
|
||||
source_wgs84 = OfficialVectorAcquisitionService._dimensional(
|
||||
source_geometry,
|
||||
dimension,
|
||||
)
|
||||
source_metric = (
|
||||
OfficialVectorAcquisitionService._dimensional(
|
||||
transform(_TO_LAMBERT72.transform, source_wgs84),
|
||||
dimension,
|
||||
)
|
||||
if source_wgs84 is not None
|
||||
else None
|
||||
)
|
||||
if source_metric is None or not source_metric.intersects(scope_metric):
|
||||
return None
|
||||
clipped_metric = OfficialVectorAcquisitionService._dimensional(
|
||||
source_metric.intersection(scope_metric),
|
||||
dimension,
|
||||
)
|
||||
if clipped_metric is None:
|
||||
return None
|
||||
clipped_wgs84 = OfficialVectorAcquisitionService._dimensional(
|
||||
transform(_TO_WGS84.transform, clipped_metric),
|
||||
dimension,
|
||||
)
|
||||
if clipped_wgs84 is None:
|
||||
return None
|
||||
raw = dict(feature.get("properties") or {})
|
||||
identity = (
|
||||
raw.get(product.identity_field or "")
|
||||
or feature.get("id")
|
||||
or raw.get("OBJECTID")
|
||||
)
|
||||
if identity in (None, ""):
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_PROVIDER_INVALID_RESPONSE",
|
||||
message=f"{product.display_name} returned a feature without an official identity",
|
||||
status_code=502,
|
||||
)
|
||||
feature_id = f"{product.collection}:{identity}"
|
||||
properties = {
|
||||
**raw,
|
||||
"source_name": product.source_name,
|
||||
"source_collection": product.collection,
|
||||
"source_feature_id": feature_id,
|
||||
"reference_layer_name": product.reference_layer_name,
|
||||
"theme": product.theme,
|
||||
"authority_level": product.authority_level,
|
||||
"coverage_scope": coverage_scope,
|
||||
"coverage_zones": list(product.coverage_zones),
|
||||
"source_version": product.source_version,
|
||||
"attribution": product.attribution,
|
||||
"geometry_clipped_to_selection": not scope_metric.covers(source_metric),
|
||||
}
|
||||
if dimension == 2:
|
||||
properties["clipped_area_ha"] = round(float(clipped_metric.area) / 10_000.0, 8)
|
||||
else:
|
||||
properties["clipped_length_km"] = round(float(clipped_metric.length) / 1_000.0, 8)
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": feature_id,
|
||||
"geometry": mapping(clipped_wgs84),
|
||||
"properties": properties,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_feature(
|
||||
product: OfficialVectorProduct,
|
||||
@@ -534,6 +1167,13 @@ class OfficialVectorAcquisitionService:
|
||||
scope_metric: Any,
|
||||
coverage_scope: str,
|
||||
) -> dict[str, Any] | None:
|
||||
if product.endpoint_kind in {"spw_arcgis", "urbis_wfs"}:
|
||||
return OfficialVectorAcquisitionService._normalize_regional_feature(
|
||||
product,
|
||||
feature,
|
||||
scope_metric,
|
||||
coverage_scope,
|
||||
)
|
||||
try:
|
||||
source_wgs84 = OfficialVectorAcquisitionService._polygonal(shape(feature.get("geometry")))
|
||||
except Exception as exc:
|
||||
@@ -667,32 +1307,21 @@ class OfficialVectorAcquisitionService:
|
||||
response_hashes: list[str] = []
|
||||
total_bytes = candidate_count = 0
|
||||
expected_total: int | None = None
|
||||
next_url = (
|
||||
OfficialVectorAcquisitionService._nature_url(settings, tuple(scope_wgs84.bounds), 0)
|
||||
if product.theme == "nature_value"
|
||||
else OfficialVectorAcquisitionService._soil_url(settings, tuple(scope_metric.bounds), 0)
|
||||
next_url = OfficialVectorAcquisitionService._page_url(
|
||||
product,
|
||||
settings,
|
||||
scope_wgs84,
|
||||
scope_metric,
|
||||
0,
|
||||
)
|
||||
start_index = 0
|
||||
seen_pages: set[str] = set()
|
||||
while next_url:
|
||||
parsed = urlparse(next_url)
|
||||
configured_url = (
|
||||
settings.bwk_wfs_url
|
||||
if product.theme == "nature_value"
|
||||
else settings.dov_soil_wfs_url
|
||||
OfficialVectorAcquisitionService._validate_page_url(
|
||||
next_url,
|
||||
product,
|
||||
settings,
|
||||
)
|
||||
base = urlparse(configured_url)
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or base.scheme != "https"
|
||||
or parsed.netloc.casefold() != base.netloc.casefold()
|
||||
or parsed.path != base.path
|
||||
):
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_PROVIDER_INVALID_PAGINATION",
|
||||
message="The official WFS request escaped the governed endpoint",
|
||||
status_code=502,
|
||||
)
|
||||
if next_url in seen_pages:
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_PROVIDER_PAGINATION_LOOP",
|
||||
@@ -777,7 +1406,26 @@ class OfficialVectorAcquisitionService:
|
||||
status_code=502,
|
||||
)
|
||||
start_index += returned_count
|
||||
if returned_count == 0 or (
|
||||
arcgis_has_more = payload.get("exceededTransferLimit") is True
|
||||
if product.endpoint_kind == "spw_arcgis":
|
||||
if arcgis_has_more and returned_count == 0:
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_PROVIDER_INCOMPLETE_RESPONSE",
|
||||
message="The SPW provider reported more records but returned an empty page",
|
||||
status_code=502,
|
||||
)
|
||||
next_url = (
|
||||
OfficialVectorAcquisitionService._page_url(
|
||||
product,
|
||||
settings,
|
||||
scope_wgs84,
|
||||
scope_metric,
|
||||
start_index,
|
||||
)
|
||||
if arcgis_has_more
|
||||
else None
|
||||
)
|
||||
elif returned_count == 0 or (
|
||||
expected_total is not None and start_index >= expected_total
|
||||
) or (expected_total is None and returned_count < settings.official_vector_page_size):
|
||||
if expected_total is not None and start_index != expected_total:
|
||||
@@ -788,13 +1436,13 @@ class OfficialVectorAcquisitionService:
|
||||
status_code=502,
|
||||
)
|
||||
next_url = None
|
||||
elif product.theme == "nature_value":
|
||||
next_url = OfficialVectorAcquisitionService._nature_url(
|
||||
settings, tuple(scope_wgs84.bounds), start_index
|
||||
)
|
||||
else:
|
||||
next_url = OfficialVectorAcquisitionService._soil_url(
|
||||
settings, tuple(scope_metric.bounds), start_index
|
||||
next_url = OfficialVectorAcquisitionService._page_url(
|
||||
product,
|
||||
settings,
|
||||
scope_wgs84,
|
||||
scope_metric,
|
||||
start_index,
|
||||
)
|
||||
return retained, {
|
||||
"candidate_feature_count": candidate_count,
|
||||
@@ -884,7 +1532,7 @@ class OfficialVectorAcquisitionService:
|
||||
product = OfficialVectorAcquisitionService._product(payload.product_key)
|
||||
scope_wgs84, scope_metric, bbox_values, metric_bounds = (
|
||||
OfficialVectorAcquisitionService._validate_scope(
|
||||
db, project_id, payload, resolved_settings
|
||||
db, project_id, payload, resolved_settings, product
|
||||
)
|
||||
)
|
||||
request_identity = {
|
||||
@@ -906,9 +1554,13 @@ class OfficialVectorAcquisitionService:
|
||||
)
|
||||
area = db.get(Area, payload.area_id) if payload.area_id else None
|
||||
coverage_scope = (
|
||||
"municipality"
|
||||
if area is not None and area.name.strip().lower().startswith("gemeente ")
|
||||
else "bounded_selection"
|
||||
product.coverage_zones[0]
|
||||
if product.requires_coverage_area
|
||||
else (
|
||||
"municipality"
|
||||
if area is not None and area.name.strip().lower().startswith("gemeente ")
|
||||
else "bounded_selection"
|
||||
)
|
||||
)
|
||||
features, transfer = OfficialVectorAcquisitionService._fetch_features(
|
||||
product,
|
||||
@@ -942,6 +1594,7 @@ class OfficialVectorAcquisitionService:
|
||||
"theme": product.theme,
|
||||
"layer_type": product.reference_layer_name,
|
||||
"coverage_scope": coverage_scope,
|
||||
"coverage_zones": list(product.coverage_zones),
|
||||
"geometry_clipped_to_area": payload.area_id is not None,
|
||||
"geometry_clipped_to_selection": True,
|
||||
"bbox_epsg4326": bbox_values,
|
||||
@@ -994,7 +1647,7 @@ class OfficialVectorAcquisitionService:
|
||||
dataset_role="reference",
|
||||
reference_layer_name=product.reference_layer_name,
|
||||
temporal_series_key=f"{product.source_name}:{product.key}:{request_hash[:24]}",
|
||||
observed_at=product.observed_at,
|
||||
observed_at=product.observed_at or acquired_at,
|
||||
valid_from=product.valid_from,
|
||||
valid_to=product.valid_to,
|
||||
temporal_granularity="period" if product.valid_from else "snapshot",
|
||||
|
||||
@@ -45,6 +45,19 @@ PRECLIPPED_MUNICIPALITY_PARTITION_OPERATOR_TOOLS = {
|
||||
|
||||
|
||||
SEMANTIC_SELECTION_METRICS: dict[str, tuple[dict[str, Any], ...]] = {
|
||||
"administrative": (
|
||||
{
|
||||
"metric_key": "covered_area",
|
||||
"method": "intersection_area",
|
||||
"label": "Bestuurlijk ingedeelde oppervlakte",
|
||||
"unit": "ha",
|
||||
"geometry_dimension": 2,
|
||||
"warning": (
|
||||
"Dit is de doorsnede met één bestuurlijk schaalniveau uit de gekozen NGI-laag; "
|
||||
"het is geen kadastrale of juridische grensopmeting."
|
||||
),
|
||||
},
|
||||
),
|
||||
"buildings": (
|
||||
{
|
||||
"metric_key": "footprint_area",
|
||||
@@ -113,9 +126,14 @@ SEMANTIC_SELECTION_METRICS: dict[str, tuple[dict[str, Any], ...]] = {
|
||||
"warning": "Historische bodemkartering op schaal 1:20.000; actuele lokale bodem- en drainagetoestand kan afwijken.",
|
||||
},
|
||||
),
|
||||
# Maritieme plan- en rapportagezones kunnen elkaar overlappen. Een
|
||||
# opgetelde oppervlakte zou daarom geen unieke zeeoppervlakte voorstellen.
|
||||
"maritime_planning": (),
|
||||
"marine_environment": (),
|
||||
}
|
||||
|
||||
SEMANTIC_COUNT_LABELS = {
|
||||
"administrative": "Bestuursgebieden",
|
||||
"buildings": "Gebouwen",
|
||||
"population": "Statistische sectoren",
|
||||
"forest": "Bosvlakken",
|
||||
@@ -125,6 +143,8 @@ SEMANTIC_COUNT_LABELS = {
|
||||
"nature_value": "BWK-kaartvlakken",
|
||||
"agriculture": "Landbouwgebruikspercelen",
|
||||
"soil": "Bodemkaartvlakken",
|
||||
"maritime_planning": "Maritieme planobjecten",
|
||||
"marine_environment": "Mariene rapportagezones",
|
||||
}
|
||||
|
||||
# Sprint 205 initially normalized two official comma-separated ALZ group labels
|
||||
@@ -157,6 +177,12 @@ class VectorFeatureService:
|
||||
source_metadata.get("layer_type"),
|
||||
)
|
||||
aliases = {
|
||||
"belgium_land_boundary": "administrative",
|
||||
"belgium_regions": "administrative",
|
||||
"belgium_provinces": "administrative",
|
||||
"belgium_municipalities": "administrative",
|
||||
"marine_spatial_plan_2026": "maritime_planning",
|
||||
"marine_legal_scopes": "marine_environment",
|
||||
"building": "buildings",
|
||||
"bebouwing": "buildings",
|
||||
"population": "population",
|
||||
|
||||
Reference in New Issue
Block a user