diff --git a/.env.example b/.env.example index 28c76bf6..e910f0a5 100644 --- a/.env.example +++ b/.env.example @@ -28,6 +28,10 @@ GRB_CACHE_TTL_HOURS=24 OFFICIAL_VECTOR_ENABLED=true BWK_WFS_URL=https://geo.api.vlaanderen.be/BWK/wfs DOV_SOIL_WFS_URL=https://www.dov.vlaanderen.be/geoserver/wfs +SPW_PICC_ENABLED=true +SPW_PICC_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer +URBIS_ENABLED=true +URBIS_WFS_URL=https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows OFFICIAL_VECTOR_MIN_SIDE_M=10 OFFICIAL_VECTOR_MAX_SIDE_M=20000 OFFICIAL_VECTOR_PAGE_SIZE=1000 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f0fceb4..9c8885e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,27 @@ # Changelog +## Post-RC Belgium data federation (2026-07-19) + +- Made persisted NGI administrative, RBINS marine reporting and Belgian + marine-plan layers first-class analytical map themes with source-appropriate + selection metrics and truthful workbench readiness. +- Extended the governed Statbel operator from regional scopes to one reviewed + national Belgium edition while retaining plan, stage, named review, + checksum-confirmed apply and Mol baseline gates. +- Added bounded SPW/PICC building, road and hydrography acquisition for + Wallonia and bounded UrbIS building/cadastral acquisition for Brussels. + Regional output remains split by authority and persists only through the + existing DatasetService/vector pipeline. +- Made map-source choice depend on resolved coverage zones rather than the + active technical project, including split handling for cross-region + selections. +- Revalidated MDK bathymetry and kept acquisition fail-closed because the + official hostname still presents a mismatched TLS certificate. No + water-volume or fabricated bathymetry metric was added. +- Added Docker, single-container Unraid and Dockerman template controls for + SPW/PICC and UrbIS. + ## Autonomous Belgium and North Sea RC program (2026-07-17) - Expanded the release-candidate geography from Mol/Kempen to all of Belgium diff --git a/backend/README.md b/backend/README.md index e7ce614b..368e5536 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1854,21 +1854,25 @@ readiness state such as TLS or endpoint failure. Runtime controls are `MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS` and `MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB`. TLS verification cannot be disabled. -## Governed forest, agriculture, nature and soil acquisition +## Governed regional official-vector acquisition The thematic raster registry includes forest and agricultural land-use masks derived from Landgebruik Vlaanderen 2025 classes 12 and 13/14. They use the existing thematic acquisition and selection routes. -Two polygon products are exposed through +Eight fixed products are exposed through `/datasets/official-vector/products` and -`/datasets/official-vector/acquire`: INBO BWK/Natura 2000 2025 and the DOV -digital soil map. Both require an EPSG:4326 rectangle, optionally intersect it -with a persisted Area, clip in EPSG:31370 and persist through -`DatasetService.import_vector_bytes`. +`/datasets/official-vector/acquire`: INBO BWK/Natura 2000 2025 and DOV soil +for Flanders; PICC buildings, roads, hydrographic axes and surfaces for +Wallonia; and UrbIS buildings and cadastral parcels for Brussels. All require +an EPSG:4326 rectangle, clip in a provider-appropriate metric CRS and persist +through `DatasetService.import_vector_bytes`. SPW/PICC and UrbIS additionally +require a persisted exact regional coverage Area and never write directly to +`vector_features`. Runtime controls are `OFFICIAL_VECTOR_ENABLED`, `BWK_WFS_URL`, -`DOV_SOIL_WFS_URL`, `OFFICIAL_VECTOR_MIN_SIDE_M`, +`DOV_SOIL_WFS_URL`, `SPW_PICC_ENABLED`, `SPW_PICC_MAPSERVER_URL`, +`URBIS_ENABLED`, `URBIS_WFS_URL`, `OFFICIAL_VECTOR_MIN_SIDE_M`, `OFFICIAL_VECTOR_MAX_SIDE_M`, `OFFICIAL_VECTOR_PAGE_SIZE`, `OFFICIAL_VECTOR_MAX_PAGES`, `OFFICIAL_VECTOR_MAX_FEATURES`, `OFFICIAL_VECTOR_TIMEOUT_SECONDS`, `OFFICIAL_VECTOR_MAX_RESPONSE_MB`, diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 5ead79c2..33ab2959 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -152,6 +152,19 @@ class Settings(BaseSettings): le=8760, validation_alias="OFFICIAL_VECTOR_CACHE_TTL_HOURS", ) + spw_picc_enabled: bool = Field(default=True, validation_alias="SPW_PICC_ENABLED") + spw_picc_mapserver_url: str = Field( + default=( + "https://geoservices.wallonie.be/arcgis/rest/services/" + "TOPOGRAPHIE/PICC_VDIFF/MapServer" + ), + validation_alias="SPW_PICC_MAPSERVER_URL", + ) + urbis_enabled: bool = Field(default=True, validation_alias="URBIS_ENABLED") + urbis_wfs_url: str = Field( + default="https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows", + validation_alias="URBIS_WFS_URL", + ) dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED") dhmv_wcs_url: str = Field( default="https://geo.api.vlaanderen.be/DHMV/wcs", diff --git a/backend/app/schemas/official_vector.py b/backend/app/schemas/official_vector.py index 48242049..cfe293d6 100644 --- a/backend/app/schemas/official_vector.py +++ b/backend/app/schemas/official_vector.py @@ -32,6 +32,7 @@ class OfficialVectorProductRead(BaseModel): attribution: str license_note: str limitation_message: str + coverage_zones: list[str] class OfficialVectorAcquisitionResult(BaseModel): diff --git a/backend/app/services/coverage_registry_service.py b/backend/app/services/coverage_registry_service.py index 76753aa0..fb463b1f 100644 --- a/backend/app/services/coverage_registry_service.py +++ b/backend/app/services/coverage_registry_service.py @@ -79,6 +79,7 @@ class _SourceDefinition: contract: CoverageSourceContract materialized_layer_names: tuple[str, ...] = () materialized_source_names: tuple[str, ...] = () + operational_themes: tuple[str, ...] = () def _contract( @@ -98,6 +99,7 @@ def _contract( limitation_message: str, materialized_layer_names: tuple[str, ...] = (), materialized_source_names: tuple[str, ...] = (), + operational_themes: tuple[str, ...] = (), ) -> _SourceDefinition: return _SourceDefinition( contract=CoverageSourceContract( @@ -117,6 +119,7 @@ def _contract( ), materialized_layer_names=materialized_layer_names, materialized_source_names=materialized_source_names or (source_name,), + operational_themes=operational_themes, ) @@ -156,12 +159,17 @@ SOURCE_DEFINITIONS = ( themes=("admin", "population"), native_layers=("statistical_sectors", "population_statistics"), geometry_types=("Polygon", "MultiPolygon", "Tabular"), - acquisition_mode="catalog_only", - integration_status="not_configured", + acquisition_mode="operator_archive", + integration_status="operational", source_url="https://statbel.fgov.be/en/open-data", attribution="Statbel", license_note="Consult the license of the selected Statbel release.", - limitation_message="The catalog is audited, but no national bounded acquisition adapter is configured yet.", + limitation_message=( + "National editions require the governed plan-stage-review-apply operator; " + "population in partially selected sectors is area-weighted." + ), + materialized_layer_names=("population",), + operational_themes=("population",), ), _contract( source_name="digitaal_vlaanderen", @@ -219,12 +227,17 @@ SOURCE_DEFINITIONS = ( ), native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"), geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"), - acquisition_mode="catalog_only", - integration_status="not_configured", + acquisition_mode="bounded_api", + integration_status="operational", source_url="https://geoportail.wallonie.be/catalogue", attribution="Service public de Wallonie", license_note="Consult the license of each Geoportail Wallonie product.", - limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.", + limitation_message=( + "Bounded PICC buildings, road axes and hydrography are operational; " + "other Walloon themes remain unavailable until separately governed." + ), + materialized_source_names=("spw_picc",), + operational_themes=("buildings", "roads", "surface_water"), ), _contract( source_name="urbis", @@ -234,12 +247,17 @@ SOURCE_DEFINITIONS = ( themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"), native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"), geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"), - acquisition_mode="catalog_only", - integration_status="not_configured", + acquisition_mode="bounded_api", + integration_status="operational", source_url="https://datastore.brussels", attribution="Brussels UrbIS", license_note="Consult the license of the selected UrbIS dataset.", - limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.", + limitation_message=( + "Bounded UrbIS buildings and cadastral parcels are operational; " + "other Brussels themes remain unavailable until separately governed." + ), + materialized_source_names=("urbis",), + operational_themes=("buildings", "parcels"), ), _contract( source_name="rbins_marine_reporting_units", @@ -330,6 +348,18 @@ FLANDERS_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = { "flood_climate": {"vmm_flood_hazard": ()}, } +REGIONAL_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = { + "spw_geoportail": { + "buildings": ("buildings",), + "roads": ("roads",), + "surface_water": ("water",), + }, + "urbis": { + "buildings": ("buildings",), + "parcels": ("parcels",), + }, +} + class CoverageRegistryService: @staticmethod @@ -403,6 +433,11 @@ class CoverageRegistryService: if dataset.source_name not in theme_sources: continue layer_names = theme_sources[dataset.source_name] + elif definition.contract.source_name in REGIONAL_THEME_DATASETS: + theme_layers = REGIONAL_THEME_DATASETS[definition.contract.source_name].get(theme) + if theme_layers is None: + continue + layer_names = theme_layers metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {} coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or [] if isinstance(coverage_zones, str): @@ -448,10 +483,17 @@ class CoverageRegistryService: materialized.extend(matches) if matches: source_statuses.append("operational") - elif definition.contract.integration_status == "operational": + elif ( + definition.contract.integration_status == "operational" + and (not definition.operational_themes or theme in definition.operational_themes) + ): source_statuses.append("partial") else: - source_statuses.append(definition.contract.integration_status) + source_statuses.append( + "not_configured" + if definition.contract.integration_status == "operational" + else definition.contract.integration_status + ) limitations.append(definition.contract.limitation_message) best_status = max(source_statuses, key=STATUS_RANK.__getitem__) diff --git a/backend/app/services/official_vector_acquisition_service.py b/backend/app/services/official_vector_acquisition_service.py index 3d9cacec..ab7f93c5 100644 --- a/backend/app/services/official_vector_acquisition_service.py +++ b/backend/app/services/official_vector_acquisition_service.py @@ -15,7 +15,7 @@ from uuid import UUID from geoalchemy2.shape import to_shape from pyproj import Transformer -from shapely.geometry import MultiPolygon, Polygon, box, mapping, shape +from shapely.geometry import LineString, MultiLineString, MultiPolygon, Polygon, box, mapping, shape from shapely.ops import transform, unary_union from shapely.validation import make_valid @@ -60,11 +60,17 @@ class OfficialVectorProduct: license_note: str limitation_message: str source: str - observed_at: datetime + observed_at: datetime | None valid_from: datetime | None valid_to: datetime | None primary_metric: dict[str, Any] selection_metrics: tuple[dict[str, Any], ...] + geometry_types: tuple[str, ...] = ("Polygon", "MultiPolygon") + coverage_zones: tuple[str, ...] = ("flanders",) + endpoint_kind: str = "wfs" + response_crs: str = "EPSG:4326" + identity_field: str | None = None + requires_coverage_area: bool = False class OfficialVectorAcquisitionService: @@ -178,6 +184,7 @@ class OfficialVectorAcquisitionService: "warning_only_when_estimate": False, }, ), + endpoint_kind="bwk_wfs", ), OfficialVectorProduct( key="dov_soil_types", @@ -252,6 +259,307 @@ class OfficialVectorAcquisitionService: "filter_values": ["Antropogeen"], }, ), + endpoint_kind="dov_wfs", + ), + OfficialVectorProduct( + key="spw_picc_buildings", + display_name="PICC building footprints", + theme="buildings", + provider="Service public de Wallonie", + source_name="spw_picc", + reference_layer_name="buildings", + service_type="ArcGIS REST", + collection="11", + source_crs="EPSG:3812", + source_version="2026-07-11", + observation_label="Weekly updated PICC snapshot", + authority_level="authoritative", + catalog_url=( + "https://geoportail.wallonie.be/catalogue/" + "b795de68-726c-4bdf-a62a-a42686aa5b6f.html" + ), + attribution="Service public de Wallonie (SPW) - PICC", + license_note="CC BY 4.0; cite SPW PICC and identify modifications.", + limitation_message=( + "Topographic building footprints from PICC; these are not cadastral parcels " + "or legal building registrations." + ), + source="SPW PICC ArcGIS REST", + observed_at=None, + valid_from=None, + valid_to=None, + primary_metric={ + "metric_key": "building_footprint_area", + "method": "intersection_area", + "label": "Bebouwde voetafdruk", + "unit": "ha", + "geometry_dimension": 2, + "is_estimate": False, + }, + selection_metrics=( + { + "metric_key": "building_count", + "method": "feature_count", + "label": "Gebouwen", + "unit": "objecten", + "geometry_dimension": 2, + }, + ), + coverage_zones=("wallonia",), + endpoint_kind="spw_arcgis", + identity_field="GEOREF_ID", + requires_coverage_area=True, + ), + OfficialVectorProduct( + key="spw_picc_roads", + display_name="PICC road axes", + theme="roads", + provider="Service public de Wallonie", + source_name="spw_picc", + reference_layer_name="roads", + service_type="ArcGIS REST", + collection="21", + source_crs="EPSG:3812", + source_version="2026-07-11", + observation_label="Weekly updated PICC snapshot", + authority_level="authoritative", + catalog_url=( + "https://geoportail.wallonie.be/catalogue/" + "b795de68-726c-4bdf-a62a-a42686aa5b6f.html" + ), + attribution="Service public de Wallonie (SPW) - PICC", + license_note="CC BY 4.0; cite SPW PICC and identify modifications.", + limitation_message=( + "PICC road axes describe topographic road geometry and are not a routing " + "network or a traffic measurement." + ), + source="SPW PICC ArcGIS REST", + observed_at=None, + valid_from=None, + valid_to=None, + primary_metric={ + "metric_key": "road_length", + "method": "intersection_length", + "label": "Wegaslengte", + "unit": "km", + "geometry_dimension": 1, + "is_estimate": False, + }, + selection_metrics=( + { + "metric_key": "road_segment_count", + "method": "feature_count", + "label": "Wegsegmenten", + "unit": "objecten", + "geometry_dimension": 1, + }, + ), + geometry_types=("LineString", "MultiLineString"), + coverage_zones=("wallonia",), + endpoint_kind="spw_arcgis", + identity_field="GEOREF_ID", + requires_coverage_area=True, + ), + OfficialVectorProduct( + key="spw_picc_waterways", + display_name="PICC hydrographic axes", + theme="water", + provider="Service public de Wallonie", + source_name="spw_picc", + reference_layer_name="water", + service_type="ArcGIS REST", + collection="28", + source_crs="EPSG:3812", + source_version="2026-07-11", + observation_label="Weekly updated PICC snapshot", + authority_level="authoritative", + catalog_url=( + "https://geoportail.wallonie.be/catalogue/" + "b795de68-726c-4bdf-a62a-a42686aa5b6f.html" + ), + attribution="Service public de Wallonie (SPW) - PICC", + license_note="CC BY 4.0; cite SPW PICC and identify modifications.", + limitation_message=( + "Hydrographic axes describe mapped centre lines. They do not provide depth, " + "discharge or water volume." + ), + source="SPW PICC ArcGIS REST", + observed_at=None, + valid_from=None, + valid_to=None, + primary_metric={ + "metric_key": "waterway_length", + "method": "intersection_length", + "label": "Waterlooplengte", + "unit": "km", + "geometry_dimension": 1, + "is_estimate": False, + }, + selection_metrics=( + { + "metric_key": "waterway_segment_count", + "method": "feature_count", + "label": "Waterloopsegmenten", + "unit": "objecten", + "geometry_dimension": 1, + }, + ), + geometry_types=("LineString", "MultiLineString"), + coverage_zones=("wallonia",), + endpoint_kind="spw_arcgis", + identity_field="GEOREF_ID", + requires_coverage_area=True, + ), + OfficialVectorProduct( + key="spw_picc_water_surfaces", + display_name="PICC hydrographic surfaces", + theme="water", + provider="Service public de Wallonie", + source_name="spw_picc", + reference_layer_name="water", + service_type="ArcGIS REST", + collection="30", + source_crs="EPSG:3812", + source_version="2026-07-11", + observation_label="Weekly updated PICC snapshot", + authority_level="authoritative", + catalog_url=( + "https://geoportail.wallonie.be/catalogue/" + "b795de68-726c-4bdf-a62a-a42686aa5b6f.html" + ), + attribution="Service public de Wallonie (SPW) - PICC", + license_note="CC BY 4.0; cite SPW PICC and identify modifications.", + limitation_message=( + "Mapped hydrographic surface area is not water volume and does not imply " + "a measured water level." + ), + source="SPW PICC ArcGIS REST", + observed_at=None, + valid_from=None, + valid_to=None, + primary_metric={ + "metric_key": "water_surface_area", + "method": "intersection_area", + "label": "Wateroppervlakte", + "unit": "ha", + "geometry_dimension": 2, + "is_estimate": False, + }, + selection_metrics=( + { + "metric_key": "water_surface_count", + "method": "feature_count", + "label": "Wateroppervlakken", + "unit": "objecten", + "geometry_dimension": 2, + }, + ), + coverage_zones=("wallonia",), + endpoint_kind="spw_arcgis", + identity_field="GEOREF_ID", + requires_coverage_area=True, + ), + OfficialVectorProduct( + key="urbis_buildings", + display_name="UrbIS buildings", + theme="buildings", + provider="Paradigm Brussels", + source_name="urbis", + reference_layer_name="buildings", + service_type="WFS 2.0", + collection="urbisvector:Buildings", + source_crs="EPSG:31370", + source_version="2026-06-06", + observation_label="UrbIS revision 6 June 2026", + authority_level="authoritative", + catalog_url=( + "https://datastore.brussels/web/data/dataset/" + "2cf42541-1813-11ef-8a81-00090ffe0001" + ), + attribution="Paradigm Brussels - UrbIS", + license_note="Buildings are published under CC0.", + limitation_message=( + "UrbIS building geometry is a regional topographic reference and is not " + "a legal cadastral registration." + ), + source="UrbIS WFS", + observed_at=datetime(2026, 6, 6, tzinfo=UTC), + valid_from=None, + valid_to=None, + primary_metric={ + "metric_key": "building_footprint_area", + "method": "intersection_area", + "label": "Bebouwde voetafdruk", + "unit": "ha", + "geometry_dimension": 2, + "is_estimate": False, + }, + selection_metrics=( + { + "metric_key": "building_count", + "method": "feature_count", + "label": "Gebouwen", + "unit": "objecten", + "geometry_dimension": 2, + }, + ), + coverage_zones=("brussels",), + endpoint_kind="urbis_wfs", + response_crs="EPSG:31370", + identity_field="INSPIRE_ID", + requires_coverage_area=True, + ), + OfficialVectorProduct( + key="urbis_cadastral_parcels", + display_name="UrbIS cadastral parcels", + theme="parcels", + provider="Paradigm Brussels / FPS Finance", + source_name="urbis", + reference_layer_name="parcels", + service_type="WFS 2.0", + collection="urbisvector:CadastralParcels", + source_crs="EPSG:31370", + source_version="2026-06-06", + observation_label="UrbIS revision 6 June 2026", + authority_level="authoritative", + catalog_url=( + "https://datastore.brussels/web/data/dataset/" + "2cf42541-1813-11ef-8a81-00090ffe0001" + ), + attribution="Paradigm Brussels and FPS Finance - cadastral parcel plan", + license_note=( + "The FPS Finance open-data cadastral plan licence applies to cadastral parcels." + ), + limitation_message=( + "Cadastral parcel geometry is reference data. GeoIntel does not infer " + "ownership, rights or legal boundaries beyond the published source." + ), + source="UrbIS WFS", + observed_at=datetime(2026, 1, 1, tzinfo=UTC), + valid_from=None, + valid_to=None, + primary_metric={ + "metric_key": "parcel_area", + "method": "intersection_area", + "label": "Perceeloppervlakte", + "unit": "ha", + "geometry_dimension": 2, + "is_estimate": False, + }, + selection_metrics=( + { + "metric_key": "parcel_count", + "method": "feature_count", + "label": "Percelen", + "unit": "objecten", + "geometry_dimension": 2, + }, + ), + coverage_zones=("brussels",), + endpoint_kind="urbis_wfs", + response_crs="EPSG:31370", + identity_field="INSPIRE_ID", + requires_coverage_area=True, ), ) return {product.key: product for product in products} @@ -268,7 +576,7 @@ class OfficialVectorAcquisitionService: reference_layer_name=product.reference_layer_name, service_type=product.service_type, collection=product.collection, - geometry_types=["Polygon", "MultiPolygon"], + geometry_types=list(product.geometry_types), source_crs=product.source_crs, source_version=product.source_version, observation_label=product.observation_label, @@ -277,6 +585,7 @@ class OfficialVectorAcquisitionService: attribution=product.attribution, license_note=product.license_note, limitation_message=product.limitation_message, + coverage_zones=list(product.coverage_zones), ).model_dump() for product in OfficialVectorAcquisitionService._products().values() ] @@ -320,12 +629,44 @@ class OfficialVectorAcquisitionService: result = make_valid(result) return result if not result.is_empty and result.is_valid else None + @staticmethod + def _dimensional(geometry: Any, dimension: int) -> Any | None: + if geometry is None or geometry.is_empty: + return None + if not geometry.is_valid: + geometry = make_valid(geometry) + parts: list[Any] = [] + + def collect(item: Any) -> None: + if item is None or item.is_empty: + return + if dimension == 2 and isinstance(item, Polygon): + parts.append(item) + elif dimension == 2 and isinstance(item, MultiPolygon): + parts.extend(part for part in item.geoms if not part.is_empty) + elif dimension == 1 and isinstance(item, LineString): + parts.append(item) + elif dimension == 1 and isinstance(item, MultiLineString): + parts.extend(part for part in item.geoms if not part.is_empty) + elif hasattr(item, "geoms"): + for part in item.geoms: + collect(part) + + collect(geometry) + if not parts: + return None + result = unary_union(parts) + if not result.is_valid: + result = make_valid(result) + return result if not result.is_empty and result.is_valid else None + @staticmethod def _validate_scope( db, project_id: UUID, payload: OfficialVectorAcquireRequest, settings: Settings, + product: OfficialVectorProduct, ) -> tuple[Any, Any, list[float], list[float]]: if not settings.official_vector_enabled: raise AppError( @@ -333,6 +674,18 @@ class OfficialVectorAcquisitionService: message="Bounded official vector acquisition is disabled", status_code=503, ) + if product.endpoint_kind == "spw_arcgis" and not settings.spw_picc_enabled: + raise AppError( + code="SPW_PICC_NOT_CONFIGURED", + message="Bounded SPW PICC acquisition is disabled", + status_code=503, + ) + if product.endpoint_kind == "urbis_wfs" and not settings.urbis_enabled: + raise AppError( + code="URBIS_NOT_CONFIGURED", + message="Bounded UrbIS acquisition is disabled", + status_code=503, + ) if not db.get(Project, project_id): raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) if payload.bbox.crs.upper() != "EPSG:4326": @@ -375,6 +728,7 @@ class OfficialVectorAcquisitionService: status_code=422, ) scope_wgs84 = box(*values) + area = None if payload.area_id: area = db.get(Area, payload.area_id) if area is None: @@ -394,6 +748,53 @@ class OfficialVectorAcquisitionService: message="The selection does not intersect the selected area", status_code=400, ) + if product.requires_coverage_area: + coverage_area_names = { + "wallonia": "Wallonia", + "brussels": "Brussels-Capital Region", + } + required_names = [ + coverage_area_names[zone] + for zone in product.coverage_zones + if zone in coverage_area_names + ] + coverage_rows = ( + [area] + if area is not None and area.name in required_names + else db.query(Area) + .filter( + Area.project_id == project_id, + Area.name.in_(required_names), + ) + .all() + ) + coverage_geometries = [ + to_shape(item.geometry) + for item in coverage_rows + if item is not None and item.geometry is not None + ] + coverage_geometry = ( + OfficialVectorAcquisitionService._polygonal(unary_union(coverage_geometries)) + if coverage_geometries + else None + ) + if coverage_geometry is None: + raise AppError( + code="OFFICIAL_VECTOR_COVERAGE_NOT_READY", + message="The official regional coverage boundary is not persisted in this project", + details={"required_areas": required_names}, + status_code=409, + ) + scope_wgs84 = OfficialVectorAcquisitionService._polygonal( + scope_wgs84.intersection(coverage_geometry) + ) + if scope_wgs84 is None: + raise AppError( + code="OFFICIAL_VECTOR_OUTSIDE_COVERAGE", + message="The selection does not intersect the official product coverage", + details={"coverage_zones": list(product.coverage_zones)}, + status_code=422, + ) scope_metric = OfficialVectorAcquisitionService._polygonal( transform(_TO_LAMBERT72.transform, scope_wgs84) ) @@ -425,6 +826,21 @@ class OfficialVectorAcquisitionService: request, timeout=settings.official_vector_timeout_seconds, ) as response: + content_type = "" + if hasattr(response, "getheader"): + content_type = str(response.getheader("Content-Type") or "") + elif hasattr(response, "headers"): + content_type = str(response.headers.get("Content-Type") or "") + if content_type and not any( + allowed in content_type.lower() + for allowed in ("application/json", "application/geo+json") + ): + raise AppError( + code="OFFICIAL_VECTOR_PROVIDER_INVALID_CONTENT_TYPE", + message="The official vector provider returned an unsupported content type", + details={"content_type": content_type}, + status_code=502, + ) limit = settings.official_vector_max_response_mb * 1024 * 1024 content = response.read(limit + 1) except HTTPError as exc: @@ -502,6 +918,137 @@ class OfficialVectorAcquisitionService: ) return f"{settings.dov_soil_wfs_url.rstrip('?')}?{query}" + @staticmethod + def _spw_url( + settings: Settings, + product: OfficialVectorProduct, + bbox_values: tuple[float, ...], + start_index: int, + ) -> str: + query = urlencode( + { + "where": "1=1", + "geometry": ",".join(f"{value:.8f}" for value in bbox_values), + "geometryType": "esriGeometryEnvelope", + "inSR": "4326", + "outSR": "4326", + "spatialRel": "esriSpatialRelIntersects", + "outFields": "*", + "returnGeometry": "true", + "returnZ": "false", + "returnM": "false", + "resultOffset": start_index, + "resultRecordCount": min(settings.official_vector_page_size, 2000), + "orderByFields": "OBJECTID", + "f": "geojson", + } + ) + base = settings.spw_picc_mapserver_url.rstrip("/") + return f"{base}/{product.collection}/query?{query}" + + @staticmethod + def _urbis_url( + settings: Settings, + product: OfficialVectorProduct, + metric_bbox: tuple[float, ...], + start_index: int, + ) -> str: + query = urlencode( + { + "service": "WFS", + "version": "2.0.0", + "request": "GetFeature", + "typeNames": product.collection, + "srsName": "EPSG:31370", + "bbox": ",".join(f"{value:.3f}" for value in metric_bbox) + ",EPSG:31370", + "count": settings.official_vector_page_size, + "startIndex": start_index, + "sortBy": product.identity_field or "INSPIRE_ID", + "outputFormat": "application/json", + } + ) + return f"{settings.urbis_wfs_url.rstrip('?')}?{query}" + + @staticmethod + def _page_url( + product: OfficialVectorProduct, + settings: Settings, + scope_wgs84: Any, + scope_metric: Any, + start_index: int, + ) -> str: + if product.endpoint_kind == "bwk_wfs": + return OfficialVectorAcquisitionService._nature_url( + settings, + tuple(scope_wgs84.bounds), + start_index, + ) + if product.endpoint_kind == "dov_wfs": + return OfficialVectorAcquisitionService._soil_url( + settings, + tuple(scope_metric.bounds), + start_index, + ) + if product.endpoint_kind == "spw_arcgis": + return OfficialVectorAcquisitionService._spw_url( + settings, + product, + tuple(scope_wgs84.bounds), + start_index, + ) + if product.endpoint_kind == "urbis_wfs": + return OfficialVectorAcquisitionService._urbis_url( + settings, + product, + tuple(scope_metric.bounds), + start_index, + ) + raise AppError( + code="OFFICIAL_VECTOR_PRODUCT_NOT_SUPPORTED", + message="The official vector product has no governed acquisition adapter", + status_code=422, + ) + + @staticmethod + def _validate_page_url( + url: str, + product: OfficialVectorProduct, + settings: Settings, + ) -> None: + parsed = urlparse(url) + configured_url = { + "bwk_wfs": settings.bwk_wfs_url, + "dov_wfs": settings.dov_soil_wfs_url, + "spw_arcgis": settings.spw_picc_mapserver_url, + "urbis_wfs": settings.urbis_wfs_url, + }.get(product.endpoint_kind) + if configured_url is None: + raise AppError( + code="OFFICIAL_VECTOR_PRODUCT_NOT_SUPPORTED", + message="The official vector product has no configured endpoint", + status_code=422, + ) + base = urlparse(configured_url) + expected_path = ( + f"{base.path.rstrip('/')}/{product.collection}/query" + if product.endpoint_kind == "spw_arcgis" + else base.path + ) + if ( + parsed.scheme != "https" + or base.scheme != "https" + or parsed.netloc.casefold() != base.netloc.casefold() + or parsed.path != expected_path + or parsed.username + or parsed.password + or parsed.fragment + ): + raise AppError( + code="OFFICIAL_VECTOR_PROVIDER_INVALID_PAGINATION", + message="The official vector request escaped the governed HTTPS endpoint", + status_code=502, + ) + @staticmethod def _habitat_breakdown(properties: dict[str, Any]) -> tuple[list[dict[str, Any]], float, float, float]: entries: list[dict[str, Any]] = [] @@ -527,6 +1074,92 @@ class OfficialVectorAcquisitionService: uncertain_share = 100.0 return entries, min(100.0, natura_share), min(100.0, regional_share), min(100.0, uncertain_share) + @staticmethod + def _normalize_regional_feature( + product: OfficialVectorProduct, + feature: dict[str, Any], + scope_metric: Any, + coverage_scope: str, + ) -> dict[str, Any] | None: + dimension = 2 if any("Polygon" in item for item in product.geometry_types) else 1 + try: + source_geometry = shape(feature.get("geometry")) + except Exception as exc: + raise AppError( + code="OFFICIAL_VECTOR_PROVIDER_INVALID_GEOMETRY", + message=f"{product.display_name} returned invalid geometry", + status_code=502, + ) from exc + if product.response_crs == "EPSG:31370": + source_metric = OfficialVectorAcquisitionService._dimensional( + source_geometry, + dimension, + ) + else: + source_wgs84 = OfficialVectorAcquisitionService._dimensional( + source_geometry, + dimension, + ) + source_metric = ( + OfficialVectorAcquisitionService._dimensional( + transform(_TO_LAMBERT72.transform, source_wgs84), + dimension, + ) + if source_wgs84 is not None + else None + ) + if source_metric is None or not source_metric.intersects(scope_metric): + return None + clipped_metric = OfficialVectorAcquisitionService._dimensional( + source_metric.intersection(scope_metric), + dimension, + ) + if clipped_metric is None: + return None + clipped_wgs84 = OfficialVectorAcquisitionService._dimensional( + transform(_TO_WGS84.transform, clipped_metric), + dimension, + ) + if clipped_wgs84 is None: + return None + raw = dict(feature.get("properties") or {}) + identity = ( + raw.get(product.identity_field or "") + or feature.get("id") + or raw.get("OBJECTID") + ) + if identity in (None, ""): + raise AppError( + code="OFFICIAL_VECTOR_PROVIDER_INVALID_RESPONSE", + message=f"{product.display_name} returned a feature without an official identity", + status_code=502, + ) + feature_id = f"{product.collection}:{identity}" + properties = { + **raw, + "source_name": product.source_name, + "source_collection": product.collection, + "source_feature_id": feature_id, + "reference_layer_name": product.reference_layer_name, + "theme": product.theme, + "authority_level": product.authority_level, + "coverage_scope": coverage_scope, + "coverage_zones": list(product.coverage_zones), + "source_version": product.source_version, + "attribution": product.attribution, + "geometry_clipped_to_selection": not scope_metric.covers(source_metric), + } + if dimension == 2: + properties["clipped_area_ha"] = round(float(clipped_metric.area) / 10_000.0, 8) + else: + properties["clipped_length_km"] = round(float(clipped_metric.length) / 1_000.0, 8) + return { + "type": "Feature", + "id": feature_id, + "geometry": mapping(clipped_wgs84), + "properties": properties, + } + @staticmethod def _normalize_feature( product: OfficialVectorProduct, @@ -534,6 +1167,13 @@ class OfficialVectorAcquisitionService: scope_metric: Any, coverage_scope: str, ) -> dict[str, Any] | None: + if product.endpoint_kind in {"spw_arcgis", "urbis_wfs"}: + return OfficialVectorAcquisitionService._normalize_regional_feature( + product, + feature, + scope_metric, + coverage_scope, + ) try: source_wgs84 = OfficialVectorAcquisitionService._polygonal(shape(feature.get("geometry"))) except Exception as exc: @@ -667,32 +1307,21 @@ class OfficialVectorAcquisitionService: response_hashes: list[str] = [] total_bytes = candidate_count = 0 expected_total: int | None = None - next_url = ( - OfficialVectorAcquisitionService._nature_url(settings, tuple(scope_wgs84.bounds), 0) - if product.theme == "nature_value" - else OfficialVectorAcquisitionService._soil_url(settings, tuple(scope_metric.bounds), 0) + next_url = OfficialVectorAcquisitionService._page_url( + product, + settings, + scope_wgs84, + scope_metric, + 0, ) start_index = 0 seen_pages: set[str] = set() while next_url: - parsed = urlparse(next_url) - configured_url = ( - settings.bwk_wfs_url - if product.theme == "nature_value" - else settings.dov_soil_wfs_url + OfficialVectorAcquisitionService._validate_page_url( + next_url, + product, + settings, ) - base = urlparse(configured_url) - if ( - parsed.scheme != "https" - or base.scheme != "https" - or parsed.netloc.casefold() != base.netloc.casefold() - or parsed.path != base.path - ): - raise AppError( - code="OFFICIAL_VECTOR_PROVIDER_INVALID_PAGINATION", - message="The official WFS request escaped the governed endpoint", - status_code=502, - ) if next_url in seen_pages: raise AppError( code="OFFICIAL_VECTOR_PROVIDER_PAGINATION_LOOP", @@ -777,7 +1406,26 @@ class OfficialVectorAcquisitionService: status_code=502, ) start_index += returned_count - if returned_count == 0 or ( + arcgis_has_more = payload.get("exceededTransferLimit") is True + if product.endpoint_kind == "spw_arcgis": + if arcgis_has_more and returned_count == 0: + raise AppError( + code="OFFICIAL_VECTOR_PROVIDER_INCOMPLETE_RESPONSE", + message="The SPW provider reported more records but returned an empty page", + status_code=502, + ) + next_url = ( + OfficialVectorAcquisitionService._page_url( + product, + settings, + scope_wgs84, + scope_metric, + start_index, + ) + if arcgis_has_more + else None + ) + elif returned_count == 0 or ( expected_total is not None and start_index >= expected_total ) or (expected_total is None and returned_count < settings.official_vector_page_size): if expected_total is not None and start_index != expected_total: @@ -788,13 +1436,13 @@ class OfficialVectorAcquisitionService: status_code=502, ) next_url = None - elif product.theme == "nature_value": - next_url = OfficialVectorAcquisitionService._nature_url( - settings, tuple(scope_wgs84.bounds), start_index - ) else: - next_url = OfficialVectorAcquisitionService._soil_url( - settings, tuple(scope_metric.bounds), start_index + next_url = OfficialVectorAcquisitionService._page_url( + product, + settings, + scope_wgs84, + scope_metric, + start_index, ) return retained, { "candidate_feature_count": candidate_count, @@ -884,7 +1532,7 @@ class OfficialVectorAcquisitionService: product = OfficialVectorAcquisitionService._product(payload.product_key) scope_wgs84, scope_metric, bbox_values, metric_bounds = ( OfficialVectorAcquisitionService._validate_scope( - db, project_id, payload, resolved_settings + db, project_id, payload, resolved_settings, product ) ) request_identity = { @@ -906,9 +1554,13 @@ class OfficialVectorAcquisitionService: ) area = db.get(Area, payload.area_id) if payload.area_id else None coverage_scope = ( - "municipality" - if area is not None and area.name.strip().lower().startswith("gemeente ") - else "bounded_selection" + product.coverage_zones[0] + if product.requires_coverage_area + else ( + "municipality" + if area is not None and area.name.strip().lower().startswith("gemeente ") + else "bounded_selection" + ) ) features, transfer = OfficialVectorAcquisitionService._fetch_features( product, @@ -942,6 +1594,7 @@ class OfficialVectorAcquisitionService: "theme": product.theme, "layer_type": product.reference_layer_name, "coverage_scope": coverage_scope, + "coverage_zones": list(product.coverage_zones), "geometry_clipped_to_area": payload.area_id is not None, "geometry_clipped_to_selection": True, "bbox_epsg4326": bbox_values, @@ -994,7 +1647,7 @@ class OfficialVectorAcquisitionService: dataset_role="reference", reference_layer_name=product.reference_layer_name, temporal_series_key=f"{product.source_name}:{product.key}:{request_hash[:24]}", - observed_at=product.observed_at, + observed_at=product.observed_at or acquired_at, valid_from=product.valid_from, valid_to=product.valid_to, temporal_granularity="period" if product.valid_from else "snapshot", diff --git a/backend/app/services/vector_feature_service.py b/backend/app/services/vector_feature_service.py index dc280e50..dc6a5ebf 100644 --- a/backend/app/services/vector_feature_service.py +++ b/backend/app/services/vector_feature_service.py @@ -45,6 +45,19 @@ PRECLIPPED_MUNICIPALITY_PARTITION_OPERATOR_TOOLS = { SEMANTIC_SELECTION_METRICS: dict[str, tuple[dict[str, Any], ...]] = { + "administrative": ( + { + "metric_key": "covered_area", + "method": "intersection_area", + "label": "Bestuurlijk ingedeelde oppervlakte", + "unit": "ha", + "geometry_dimension": 2, + "warning": ( + "Dit is de doorsnede met één bestuurlijk schaalniveau uit de gekozen NGI-laag; " + "het is geen kadastrale of juridische grensopmeting." + ), + }, + ), "buildings": ( { "metric_key": "footprint_area", @@ -113,9 +126,14 @@ SEMANTIC_SELECTION_METRICS: dict[str, tuple[dict[str, Any], ...]] = { "warning": "Historische bodemkartering op schaal 1:20.000; actuele lokale bodem- en drainagetoestand kan afwijken.", }, ), + # Maritieme plan- en rapportagezones kunnen elkaar overlappen. Een + # opgetelde oppervlakte zou daarom geen unieke zeeoppervlakte voorstellen. + "maritime_planning": (), + "marine_environment": (), } SEMANTIC_COUNT_LABELS = { + "administrative": "Bestuursgebieden", "buildings": "Gebouwen", "population": "Statistische sectoren", "forest": "Bosvlakken", @@ -125,6 +143,8 @@ SEMANTIC_COUNT_LABELS = { "nature_value": "BWK-kaartvlakken", "agriculture": "Landbouwgebruikspercelen", "soil": "Bodemkaartvlakken", + "maritime_planning": "Maritieme planobjecten", + "marine_environment": "Mariene rapportagezones", } # Sprint 205 initially normalized two official comma-separated ALZ group labels @@ -157,6 +177,12 @@ class VectorFeatureService: source_metadata.get("layer_type"), ) aliases = { + "belgium_land_boundary": "administrative", + "belgium_regions": "administrative", + "belgium_provinces": "administrative", + "belgium_municipalities": "administrative", + "marine_spatial_plan_2026": "maritime_planning", + "marine_legal_scopes": "marine_environment", "building": "buildings", "bebouwing": "buildings", "population": "population", diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py index de84ccd6..a827950c 100644 --- a/backend/tests/test_docker_runtime_config.py +++ b/backend/tests/test_docker_runtime_config.py @@ -250,6 +250,33 @@ def test_runtime_sets_writable_ultralytics_config_directory() -> None: assert "YOLO_CONFIG_DIR=/app/storage/ultralytics" in unraid_env +def test_regional_official_vector_sources_are_configurable_in_every_runtime() -> None: + compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8") + unraid_compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8") + run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text( + encoding="utf-8" + ) + env_example = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text( + encoding="utf-8" + ) + template = (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text( + encoding="utf-8" + ) + + for key in ( + "SPW_PICC_ENABLED", + "SPW_PICC_MAPSERVER_URL", + "URBIS_ENABLED", + "URBIS_WFS_URL", + ): + assert key in compose + assert key in unraid_compose + assert f'{key}="${{{key}:-' in run_script + assert f'-e {key}="${key}"' in run_script + assert f"{key}=" in env_example + assert f'Target="{key}"' in template + + def test_docker_build_contexts_exclude_vendor_build_and_cache_outputs() -> None: required_patterns = { "node_modules", diff --git a/backend/tests/test_post_rc_regional_official_vector.py b/backend/tests/test_post_rc_regional_official_vector.py new file mode 100644 index 00000000..c1fdd9c9 --- /dev/null +++ b/backend/tests/test_post_rc_regional_official_vector.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace +from urllib.parse import parse_qs, urlparse +from uuid import uuid4 + +import pytest +from geoalchemy2.shape import from_shape +from pyproj import Transformer +from shapely.geometry import MultiPolygon, Polygon + +from app.core.config import Settings +from app.core.errors import AppError +from app.models import Area, Dataset, Project +from app.schemas.official_vector import OfficialVectorAcquireRequest +from app.services.dataset_service import DatasetService +from app.services.official_vector_acquisition_service import ( + OfficialVectorAcquisitionService, + _TO_LAMBERT72, +) + + +class FakeQuery: + def __init__(self, result=None): + self.result = result + + def filter(self, *_args): + return self + + def order_by(self, *_args): + return self + + def all(self): + return self.result if isinstance(self.result, list) else [] + + +class FakeSession: + def __init__(self, rows=None, query_result=None): + self.rows = rows or {} + self.query_result = query_result + + def get(self, model, row_id): + return self.rows.get((model, row_id)) + + def query(self, _model): + return FakeQuery(self.query_result) + + +class JsonResponse: + def __init__(self, payload, content_type="application/geo+json"): + self.content = json.dumps(payload).encode("utf-8") + self.content_type = content_type + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, size=-1): + return self.content if size < 0 else self.content[:size] + + def getheader(self, name): + return self.content_type if name.lower() == "content-type" else None + + +def request(product_key: str, bbox: tuple[float, float, float, float], area_id=None): + return OfficialVectorAcquireRequest( + bbox={ + "min_x": bbox[0], + "min_y": bbox[1], + "max_x": bbox[2], + "max_y": bbox[3], + "crs": "EPSG:4326", + }, + area_id=area_id, + product_key=product_key, + force_refresh=True, + ) + + +def area(project_id, name: str, bounds: tuple[float, float, float, float]): + min_x, min_y, max_x, max_y = bounds + geometry = MultiPolygon( + [ + Polygon( + [ + (min_x, min_y), + (max_x, min_y), + (max_x, max_y), + (min_x, max_y), + (min_x, min_y), + ] + ) + ] + ) + return Area( + id=uuid4(), + project_id=project_id, + name=name, + geometry=from_shape(geometry, srid=4326), + ) + + +def test_regional_product_registry_is_explicit_and_source_specific() -> None: + products = { + item["key"]: item + for item in OfficialVectorAcquisitionService.list_products() + } + + assert products["spw_picc_buildings"]["coverage_zones"] == ["wallonia"] + assert products["spw_picc_roads"]["geometry_types"] == [ + "LineString", + "MultiLineString", + ] + assert products["spw_picc_waterways"]["collection"] == "28" + assert products["spw_picc_water_surfaces"]["collection"] == "30" + assert products["urbis_buildings"]["coverage_zones"] == ["brussels"] + assert products["urbis_buildings"]["license_note"] == "Buildings are published under CC0." + assert "FPS Finance" in products["urbis_cadastral_parcels"]["license_note"] + + +def test_spw_arcgis_paging_is_bounded_stable_and_clipped() -> None: + product = OfficialVectorAcquisitionService._product("spw_picc_buildings") + scope = Polygon( + [(4.55, 50.58), (4.56, 50.58), (4.56, 50.59), (4.55, 50.59), (4.55, 50.58)] + ) + scope_metric = Polygon( + [ + _TO_LAMBERT72.transform(x, y) + for x, y in scope.exterior.coords + ] + ) + offsets = [] + + def feature(object_id: int, min_x: float): + return { + "type": "Feature", + "id": object_id, + "geometry": { + "type": "Polygon", + "coordinates": [[ + [min_x, 50.581], + [min_x + 0.002, 50.581], + [min_x + 0.002, 50.583], + [min_x, 50.583], + [min_x, 50.581], + ]], + }, + "properties": {"OBJECTID": object_id, "GEOREF_ID": f"wallonia-{object_id}"}, + } + + def opener(raw_request, timeout): + assert timeout == 180 + query = parse_qs(urlparse(raw_request.full_url).query) + assert query["orderByFields"] == ["OBJECTID"] + assert query["f"] == ["geojson"] + offset = int(query["resultOffset"][0]) + offsets.append(offset) + return JsonResponse( + { + "type": "FeatureCollection", + "features": [feature(offset + 1, 4.551 + offset * 0.0001)], + "exceededTransferLimit": offset == 0, + } + ) + + features, transfer = OfficialVectorAcquisitionService._fetch_features( + product, + scope, + scope_metric, + "wallonia", + Settings(_env_file=None, OFFICIAL_VECTOR_PAGE_SIZE=1), + opener, + ) + + assert offsets == [0, 1] + assert transfer["page_count"] == 2 + assert transfer["reference_truncated"] is False + assert {item["properties"]["source_feature_id"] for item in features} == { + "11:wallonia-1", + "11:wallonia-2", + } + assert all(item["properties"]["coverage_scope"] == "wallonia" for item in features) + assert all(item["properties"]["clipped_area_ha"] > 0 for item in features) + + +def test_regional_products_require_the_persisted_authoritative_coverage_area() -> None: + project_id = uuid4() + db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")}) + + with pytest.raises(AppError) as exc_info: + OfficialVectorAcquisitionService.acquire( + db, + project_id, + request("spw_picc_buildings", (4.55, 50.58, 4.56, 50.59)), + settings=Settings(_env_file=None), + ) + + assert exc_info.value.code == "OFFICIAL_VECTOR_COVERAGE_NOT_READY" + + +def test_urbis_wfs_transforms_lambert72_and_persists_through_dataset_service( + monkeypatch, +) -> None: + project_id, dataset_id = uuid4(), uuid4() + brussels = area(project_id, "Brussels-Capital Region", (4.25, 50.75, 4.5, 50.95)) + db = FakeSession( + { + (Project, project_id): Project(id=project_id, name="Belgium"), + (Area, brussels.id): brussels, + }, + query_result=[brussels], + ) + to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) + min_x, min_y = to_lambert.transform(4.35, 50.84) + max_x, max_y = to_lambert.transform(4.351, 50.841) + captured = {} + + def opener(raw_request, timeout): + assert timeout == 180 + query = parse_qs(urlparse(raw_request.full_url).query) + assert query["typeNames"] == ["urbisvector:Buildings"] + assert query["srsName"] == ["EPSG:31370"] + assert query["sortBy"] == ["INSPIRE_ID"] + return JsonResponse( + { + "type": "FeatureCollection", + "numberMatched": 1, + "numberReturned": 1, + "features": [ + { + "type": "Feature", + "id": "Buildings.1", + "geometry": { + "type": "MultiPolygon", + "coordinates": [[[ + [min_x, min_y], + [max_x, min_y], + [max_x, max_y], + [min_x, max_y], + [min_x, min_y], + ]]], + }, + "properties": { + "INSPIRE_ID": "https://databrussels.be/id/building/1", + "AREA": 75, + }, + } + ], + }, + "application/json", + ) + + def persist(_db, **kwargs): + captured.update(kwargs) + dataset = Dataset( + id=dataset_id, + project_id=project_id, + area_id=brussels.id, + name=kwargs["filename"], + dataset_type="vector", + source=kwargs["source"], + dataset_role=kwargs["dataset_role"], + source_name=kwargs["source_name"], + reference_layer_name=kwargs["reference_layer_name"], + temporal_series_key=kwargs["temporal_series_key"], + observed_at=kwargs["observed_at"], + source_version=kwargs["source_version"], + source_metadata=kwargs["source_metadata"], + provenance_metadata=kwargs["provenance_metadata"], + metadata_json={"feature_count": 1}, + status="ready", + ) + db.rows[(Dataset, dataset_id)] = dataset + return SimpleNamespace(id=dataset_id) + + monkeypatch.setattr(DatasetService, "import_vector_bytes", persist) + result = OfficialVectorAcquisitionService.acquire( + db, + project_id, + request( + "urbis_buildings", + (4.349, 50.839, 4.352, 50.842), + area_id=brussels.id, + ), + settings=Settings(_env_file=None), + opener=opener, + ) + + assert result["output_dataset_id"] == str(dataset_id) + assert captured["source_name"] == "urbis" + assert captured["reference_layer_name"] == "buildings" + assert captured["source_metadata"]["coverage_zones"] == ["brussels"] + assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == ( + "building_footprint_area" + ) + collection = json.loads(captured["content"]) + geometry = collection["features"][0]["geometry"] + assert geometry["type"] in {"Polygon", "MultiPolygon"} + first_coordinate = ( + geometry["coordinates"][0][0][0] + if geometry["type"] == "MultiPolygon" + else geometry["coordinates"][0][0] + ) + assert 4.34999 <= first_coordinate[0] <= 4.35101 + assert 50.83999 <= first_coordinate[1] <= 50.84101 diff --git a/backend/tests/test_rc4_national_coverage.py b/backend/tests/test_rc4_national_coverage.py index 3d91fc19..69f67127 100644 --- a/backend/tests/test_rc4_national_coverage.py +++ b/backend/tests/test_rc4_national_coverage.py @@ -12,6 +12,7 @@ from app.main import app from app.models import Area, Dataset, Project from app.schemas.coverage import CoverageBBox from app.services.coverage_registry_service import CoverageRegistryService, THEMES, ZONES +from app.services.vector_feature_service import VectorFeatureService class FakeQuery: @@ -72,6 +73,44 @@ def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider assert response.json()["data"]["themes"] == list(THEMES) +def test_national_and_maritime_reference_layers_are_selection_analyzable() -> None: + cases = ( + ("ngi_adminvector", "belgium_municipalities", "administrative"), + ("rbins_marine_reporting_units", "marine_legal_scopes", "marine_environment"), + ("rbins_msp_2026", "marine_spatial_plan_2026", "maritime_planning"), + ) + for source_name, layer_name, expected_theme in cases: + dataset = Dataset( + id=uuid4(), + project_id=uuid4(), + name=f"{layer_name}.geojson", + dataset_type="vector", + source="operator_official_import", + source_name=source_name, + reference_layer_name=layer_name, + source_metadata={"authority_level": "authoritative"}, + status="ready", + ) + + assert VectorFeatureService._dataset_theme(dataset) == expected_theme + assert VectorFeatureService.supports_selection_summary(dataset) is True + + +def test_national_scope_operator_assigns_explicit_map_themes() -> None: + root = Path(__file__).resolve().parents[2] + operator = (root / "scripts" / "provision_belgium_north_sea_scope.py").read_text(encoding="utf-8") + map_workspace = (root / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text( + encoding="utf-8" + ) + + assert '"belgium_municipalities": "administrative"' in operator + assert '"marine_legal_scopes": "marine_environment"' in operator + assert '"marine_spatial_plan_2026": "maritime_planning"' in operator + assert "id: 'administrative'" in map_workspace + assert "id: 'maritime_planning'" in map_workspace + assert "id: 'marine_environment'" in map_workspace + + def test_coverage_resolver_only_reports_operational_for_materialized_ready_dataset() -> None: project_id = uuid4() project = SimpleNamespace(id=project_id) diff --git a/backend/tests/test_sprint194_regional_timeseries.py b/backend/tests/test_sprint194_regional_timeseries.py index 2c1b1fb7..9f835568 100644 --- a/backend/tests/test_sprint194_regional_timeseries.py +++ b/backend/tests/test_sprint194_regional_timeseries.py @@ -55,16 +55,22 @@ def test_population_operator_filters_to_the_approved_scope() -> None: module = load_script("provision_mol_population_history.py") regional = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] mol = module.GEOGRAPHIC_SCOPES["mol"] + belgium = module.GEOGRAPHIC_SCOPES["belgium"] regional_rows = module.population_rows(population_archive(), regional) mol_rows = module.population_rows(population_archive(), mol) + national_rows = module.population_rows(population_archive(), belgium) assert set(regional_rows) == {"13025A00-", "13008A00-"} assert regional_rows["13008A00-"]["municipality"] == "Geel" assert regional_rows["13008A00-"]["nis_code"] == "13008" assert set(mol_rows) == {"13025A00-"} + assert set(national_rows) == {"13025A00-", "13008A00-", "11002A00-"} + assert national_rows["11002A00-"]["municipality"] == "Antwerpen" + assert belgium.all_municipalities is True assert module.series_key(regional) == "statbel:population-statistical-sector:kempen-transport-region" assert module.series_key(mol) == "statbel:population-statistical-sector:mol" + assert module.series_key(belgium) == "statbel:population-statistical-sector:belgium" def test_population_operator_resolves_the_persisted_scope_boundary(tmp_path: Path) -> None: @@ -90,6 +96,48 @@ def test_population_operator_resolves_the_persisted_scope_boundary(tmp_path: Pat assert module.resolve_boundary_path(args, scope) == boundary +def test_population_operator_resolves_checksum_verified_belgium_boundary(tmp_path: Path) -> None: + module = load_script("provision_mol_population_history.py") + scope = module.GEOGRAPHIC_SCOPES["belgium"] + scope_dir = tmp_path / "belgium-north-sea" + scope_dir.mkdir(parents=True) + boundary = scope_dir / "belgium_land_boundary.geojson" + boundary.write_text( + json.dumps( + { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Polygon", + "coordinates": [[[2.5, 49.5], [6.4, 49.5], [6.4, 51.5], [2.5, 51.5], [2.5, 49.5]]], + }, + "properties": {}, + } + ], + } + ), + encoding="utf-8", + ) + (scope_dir / "manifest.json").write_text( + json.dumps( + { + "scope": "belgium-and-belgian-north-sea", + "artifacts": { + "belgium_land_boundary": { + "sha256": module.sha256_path(boundary), + } + }, + } + ), + encoding="utf-8", + ) + args = argparse.Namespace(boundary_path=None, scope_output_root=tmp_path) + + assert module.resolve_boundary_path(args, scope) == boundary + + def test_regional_coordinator_builds_explicit_population_and_landuse_commands(tmp_path: Path) -> None: module = load_script("provision_regional_timeseries.py") scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"] diff --git a/backend/tests/test_sprint227_statbel_population_preflight.py b/backend/tests/test_sprint227_statbel_population_preflight.py index 2a7607db..37cc5251 100644 --- a/backend/tests/test_sprint227_statbel_population_preflight.py +++ b/backend/tests/test_sprint227_statbel_population_preflight.py @@ -205,6 +205,42 @@ def test_preflight_reconciles_spatial_and_unlocated_population(tmp_path: Path) - assert len(manifest["schemas"]["geometry_schema_sha256"]) == 64 +def test_preflight_supports_the_complete_national_scope() -> None: + result = PREFLIGHT.validate_statbel_release( + year=2025, + layout="new", + population_content=population_archive(), + population_url=POPULATION_URL, + geometry_content=geometry_archive(), + geometry_url=GEOMETRY_URL, + scope=PREFLIGHT.GEOGRAPHIC_SCOPES["belgium"], + ) + + accounting = result.manifest["scope_accounting"] + assert accounting["scope_key"] == "belgium" + assert accounting["member_count"] == 2 + assert accounting["member_nis_codes"] == ["13008", "13025"] + assert accounting["spatial_population_total"] == 240 + assert accounting["unlocated_population_total"] == 3 + assert accounting["accounted_population_total"] == 243 + + +def test_national_preflight_rejects_an_unscoped_baseline(tmp_path: Path) -> None: + with pytest.raises(PREFLIGHT.StatbelPreflightError) as exc_info: + PREFLIGHT.validate_statbel_release( + year=2025, + layout="new", + population_content=population_archive(), + population_url=POPULATION_URL, + geometry_content=geometry_archive(), + geometry_url=GEOMETRY_URL, + scope=PREFLIGHT.GEOGRAPHIC_SCOPES["belgium"], + baseline_snapshot=baseline_snapshot(tmp_path / "baseline.geojson"), + ) + + assert exc_info.value.code == "STATBEL_BASELINE_SCOPE_MISMATCH" + + def test_preflight_reports_bounded_topology_repairs(tmp_path: Path) -> None: result = validate(tmp_path, geometry_content=geometry_archive(repairable_invalid=True)) diff --git a/backend/tests/test_sprint240_official_flemish_themes.py b/backend/tests/test_sprint240_official_flemish_themes.py index 41fd812a..6ce0dc85 100644 --- a/backend/tests/test_sprint240_official_flemish_themes.py +++ b/backend/tests/test_sprint240_official_flemish_themes.py @@ -125,7 +125,16 @@ def test_product_registries_expose_honest_forest_agriculture_nature_and_soil() - assert raster["agricultural_land_use_2025"]["included_source_values"] == [13, 14] assert "geen juridische bosgrens" in raster["forest_land_use_2025"]["limitation_message"].lower() assert "geen alz-perceelaangifte" in raster["agricultural_land_use_2025"]["limitation_message"].lower() - assert set(vector) == {"bwk_natura2000_2025", "dov_soil_types"} + assert { + "bwk_natura2000_2025", + "dov_soil_types", + "spw_picc_buildings", + "spw_picc_roads", + "spw_picc_waterways", + "spw_picc_water_surfaces", + "urbis_buildings", + "urbis_cadastral_parcels", + } == set(vector) assert vector["bwk_natura2000_2025"]["authority_level"] == "authoritative" assert vector["dov_soil_types"]["authority_level"] == "authoritative_historical_baseline" assert "1949-1971" in vector["dov_soil_types"]["observation_label"] @@ -392,7 +401,7 @@ def test_official_vector_routes_and_frontend_use_canonical_backend_path(monkeypa assert products_response.status_code == 200 assert set(products_response.json()) == {"data"} - assert products_response.json()["data"]["total"] == 2 + assert products_response.json()["data"]["total"] == 8 assert acquire_response.status_code == 200 assert set(acquire_response.json()) == {"data"} assert acquire_response.json()["data"]["job_type"] == "vector.official.acquire" diff --git a/deploy/unraid/geointel-unraid-template.xml b/deploy/unraid/geointel-unraid-template.xml index 2f0daf00..6f7d72cf 100644 --- a/deploy/unraid/geointel-unraid-template.xml +++ b/deploy/unraid/geointel-unraid-template.xml @@ -47,14 +47,18 @@ 10 2 900 - true + true https://geo.api.vlaanderen.be/BWK/wfs https://www.dov.vlaanderen.be/geoserver/wfs + true + https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer + true + https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows 10 - 20000 + 20000 1000 200 - 100000 + 100000 180 20 256 diff --git a/deploy/unraid/geointel.env.example b/deploy/unraid/geointel.env.example index 4c3af412..0b6d561f 100644 --- a/deploy/unraid/geointel.env.example +++ b/deploy/unraid/geointel.env.example @@ -49,10 +49,14 @@ SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS=10 SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB=2 SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS=900 -# Bounded official BWK/Natura 2000 and DOV soil polygons, loaded only after a map selection. +# Bounded official Flemish, Walloon and Brussels vectors, loaded only after a map selection. OFFICIAL_VECTOR_ENABLED=true BWK_WFS_URL=https://geo.api.vlaanderen.be/BWK/wfs DOV_SOIL_WFS_URL=https://www.dov.vlaanderen.be/geoserver/wfs +SPW_PICC_ENABLED=true +SPW_PICC_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer +URBIS_ENABLED=true +URBIS_WFS_URL=https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows OFFICIAL_VECTOR_MIN_SIDE_M=10 OFFICIAL_VECTOR_MAX_SIDE_M=20000 OFFICIAL_VECTOR_PAGE_SIZE=1000 diff --git a/deploy/unraid/run-dockerman-container.sh b/deploy/unraid/run-dockerman-container.sh index cced617b..26e05f16 100644 --- a/deploy/unraid/run-dockerman-container.sh +++ b/deploy/unraid/run-dockerman-container.sh @@ -40,6 +40,10 @@ SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS="${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS OFFICIAL_VECTOR_ENABLED="${OFFICIAL_VECTOR_ENABLED:-true}" BWK_WFS_URL="${BWK_WFS_URL:-https://geo.api.vlaanderen.be/BWK/wfs}" DOV_SOIL_WFS_URL="${DOV_SOIL_WFS_URL:-https://www.dov.vlaanderen.be/geoserver/wfs}" +SPW_PICC_ENABLED="${SPW_PICC_ENABLED:-true}" +SPW_PICC_MAPSERVER_URL="${SPW_PICC_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer}" +URBIS_ENABLED="${URBIS_ENABLED:-true}" +URBIS_WFS_URL="${URBIS_WFS_URL:-https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows}" OFFICIAL_VECTOR_MIN_SIDE_M="${OFFICIAL_VECTOR_MIN_SIDE_M:-10}" OFFICIAL_VECTOR_MAX_SIDE_M="${OFFICIAL_VECTOR_MAX_SIDE_M:-20000}" OFFICIAL_VECTOR_PAGE_SIZE="${OFFICIAL_VECTOR_PAGE_SIZE:-1000}" @@ -204,6 +208,10 @@ docker run -d \ -e OFFICIAL_VECTOR_ENABLED="$OFFICIAL_VECTOR_ENABLED" \ -e BWK_WFS_URL="$BWK_WFS_URL" \ -e DOV_SOIL_WFS_URL="$DOV_SOIL_WFS_URL" \ + -e SPW_PICC_ENABLED="$SPW_PICC_ENABLED" \ + -e SPW_PICC_MAPSERVER_URL="$SPW_PICC_MAPSERVER_URL" \ + -e URBIS_ENABLED="$URBIS_ENABLED" \ + -e URBIS_WFS_URL="$URBIS_WFS_URL" \ -e OFFICIAL_VECTOR_MIN_SIDE_M="$OFFICIAL_VECTOR_MIN_SIDE_M" \ -e OFFICIAL_VECTOR_MAX_SIDE_M="$OFFICIAL_VECTOR_MAX_SIDE_M" \ -e OFFICIAL_VECTOR_PAGE_SIZE="$OFFICIAL_VECTOR_PAGE_SIZE" \ diff --git a/docker-compose.unraid.yml b/docker-compose.unraid.yml index 0a014cdf..9cf1d9f8 100644 --- a/docker-compose.unraid.yml +++ b/docker-compose.unraid.yml @@ -41,6 +41,22 @@ services: SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10} SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2} SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900} + OFFICIAL_VECTOR_ENABLED: ${OFFICIAL_VECTOR_ENABLED:-true} + BWK_WFS_URL: ${BWK_WFS_URL:-https://geo.api.vlaanderen.be/BWK/wfs} + DOV_SOIL_WFS_URL: ${DOV_SOIL_WFS_URL:-https://www.dov.vlaanderen.be/geoserver/wfs} + SPW_PICC_ENABLED: ${SPW_PICC_ENABLED:-true} + SPW_PICC_MAPSERVER_URL: ${SPW_PICC_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer} + URBIS_ENABLED: ${URBIS_ENABLED:-true} + URBIS_WFS_URL: ${URBIS_WFS_URL:-https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows} + OFFICIAL_VECTOR_MIN_SIDE_M: ${OFFICIAL_VECTOR_MIN_SIDE_M:-10} + OFFICIAL_VECTOR_MAX_SIDE_M: ${OFFICIAL_VECTOR_MAX_SIDE_M:-20000} + OFFICIAL_VECTOR_PAGE_SIZE: ${OFFICIAL_VECTOR_PAGE_SIZE:-1000} + OFFICIAL_VECTOR_MAX_PAGES: ${OFFICIAL_VECTOR_MAX_PAGES:-200} + OFFICIAL_VECTOR_MAX_FEATURES: ${OFFICIAL_VECTOR_MAX_FEATURES:-100000} + OFFICIAL_VECTOR_TIMEOUT_SECONDS: ${OFFICIAL_VECTOR_TIMEOUT_SECONDS:-180} + OFFICIAL_VECTOR_MAX_RESPONSE_MB: ${OFFICIAL_VECTOR_MAX_RESPONSE_MB:-20} + OFFICIAL_VECTOR_MAX_TOTAL_RESPONSE_MB: ${OFFICIAL_VECTOR_MAX_TOTAL_RESPONSE_MB:-256} + OFFICIAL_VECTOR_CACHE_TTL_HOURS: ${OFFICIAL_VECTOR_CACHE_TTL_HOURS:-24} DHMV_ENABLED: ${DHMV_ENABLED:-true} DHMV_WCS_URL: ${DHMV_WCS_URL:-https://geo.api.vlaanderen.be/DHMV/wcs} DHMV_RESOLUTION_M: ${DHMV_RESOLUTION_M:-5.0} diff --git a/docker-compose.yml b/docker-compose.yml index e4fc00b3..51cbdb33 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,6 +46,10 @@ services: OFFICIAL_VECTOR_ENABLED: ${OFFICIAL_VECTOR_ENABLED:-true} BWK_WFS_URL: ${BWK_WFS_URL:-https://geo.api.vlaanderen.be/BWK/wfs} DOV_SOIL_WFS_URL: ${DOV_SOIL_WFS_URL:-https://www.dov.vlaanderen.be/geoserver/wfs} + SPW_PICC_ENABLED: ${SPW_PICC_ENABLED:-true} + SPW_PICC_MAPSERVER_URL: ${SPW_PICC_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer} + URBIS_ENABLED: ${URBIS_ENABLED:-true} + URBIS_WFS_URL: ${URBIS_WFS_URL:-https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows} OFFICIAL_VECTOR_MIN_SIDE_M: ${OFFICIAL_VECTOR_MIN_SIDE_M:-10} OFFICIAL_VECTOR_MAX_SIDE_M: ${OFFICIAL_VECTOR_MAX_SIDE_M:-20000} OFFICIAL_VECTOR_PAGE_SIZE: ${OFFICIAL_VECTOR_PAGE_SIZE:-1000} diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 57b193ef..b254ec3a 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -2264,13 +2264,21 @@ return an empty, honest result. No provider request occurs during analysis. Future provider output continues to use DatasetService and, for vectors, VectorFeatureService. Arbitrary service URLs, browser-side fetches, insecure TLS bypasses and startup downloads remain forbidden. -## Governed official nature and soil acquisition +## Governed regional official-vector acquisition ### GET `/api/v1/projects/{project_id}/datasets/official-vector/products` Returns the fixed official vector registry in the canonical `{ "data": ... }` -envelope. The allowlist contains `bwk_natura2000_2025` and `dov_soil_types`; -arbitrary collection names or URLs are never accepted. +envelope. Every item includes its theme, geometry contract, authority, +licence/attribution, provider-native collection, query mode and +`coverage_zones`. The fixed allowlist contains: + +- Flanders: `bwk_natura2000_2025` and `dov_soil_types`; +- Wallonia: `spw_picc_buildings`, `spw_picc_roads`, + `spw_picc_waterways` and `spw_picc_water_surfaces`; +- Brussels: `urbis_buildings` and `urbis_cadastral_parcels`. + +Arbitrary collection names or URLs are never accepted. ### POST `/api/v1/projects/{project_id}/datasets/official-vector/acquire` @@ -2293,16 +2301,29 @@ Request: The synchronous `vector.official.acquire` Job validates the metric request size, intersects `bbox` with the persisted Area, retrieves every bounded page, -clips polygon geometry in EPSG:31370 and persists EPSG:4326 features through -`DatasetService`. A repeated exact request can reuse the 24-hour cache. -Provider errors, unstable or incomplete WFS pagination and safety-limit violations -fail without persisting a truncated Dataset. +clips source geometry in the provider-native metric CRS and persists +EPSG:4326 features through `DatasetService`. A repeated exact request can +reuse the 24-hour cache. Provider errors, unstable/incomplete pagination and +safety-limit violations fail without persisting a truncated Dataset. + +Walloon and Brussels products additionally require a persisted exact regional +coverage Area (`Wallonia` or `Brussels-Capital Region`). SPW/PICC uses stable +`OBJECTID` ArcGIS REST paging; UrbIS uses WFS 2.0 paging ordered by +`INSPIRE_ID`. A caller cannot make a Flemish product cover Wallonia, or merge +different regional authorities into one Dataset. `bwk_natura2000_2025` preserves BWK `EVAL`, `EENH*`, `HAB*` and `PHAB*` semantics. `dov_soil_types` preserves mapped soil, texture, drainage, profile and substrate classes and is dated as the 1949-1971 survey period. Neither contract accepts a caller-supplied endpoint. +PICC building footprints, road axes and hydrographic axes/surfaces retain +their source identifiers and report source-appropriate counts, footprint/ +surface hectares or line kilometres. UrbIS building footprints and cadastral +parcels remain separate products with their own Paradigm/FPS Finance licence +notes. These products do not claim semantic parity with GRB and do not expose +water volume. + ## Governed Landgebruik Vlaanderen forest and agriculture The existing diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index ff5e65fc..04f7ea23 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,5 +1,28 @@ ## Autonomous RC program for Belgium and the Belgian North Sea (2026-07-17) +### Post-RC national data federation (2026-07-19) + +- Added explicit administrative and maritime themes for persisted NGI/RBINS + reference data and made readiness depend on analyzable themes rather than a + raw Dataset count. +- Generalized the governed Statbel population scripts to the Belgium boundary + and complete municipality inventory while retaining all release-promotion + safety gates. +- Added allowlisted bounded SPW/PICC ArcGIS REST and UrbIS WFS adapters with + source-specific coverage, CRS, identity, paging, geometry, licence, + provenance and metric contracts. Output still enters PostGIS only through + `DatasetService.import_vector_bytes`. +- Made the frontend resolve actual coverage zones before choosing GRB, + SPW/PICC or UrbIS and retain separate source results for mixed-zone + selections. +- Added SPW/UrbIS runtime controls to both Compose variants and the editable + Unraid/Dockerman deployment path. +- Rechecked MDK bathymetry. Strict hostname validation still fails because the + configured official hostname presents a `*.l27powered.eu` certificate, so + acquisition remains truthfully disabled. +- Focused backend and frontend suites passed before full release validation; + full local and Tower evidence follows in the final P5 gate. + ### RC-4 national and maritime coverage foundation - Added an independent coverage registry so the exact Sprint 7 diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index e0d6800c..79cccfbb 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -751,9 +751,37 @@ inferred from these plan zones. The coverage resolver reports a theme `operational` only when a matching `ready` Dataset exists in the selected project. Implemented acquisition -without project materialization is `partial`; Walloon, Brussels, Statbel and -MDK families remain `not_configured` until their own bounded adapters and -source-specific metric contracts are implemented. +without project materialization is `partial`. Statbel is an operator-managed +national source. SPW/PICC and UrbIS are bounded API sources for their explicit +themes and zones. MDK remains `not_configured` while its strict-TLS readiness +probe fails. + +The governed Statbel operator supports `belgium` as a first-class scope. It +uses the retained NGI Belgium boundary, retains every statistical-sector row +and requires the existing plan -> stage -> named review -> checksum-confirmed +apply process. Flanders, Wallonia, Brussels and bounded municipality +population queries therefore derive from one reviewed national edition rather +than mutually incompatible regional imports. No edition is downloaded or +promoted during application startup. + +## Wallonia PICC and Brussels UrbIS + +The official-vector registry exposes bounded, source-specific regional +baselines: + +- SPW/PICC MapServer layers 11 (buildings), 21 (road axes), 28 + (hydrographic axes) and 30 (hydrographic surfaces) for Wallonia; +- Paradigm UrbIS `Buildings` and `CadastralParcels` WFS collections for the + Brussels-Capital Region. + +SPW queries use stable `OBJECTID` paging and request EPSG:4326 output while +retaining EPSG:3812 as the native source CRS. UrbIS responses use EPSG:31370, +stable `INSPIRE_ID` paging and are transformed only after exact regional +clipping. Both adapters enforce bbox dimensions, page/feature/response limits, +strict HTTPS host/path validation, geometry contracts and persisted coverage +Areas. They persist only through `DatasetService`; the browser never contacts +either provider directly. Cross-zone selections remain split by authority and +metric semantics. ## Bathymetry, inland profiles and maritime scope @@ -768,10 +796,10 @@ The following sources are audited but not yet operational: - MDK Dieptemodel Belgisch Continentaal Plat/Noordzee: 20 x 20 m continuous raster in LAT, exposed through WCS/WMTS. GeoIntel now has a strict-TLS, - read-only GetCapabilities probe. On 2026-07-17 the official metadata - endpoint presented a certificate for another hostname and returned no - usable capabilities path; status therefore remains `tls_error` and raster - acquisition is disabled. + read-only GetCapabilities probe. Revalidation on 2026-07-19 still found that + `bathy.agentschapmdk.be` presents a certificate for `*.l27powered.eu`. + Hostname validation therefore fails with `tls_error`; raster acquisition + remains disabled and TLS verification cannot be bypassed. - SPW bathymetry of navigable waterways and reservoir lakes: 0.5 m bed elevation and XYZ data in mDNG. - Port of Antwerp-Bruges periodic soundings: catalog candidate pending a diff --git a/docs/POST_RC_DATA_COVERAGE_ROADMAP_BELGIUM_NORTH_SEA.md b/docs/POST_RC_DATA_COVERAGE_ROADMAP_BELGIUM_NORTH_SEA.md new file mode 100644 index 00000000..91197269 --- /dev/null +++ b/docs/POST_RC_DATA_COVERAGE_ROADMAP_BELGIUM_NORTH_SEA.md @@ -0,0 +1,236 @@ +# Post-RC Data Coverage Roadmap: Belgium And The Belgian North Sea + +## Status and purpose + +This is the active autonomous implementation board after `v1.0.0-rc.1`. +It is not an RC-12 phase. The signed RC remains immutable release evidence; +all work below targets the next version and must preserve the frozen API, +persistence, provenance and no-fake-data rules. + +The goal is to make the national workbench useful with real data: + +1. choose an understandable theme; +2. select a bounded area anywhere in Belgium or the Belgian North Sea; +3. acquire or reuse only governed data that covers that selection; +4. calculate source-appropriate metrics; +5. expose authority, time, licence, coverage and limitations; +6. keep unsupported or unavailable results visibly unavailable. + +Mol and the Kempen remain mandatory regression areas. They do not constrain +the geographic implementation. + +## Autonomous execution rules + +- Read `docs/CODEX_BOOTSTRAP_PROMPT.md` and + `docs/RC_SCOPE_FREEZE_BELGIUM_NORTH_SEA.md` before every resumed pass. +- Never run an unbounded browser-side or application-startup provider fetch. +- Every external endpoint must be allowlisted, strict-TLS and size/time limited. +- Provider output must be staged, validated and persisted through + `DatasetService` and `VectorFeatureService`. +- No source may become `operational` from catalogue evidence alone. +- Do not merge semantically different regional sources into one value. +- Do not create water volume, bathymetry, population, AI or historical values + when the required evidence is absent. +- A failed source or validation gate stops only that source. Other independent + phases may continue. +- Update `docs/CODEX_EXECUTION_LOG.md`, `docs/TODO.md`, relevant source/API + docs and this roadmap after every completed gate. + +## Global validation gate + +Run after every implementation phase: + +```bash +python -m compileall backend/app +cd backend && python -m pytest +cd frontend && npm run test:unit +cd frontend && npm run typecheck +cd frontend && npm run build +bash scripts/run_readiness_check.sh +cd backend && python -m alembic heads +cd backend && python -m alembic upgrade head --sql +bash -n scripts/live_migration_smoke.sh +``` + +When Docker/Tower is available: + +```bash +docker compose config +curl --fail http://192.168.10.150:1202/health/ready +bash scripts/run_rc8_release_journeys.sh \ + http://192.168.10.150:1202 \ + artifacts/post-rc-release-journeys \ + artifacts/post-rc-golden-areas.json +``` + +No phase is complete when compile, tests, typecheck, build, migration or the +applicable live bounded journey fails. + +## P0 - Existing national layers become usable + +**State: implemented locally; live release evidence pending under P5.** + +### Work + +- Add understandable map themes for administrative context, maritime planning + and marine reporting/legal context. +- Match persisted `ngi_adminvector`, `rbins_marine_reporting_units` and + `rbins_msp_2026` datasets explicitly; do not rely on filename substrings. +- Automatically select the first operational theme when the previous/default + theme has no matching dataset in the active workspace. +- Add source-appropriate selection summaries. Administrative polygon area may + be reported in hectares; overlapping marine use zones must not be summed as + unique sea area. +- Make workbench readiness distinguish stored context/reference layers from + actually analyzable themes. +- Add frontend and backend regression tests and verify the live national and + North Sea workspaces without console errors. + +### Exit + +- Belgium land opens with a usable NGI administrative theme. +- The Belgian North Sea opens with usable legal/reporting and marine-plan + themes. +- Drawing or selecting a bounded area produces persisted-source results with + provenance. +- An unavailable buildings/population/bathymetry source remains unavailable. + +## P1 - National Statbel population + +**State: implementation complete; reviewed live national materialization pending.** + +### Work + +- Extend the existing governed Statbel plan-stage-review-apply flow with a + national scope that uses the persisted Belgium boundary. +- Keep the 2025 REDEGEO old/new geometries separate. +- Validate archive checksums, schema, CRS, sector joins, `ZZZZ` accounting and + national totals before persistence. +- Materialize one immutable national edition through `DatasetService`; bounded + map queries continue to run in PostGIS. +- Expose inhabitants as an area-weighted estimate only when a selection cuts + statistical sectors, with an explicit warning and compatible time-series + keys. +- Never download or replace a release automatically. + +### Exit + +- A named reviewed plan is required for live apply. +- Belgium, Flanders, Wallonia and Brussels bounded population selections work + from the same authoritative Statbel edition. +- Mol remains within its documented baseline tolerance. + +## P2 - Wallonia bounded topographic baseline + +**State: implemented locally; live Wallonia journey pending under P5.** + +### Governed source + +SPW PICC ArcGIS REST: + +```text +https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer +``` + +Initial allowlisted layers: + +- `11`: building footprints; +- `21`: road axes; +- `28`: hydrographic axes; +- `30`: hydrographic surfaces. + +### Work + +- Add a bounded ArcGIS REST adapter using an EPSG:4326 selection transformed by + the official service to EPSG:3812/4326. +- Enforce maximum selection dimensions, paging, response size, feature count, + strict content type, stable layer identity and source metadata. +- Normalize geometry to EPSG:4326 while retaining provider-native attributes. +- Persist buildings, roads and water as separate immutable datasets. +- Add hectare/kilometre metrics without claiming semantic parity with GRB. +- Expose the products only for Walloon intersections and keep mixed-zone + results split. + +### Exit + +- The Wallonia golden area can explicitly acquire and analyse all allowlisted + PICC themes. +- Provider failures and record-limit truncation fail closed. +- No direct provider-to-`vector_features` write exists. + +## P3 - Brussels bounded UrbIS baseline + +**State: implemented locally; live Brussels journey pending under P5.** + +### Governed source + +Paradigm UrbIS parcels and buildings WFS, discovered and pinned from the +official datastore metadata. The exact service URL and type names must pass a +capabilities probe before implementation is marked operational. + +### Work + +- Add bounded WFS acquisition for building footprints and cadastral parcels. +- Retain the separate Paradigm and FPS Finance authorities/licences where + applicable. +- Enforce bbox, paging, response size, CRS and geometry validation. +- Persist separate immutable building and parcel datasets. +- Add source-appropriate area/count metrics and Brussels-only coverage. + +### Exit + +- The Brussels golden area can explicitly acquire and analyse buildings and + parcels. +- Catalogue-only or failed capabilities evidence remains `not_configured`. + +## P4 - Maritime and bathymetry hardening + +**State: complete with MDK acquisition visibly blocked by strict-TLS evidence.** + +### Work + +- Keep the current RBINS marine legal/reporting and 2026-2034 planning layers + operational and map-queryable. +- Probe the MDK 20 m LAT product only through strict TLS and retained metadata. +- Implement bounded GeoTIFF acquisition only after CRS, nodata, pixel + semantics, edition, licence and LAT evidence pass. +- Keep depth relative to LAT separate from water-surface elevation and volume. +- SPW mDNG bathymetry remains a separate Walloon inland/navigation source. + +### Exit + +- Maritime plan and legal/reporting queries are operational. +- Bathymetry is either a governed analytical raster with vertical datum and + uncertainty evidence, or remains visibly `not_configured`. +- No water-volume metric exists without compatible bed and water-surface data. + +## P5 - Federation, UX and release evidence + +**State: in progress.** + +### Work + +- Make on-demand products depend on the intersected zone, not on a manually + selected technical project. +- Keep cross-region selections split by source and metric semantics. +- Present loaded, available-on-demand, partial, not-configured and unsupported + states in ordinary Dutch language. +- Run land, language-boundary, Brussels, Wallonia, coast and North Sea golden + journeys against live PostGIS. +- Capture source checksums, selected bounds, result metrics, provenance, + performance, browser console/network and responsive screenshots. +- Build, push and deploy one immutable post-RC image only after every applicable + gate is green. + +### Final acceptance + +- Existing RC behavior remains green. +- At least one real bounded analytical theme is operational for Belgium, + Flanders, Wallonia, Brussels and the Belgian North Sea. +- Population is nationally operational or explicitly blocked by failed source + evidence. +- Buildings/roads/water are operational through the competent regional source + where implemented; no parity claim is made. +- No unavailable theme is presented as measured data. +- The live browser can select an area, acquire/reuse data, analyse it, inspect + provenance and export it without entering a technical project workflow. diff --git a/docs/TODO.md b/docs/TODO.md index 5eb0a254..988e72a9 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1,5 +1,20 @@ # GeoIntel TODO +## Actieve post-RC datadekkingsfase + +`docs/POST_RC_DATA_COVERAGE_ROADMAP_BELGIUM_NORTH_SEA.md` is het actieve +autonome uitvoeringsbord na `v1.0.0-rc.1`. Dit is geen RC-12. De ondertekende +RC blijft immutable bewijs; deze fase maakt echte, begrensde nationale en +maritieme databronnen bruikbaar in de volgende versie. + +- [x] P0: bestaande NGI/RBINS-lagen als kaartthema en selectieanalyse ontsluiten. +- [x] P0: readiness baseren op bruikbare analysethema's, niet alleen datasetaantal. +- [ ] P1: nationale Statbel-populatieflow veilig operationaliseren. +- [x] P2: bounded SPW PICC voor Waalse gebouwen, wegen en hydrographie. +- [x] P3: bounded UrbIS voor Brusselse gebouwen en percelen. +- [x] P4: maritieme thema's harden en bathymetrie fail-closed onderzoeken. +- [ ] P5: zonefederatie, UX, golden journeys en immutable deployment. + ## Actief RC-programma: Belgie en de Belgische Noordzee `docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md` is vanaf 2026-07-17 het enige actieve diff --git a/frontend/README.md b/frontend/README.md index 577a7527..f0607417 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -723,7 +723,10 @@ remain regression workspaces and can still be selected normally. Drawing a rectangle on the Map invokes the GeoIntel coverage resolver after a short debounce. The map displays every intersected land or legal sea zone and the active theme as `Beschikbaar`, `Gedeeltelijk`, `Niet gekoppeld` or `Niet -ondersteund`. A coastal or cross-region selection remains visibly split. The +ondersteund`. The same resolved zones determine which on-demand source is +eligible: GRB for Flanders, SPW/PICC for Wallonia and UrbIS for Brussels. +Products are filtered by both theme and `coverage_zones`; a coastal or +cross-region selection remains visibly split and separately persisted. The browser never calls NGI, SPW, UrbIS, RBINS or MDK directly and never promotes an audited catalog entry to operational data without a matching ready Dataset. diff --git a/frontend/src/components/WorkbenchStatusStrip.test.tsx b/frontend/src/components/WorkbenchStatusStrip.test.tsx new file mode 100644 index 00000000..f0d886e0 --- /dev/null +++ b/frontend/src/components/WorkbenchStatusStrip.test.tsx @@ -0,0 +1,64 @@ +import { cleanup, render, screen } from '@testing-library/react' +import { afterEach, describe, expect, it } from 'vitest' +import type { AreaRead, DatasetCreateResponse, ProjectRead } from '../types' +import { WorkbenchStatusStrip } from './WorkbenchStatusStrip' + +const project = { + id: 'project-1', + name: 'Belgium and North Sea Workbench', + region: 'Belgium and Belgian North Sea', +} as unknown as ProjectRead + +const areas = [{ id: 'area-1', name: 'Belgium land' }] as unknown as AreaRead[] + +function renderStatus(datasets: DatasetCreateResponse[]): void { + render( + , + ) +} + +afterEach(cleanup) + +describe('WorkbenchStatusStrip analytical readiness', () => { + it('treats the persisted NGI administrative baseline as analyzable', () => { + renderStatus([ + { + id: 'dataset-1', + dataset_type: 'vector', + source: 'operator_official_import', + source_name: 'ngi_adminvector', + reference_layer_name: 'belgium_municipalities', + dataset_role: 'reference', + status: 'ready', + } as unknown as DatasetCreateResponse, + ]) + + expect(screen.getByText('Kaartwerkruimte is gebruiksklaar')).toBeTruthy() + expect(screen.getByText("1 analysethema")).toBeTruthy() + }) + + it('does not call an unrelated stored context file an analysis theme', () => { + renderStatus([ + { + id: 'dataset-2', + dataset_type: 'vector', + source: 'manual', + source_name: 'manual', + reference_layer_name: 'custom_context', + dataset_role: 'reference', + status: 'ready', + } as unknown as DatasetCreateResponse, + ]) + + expect(screen.getByText('Kaartwerkruimte vraagt aandacht')).toBeTruthy() + expect(screen.getByText("0 analysethema's")).toBeTruthy() + }) +}) diff --git a/frontend/src/components/WorkbenchStatusStrip.tsx b/frontend/src/components/WorkbenchStatusStrip.tsx index ba128dd0..53ee85ea 100644 --- a/frontend/src/components/WorkbenchStatusStrip.tsx +++ b/frontend/src/components/WorkbenchStatusStrip.tsx @@ -32,6 +32,59 @@ function countReferenceDatasets(datasets: DatasetCreateResponse[]): number { return datasets.filter((dataset) => dataset.dataset_role === 'reference').length } +const ANALYTICAL_SOURCE_NAMES = new Set([ + 'ngi_adminvector', + 'rbins_marine_reporting_units', + 'rbins_msp_2026', + 'grb', + 'department_omgeving_land_use', + 'historical_landuse', + 'department_omgeving_thematic_raster', + 'statbel', + 'inbo_bwk_natura2000', + 'agentschap_landbouw_zeevisserij_agricultural_parcels', + 'dov_soil_map', + 'digitaal_vlaanderen_dhmv', + 'vmm_flood_hazard', + 'vmm_vha_bathymetry_profiles', +]) + +const ANALYTICAL_LAYER_NAMES = new Set([ + 'administrative', + 'belgium_land_boundary', + 'belgium_regions', + 'belgium_provinces', + 'belgium_municipalities', + 'buildings', + 'roads', + 'water', + 'parcels', + 'population', + 'forest', + 'nature_value', + 'agriculture', + 'soil', + 'bathymetry', + 'flood_hazard', + 'elevation', + 'maritime_planning', + 'marine_environment', + 'marine_legal_scopes', + 'marine_spatial_plan_2026', +]) + +function isAnalyzableDataset(dataset: DatasetCreateResponse): boolean { + if (dataset.status !== 'ready') return false + const sourceName = String(dataset.source_name ?? dataset.source ?? '').toLowerCase() + const layerName = String( + dataset.source_metadata?.['theme'] + ?? dataset.reference_layer_name + ?? dataset.source_metadata?.['layer_type'] + ?? '', + ).toLowerCase() + return ANALYTICAL_SOURCE_NAMES.has(sourceName) || ANALYTICAL_LAYER_NAMES.has(layerName) +} + function statusLabel(state: StatusItem['state']): string { if (state === 'ready') return 'gereed' if (state === 'warning') return 'aandacht' @@ -73,6 +126,7 @@ export function WorkbenchStatusStrip({ selectedAreaHasGeometry, }: WorkbenchStatusStripProps): JSX.Element { const readyDatasets = datasets.filter((dataset) => dataset.status === 'ready').length + const analyzableDatasets = datasets.filter(isAnalyzableDataset).length const vectorDatasets = countByDatasetType(datasets, 'vector') + countByDatasetType(datasets, 'geojson') const rasterDatasets = countByDatasetType(datasets, 'raster') const referenceDatasets = countReferenceDatasets(datasets) @@ -97,9 +151,12 @@ export function WorkbenchStatusStrip({ { key: 'datasets', label: 'Bronnen', - value: `${readyDatasets}/${datasets.length} beschikbaar`, - detail: `${vectorDatasets} kaartlagen, ${rasterDatasets} luchtbeelden, ${referenceDatasets} officiële referenties`, - state: datasets.length === 0 ? 'waiting' : readyDatasets === datasets.length ? 'ready' : 'warning', + value: `${analyzableDatasets} analysethema${analyzableDatasets === 1 ? '' : "'s"}`, + detail: ( + `${readyDatasets}/${datasets.length} bronnen gereed · ` + + `${vectorDatasets} kaartlagen, ${rasterDatasets} rasters, ${referenceDatasets} officiële referenties` + ), + state: analyzableDatasets === 0 ? 'waiting' : readyDatasets === datasets.length ? 'ready' : 'warning', }, { key: 'map', diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index d57f50a7..fbeb0154 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react' +import { useCallback, useEffect, useMemo, useRef, useState, type KeyboardEvent } from 'react' import GeoMap from '../GeoMap' import type { AreaRead, CoverageResolveResponse, CoverageStatus, DatasetCreateResponse, DetectionQaResult, MapResultExportRequest, MapViewportState, OrthophotoAcquisitionResult, OrthophotoProductRead, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionMetric, VectorSelectionResponse } from '../../types' import { useMapThemeSelectionInsights, type MapThemeAcquisition, type MapThemeQuery } from '../../hooks/useMapThemeSelectionInsights' @@ -30,10 +30,12 @@ import { normalizeBboxFromCorners, operationalScopeProjectLabel, parseBboxInput, + productCoversZones, readablePropertyName, resultCountLabel, resultMetricLabel, safeFileStem, + selectedAreaCoverageZones, selectedFeatureCollection, selectionAreaSquareMetres, selectionMetricLabel, @@ -43,7 +45,26 @@ const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson' const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson' const EMPTY_TEMPORAL_SERIES: DatasetCreateResponse[] = [] -type DataThemeId = 'buildings' | 'space_occupation' | 'open_space' | 'population' | 'forest' | 'nature_value' | 'agriculture' | 'soil' | 'water' | 'bathymetry' | 'flood_hazard' | 'elevation' | 'accessibility' | 'services' | 'roads' | 'parcels' +type DataThemeId = + | 'administrative' + | 'buildings' + | 'space_occupation' + | 'open_space' + | 'population' + | 'forest' + | 'nature_value' + | 'agriculture' + | 'soil' + | 'water' + | 'bathymetry' + | 'flood_hazard' + | 'elevation' + | 'accessibility' + | 'services' + | 'roads' + | 'parcels' + | 'maritime_planning' + | 'marine_environment' interface DataTheme { id: DataThemeId @@ -67,6 +88,13 @@ interface OnDemandMapProduct extends MapThemeAcquisition { } const DATA_THEMES: DataTheme[] = [ + { + id: 'administrative', + label: 'Bestuurlijke indeling', + shortLabel: 'Bestuursgebieden', + description: 'Officiële lands-, gewest-, provincie- en gemeentegrenzen van het NGI.', + tokens: ['administrative', 'adminvector', 'belgium_land_boundary', 'belgium_regions', 'belgium_provinces', 'belgium_municipalities'], + }, { id: 'buildings', label: 'Bebouwing', @@ -179,9 +207,24 @@ const DATA_THEMES: DataTheme[] = [ description: 'Kadastrale of administratieve perceelcontouren.', tokens: ['parcels', 'parcel', 'percelen', 'perceel', 'cadastre', 'kadaster'], }, + { + id: 'maritime_planning', + label: 'Maritieme planning', + shortLabel: 'Plan- en gebruikszones', + description: 'Officiële gebruiks- en beschermingszones uit het Belgisch Marien Ruimtelijk Plan 2026-2034.', + tokens: ['maritime_planning', 'marine_spatial_plan', 'rbins_msp', 'bmsp', 'imsp26'], + }, + { + id: 'marine_environment', + label: 'Mariene rapportagezones', + shortLabel: 'Zeegebieden', + description: 'Officiële juridische en mariene rapportagegebieden voor het Belgische deel van de Noordzee.', + tokens: ['marine_environment', 'marine_legal_scopes', 'marine_reporting_units', 'rbins_marine_reporting'], + }, ] const COVERAGE_THEME_BY_MAP_THEME: Record = { + administrative: 'admin', buildings: 'buildings', space_occupation: 'land_cover_use', open_space: 'land_cover_use', @@ -198,6 +241,8 @@ const COVERAGE_THEME_BY_MAP_THEME: Record = { services: 'population', roads: 'roads', parcels: 'parcels', + maritime_planning: 'maritime_planning', + marine_environment: 'marine_environment', } function coverageStatusLabel(status: CoverageStatus): string { @@ -225,6 +270,7 @@ function coverageZoneLabel(zone: string): string { } const DATA_THEME_MAP_STYLES: Record = { + administrative: { fill: '#5f6f7f', line: '#344554' }, buildings: { fill: '#d45f3d', line: '#9f3e24' }, space_occupation: { fill: '#be3e33', line: '#8f2c24' }, open_space: { fill: '#267a46', line: '#175c32' }, @@ -241,6 +287,8 @@ const DATA_THEME_MAP_STYLES: Record services: { fill: '#b66d16', line: '#854d0e' }, roads: { fill: '#6b7280', line: '#4b5563' }, parcels: { fill: '#a7792f', line: '#7d571f' }, + maritime_planning: { fill: '#2f7f8f', line: '#145d6a' }, + marine_environment: { fill: '#3475a3', line: '#1c557d' }, } function datasetAvailabilityLabel( @@ -274,6 +322,15 @@ function datasetAvailabilityLabel( const resolutionLabel = Number.isFinite(resolution) ? `${resolution.toLocaleString('nl-BE')} m` : 'Raster' return `${resolutionLabel} officiële bron${Number.isFinite(year) ? ` · ${year}` : ''}` } + if (dataset.source_name === 'ngi_adminvector') { + return `${(dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0).toLocaleString('nl-BE')} officiële bestuursgebieden` + } + if (dataset.source_name === 'rbins_msp_2026') { + return `${(dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0).toLocaleString('nl-BE')} officiële planobjecten · 2026-2034` + } + if (dataset.source_name === 'rbins_marine_reporting_units') { + return `${(dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0).toLocaleString('nl-BE')} officiële zeegebieden` + } return `${(dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0).toLocaleString('nl-BE')} objecten beschikbaar` } @@ -313,6 +370,15 @@ function datasetMatchesTheme(dataset: DatasetCreateResponse, theme: DataTheme): if (dataset.source_name === 'dov_soil_map') { return theme.id === 'soil' } + if (dataset.source_name === 'ngi_adminvector') { + return theme.id === 'administrative' + } + if (dataset.source_name === 'rbins_msp_2026') { + return theme.id === 'maritime_planning' + } + if (dataset.source_name === 'rbins_marine_reporting_units') { + return theme.id === 'marine_environment' + } const searchText = datasetSearchText(dataset) return theme.tokens.some((token) => searchText.includes(token)) } @@ -342,8 +408,17 @@ function datasetProductKey(dataset: DatasetCreateResponse): string { function datasetCoversSelectedArea( dataset: DatasetCreateResponse, selectedAreaId: string | null, + selectedAreaName: string | null | undefined, regionalScope = false, ): boolean { + const selectedZones = selectedAreaCoverageZones(selectedAreaName) + const configuredZones = dataset.source_metadata?.['coverage_zones'] + if (selectedZones && Array.isArray(configuredZones) && configuredZones.length > 0) { + const datasetZones = configuredZones.map((zone) => String(zone)) + if (!selectedZones.some((zone) => datasetZones.includes(zone))) { + return false + } + } const coverageScope = String(dataset.source_metadata?.['coverage_scope'] ?? '') if (isPartitionedBathymetry(dataset)) { return regionalScope @@ -363,6 +438,7 @@ function rasterPartitionsForDataset( datasets: DatasetCreateResponse[], representative: DatasetCreateResponse | null, selectedAreaId: string | null, + selectedAreaName: string | null | undefined, regionalScope: boolean, ): DatasetCreateResponse[] { if (!representative) { @@ -382,7 +458,7 @@ function rasterPartitionsForDataset( ? String(dataset.source_metadata?.['partition_manifest_sha256'] ?? '') === manifestSha256 : datasetProductKey(dataset) === productKey ) - && datasetCoversSelectedArea(dataset, selectedAreaId, true), + && datasetCoversSelectedArea(dataset, selectedAreaId, selectedAreaName, true), ) .sort((left, right) => String(left.area_id ?? '').localeCompare(String(right.area_id ?? ''))) } @@ -391,12 +467,13 @@ function pickThemeDataset( datasets: DatasetCreateResponse[], theme: DataTheme, selectedAreaId: string | null, + selectedAreaName: string | null | undefined, regionalScope = false, ): DatasetCreateResponse | null { const candidates = datasets.filter( (dataset) => datasetMatchesTheme(dataset, theme) - && datasetCoversSelectedArea(dataset, selectedAreaId, regionalScope), + && datasetCoversSelectedArea(dataset, selectedAreaId, selectedAreaName, regionalScope), ) candidates.sort((left, right) => { const priorityScore = (dataset: DatasetCreateResponse) => @@ -723,7 +800,15 @@ export function MapWorkspace({ }) const activeScopeProject = projects.find((project) => project.id === selectedProjectId) ?? null const activeScopeLabel = activeScopeProject ? operationalScopeProjectLabel(activeScopeProject) : 'Werkgebied' - const flandersScopeSelected = activeScopeProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME + const selectedMapArea = areas.find((area) => area.id === selectedMapAreaId) + const selectedCoverageZones = useMemo( + () => selectedAreaCoverageZones(selectedMapArea?.name), + [selectedMapArea?.name], + ) + const flandersScopeSelected = Boolean( + selectedCoverageZones?.includes('flanders') + || (!selectedCoverageZones && activeScopeProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME), + ) const { themeInsights, themeInsightsLoading: themeResultsLoading, @@ -735,7 +820,8 @@ export function MapWorkspace({ products: officialMapProducts, loading: officialMapProductsLoading, error: officialMapProductsError, - } = useOfficialMapProducts(flandersScopeSelected ? selectedProjectId : null) + resolveCoverage, + } = useOfficialMapProducts(selectedProjectId) const { temporalComparison, temporalComparisonLoading, @@ -759,7 +845,6 @@ export function MapWorkspace({ const [fullWorkflowMode, setFullWorkflowMode] = useState<'new' | 'reuse'>('new') const [mapAnalysisDurationMs, setMapAnalysisDurationMs] = useState(null) const mapAnalysisRequestSequence = useRef(0) - const selectedMapArea = areas.find((area) => area.id === selectedMapAreaId) const regionalScopeSelected = Boolean(selectedMapArea && !isMunicipalityAreaName(selectedMapArea.name)) const featureProperties = selectedMapFeature?.properties ?? null const isBathymetryProfile = featureProperties?.['provider'] === 'vmm_vha_bathymetry_profiles' @@ -794,7 +879,7 @@ export function MapWorkspace({ .filter( (dataset) => dataset.source_name === 'vmm_flood_hazard' - && datasetCoversSelectedArea(dataset, selectedMapAreaId, regionalScopeSelected), + && datasetCoversSelectedArea(dataset, selectedMapAreaId, selectedMapArea?.name, regionalScopeSelected), ) .sort((left, right) => floodScenarioLabel(left).localeCompare(floodScenarioLabel(right), 'nl')) if (!regionalScopeSelected) { @@ -809,13 +894,19 @@ export function MapWorkspace({ } return Array.from(products.values()) }, - [availableMapDatasets, regionalScopeSelected, selectedMapAreaId], + [availableMapDatasets, regionalScopeSelected, selectedMapArea?.name, selectedMapAreaId], ) const themeDatasetMap = useMemo(() => { const result = Object.fromEntries( DATA_THEMES.map((theme) => [ theme.id, - pickThemeDataset(availableMapDatasets, theme, selectedMapAreaId, regionalScopeSelected), + pickThemeDataset( + availableMapDatasets, + theme, + selectedMapAreaId, + selectedMapArea?.name, + regionalScopeSelected, + ), ]), ) as Record const selectedFloodHazard = floodHazardDatasets.find( @@ -833,7 +924,7 @@ export function MapWorkspace({ (dataset) => dataset.source_name === 'digitaal_vlaanderen_dhmv' && datasetProductKey(dataset) === selectedDhmvProductKey - && datasetCoversSelectedArea(dataset, selectedMapAreaId, regionalScopeSelected), + && datasetCoversSelectedArea(dataset, selectedMapAreaId, selectedMapArea?.name, regionalScopeSelected), ) ?? null } if (flandersScopeSelected && officialMapProducts.thematic.length > 0) { @@ -846,8 +937,10 @@ export function MapWorkspace({ result[product.key] = null } } - if (flandersScopeSelected && officialMapProducts.officialVector.length > 0) { - for (const product of officialMapProducts.officialVector) { + if (officialMapProducts.officialVector.length > 0) { + for (const product of officialMapProducts.officialVector.filter((item) => + productCoversZones(item.coverage_zones, selectedCoverageZones), + )) { result[product.theme] = null } } @@ -865,7 +958,9 @@ export function MapWorkspace({ selectedDhmvProductKey, selectedFloodHazardDatasetId, selectedFloodHazardProductKey, + selectedMapArea?.name, selectedMapAreaId, + selectedCoverageZones, ]) const themePartitionMap = useMemo( () => @@ -876,11 +971,12 @@ export function MapWorkspace({ availableMapDatasets, themeDatasetMap[theme.id], selectedMapAreaId, + selectedMapArea?.name, regionalScopeSelected, ), ]), ) as Record, - [availableMapDatasets, regionalScopeSelected, selectedMapAreaId, themeDatasetMap], + [availableMapDatasets, regionalScopeSelected, selectedMapArea?.name, selectedMapAreaId, themeDatasetMap], ) const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0] const activeCoverageTheme = COVERAGE_THEME_BY_MAP_THEME[activeTheme.id] @@ -892,14 +988,16 @@ export function MapWorkspace({ ) ?? { operational: 0, partial: 0, not_configured: 0, unsupported: 0 }, [coverage], ) - const onDemandProductMap = useMemo( - () => { - const result = new Map() - if (!flandersScopeSelected) { - return result - } + const onDemandProductsForZones = useCallback((zones: string[] | null): OnDemandMapProduct[] => { + const result: OnDemandMapProduct[] = [] + const effectiveZones = zones ?? selectedCoverageZones + const includesFlanders = Boolean( + effectiveZones?.includes('flanders') + || (!effectiveZones && activeScopeProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME), + ) + if (includesFlanders) { for (const product of officialMapProducts.thematic) { - result.set(product.theme, { + result.push({ kind: 'thematic_raster', productKey: product.key, displayName: product.display_name, @@ -910,7 +1008,7 @@ export function MapWorkspace({ }) } for (const product of officialMapProducts.grb) { - result.set(product.key, { + result.push({ kind: 'grb', productKey: product.key, displayName: product.display_name, @@ -920,52 +1018,65 @@ export function MapWorkspace({ limitationMessage: product.limitation_message, }) } - for (const product of officialMapProducts.officialVector) { - result.set(product.theme, { - kind: 'official_vector', - productKey: product.key, - displayName: product.display_name, - theme: product.theme, - availabilityLabel: `${product.observation_label} · officiële vectorbron · laad bij selectie`, - attribution: product.attribution, - limitationMessage: product.limitation_message, - }) - } - const dhmvProduct = officialMapProducts.dhmv.find((product) => product.key === selectedDhmvProductKey) - if (dhmvProduct) { - result.set('elevation', { - kind: 'dhmv', - productKey: dhmvProduct.key, - displayName: dhmvProduct.display_name, - theme: 'elevation', - availabilityLabel: `${dhmvProduct.native_resolution_m} m · ${dhmvProduct.acquisition_period} · laad bij selectie`, - attribution: dhmvProduct.attribution, - limitationMessage: dhmvProduct.limitation_message, - }) - } - const floodProduct = officialMapProducts.floodHazard.find( - (product) => product.key === selectedFloodHazardProductKey, - ) - if (floodProduct) { - result.set('flood_hazard', { - kind: 'flood_hazard', - productKey: floodProduct.key, - displayName: floodProduct.display_name, - theme: 'flood_hazard', - availabilityLabel: `${floodProduct.native_resolution_m} m · ${floodProduct.climate_context} · T${floodProduct.return_period_years} · laad bij selectie`, - attribution: floodProduct.attribution, - limitationMessage: floodProduct.limitation_message, - }) - } - return result - }, - [ - flandersScopeSelected, - officialMapProducts, - selectedDhmvProductKey, - selectedFloodHazardProductKey, - ], - ) + } + for (const product of officialMapProducts.officialVector.filter((item) => + productCoversZones(item.coverage_zones, effectiveZones), + )) { + result.push({ + kind: 'official_vector', + productKey: product.key, + displayName: product.display_name, + theme: product.theme, + availabilityLabel: `${product.observation_label} · officiële vectorbron · laad bij selectie`, + attribution: product.attribution, + limitationMessage: product.limitation_message, + }) + } + const dhmvProduct = includesFlanders + ? officialMapProducts.dhmv.find((product) => product.key === selectedDhmvProductKey) + : null + if (dhmvProduct) { + result.push({ + kind: 'dhmv', + productKey: dhmvProduct.key, + displayName: dhmvProduct.display_name, + theme: 'elevation', + availabilityLabel: `${dhmvProduct.native_resolution_m} m · ${dhmvProduct.acquisition_period} · laad bij selectie`, + attribution: dhmvProduct.attribution, + limitationMessage: dhmvProduct.limitation_message, + }) + } + const floodProduct = includesFlanders + ? officialMapProducts.floodHazard.find( + (product) => product.key === selectedFloodHazardProductKey, + ) + : null + if (floodProduct) { + result.push({ + kind: 'flood_hazard', + productKey: floodProduct.key, + displayName: floodProduct.display_name, + theme: 'flood_hazard', + availabilityLabel: `${floodProduct.native_resolution_m} m · ${floodProduct.climate_context} · T${floodProduct.return_period_years} · laad bij selectie`, + attribution: floodProduct.attribution, + limitationMessage: floodProduct.limitation_message, + }) + } + return result + }, [ + activeScopeProject?.name, + officialMapProducts, + selectedCoverageZones, + selectedDhmvProductKey, + selectedFloodHazardProductKey, + ]) + const onDemandProductMap = useMemo(() => { + const result = new Map() + for (const product of onDemandProductsForZones(selectedCoverageZones)) { + result.set(product.theme, product) + } + return result + }, [onDemandProductsForZones, selectedCoverageZones]) const activeOnDemandMapProduct = onDemandProductMap.get(activeTheme.id) ?? null const activeThemeMapStyle = DATA_THEME_MAP_STYLES[activeTheme.id] const analysisOverlayActive = mapContentMode === 'analysis' && analysisLayerAvailable && Boolean(mapFeatureCollection) @@ -993,15 +1104,20 @@ export function MapWorkspace({ useEffect(() => { if ( - !flandersScopeSelected - || analysisMode !== 'current' + analysisMode !== 'current' || activeThemeAvailable - || (onDemandProductMap.size === 0 && !officialMapProductsError) + || ( + selectedProjectId + && officialMapProductsLoading + && onDemandProductMap.size === 0 + && !officialMapProductsError + ) ) { return } const fallbackTheme = DATA_THEMES.find((theme) => - theme.id === 'space_occupation' + flandersScopeSelected + && theme.id === 'space_occupation' && Boolean(themeDatasetMap[theme.id] || onDemandProductMap.get(theme.id)), ) ?? DATA_THEMES.find((theme) => Boolean(themeDatasetMap[theme.id] || onDemandProductMap.get(theme.id)), @@ -1019,8 +1135,10 @@ export function MapWorkspace({ analysisMode, flandersScopeSelected, officialMapProductsError, + officialMapProductsLoading, onDemandProductMap, onOpenDatasetInMap, + selectedProjectId, themeDatasetMap, ]) @@ -1528,20 +1646,37 @@ export function MapWorkspace({ } const loadAllThemeResults = async (bbox: VectorSelectionBBox, areaId?: string) => { + let resolvedZones = selectedCoverageZones + if (analysisMode === 'current' && selectedProjectId) { + const resolvedCoverage = await resolveCoverage({ + minx: bbox.min_x, + miny: bbox.min_y, + maxx: bbox.max_x, + maxy: bbox.max_y, + }) + if (!resolvedCoverage) { + clearThemeInsights() + return + } + resolvedZones = resolvedCoverage.intersected_zones + } + const resolvedProducts = analysisMode === 'current' + ? onDemandProductsForZones(resolvedZones) + : [] const availableThemes: Array> = [] for (const theme of DATA_THEMES) { - const onDemandProduct = analysisMode === 'current' - ? onDemandProductMap.get(theme.id) - : null - if (onDemandProduct) { - availableThemes.push({ - themeId: theme.id, - acquisition: { - kind: onDemandProduct.kind, - productKey: onDemandProduct.productKey, - displayName: onDemandProduct.displayName, - }, - }) + const onDemandProducts = resolvedProducts.filter((product) => product.theme === theme.id) + if (onDemandProducts.length > 0) { + for (const onDemandProduct of onDemandProducts) { + availableThemes.push({ + themeId: theme.id, + acquisition: { + kind: onDemandProduct.kind, + productKey: onDemandProduct.productKey, + displayName: onDemandProduct.displayName, + }, + }) + } continue } const dataset = themeDatasetMap[theme.id] @@ -1856,10 +1991,10 @@ export function MapWorkspace({ : activeTheme.description} - {officialMapProductsLoading && flandersScopeSelected ? ( + {officialMapProductsLoading && selectedProjectId ? (

Beschikbare Vlaamse kaartbronnen worden gecontroleerd…

) : null} - {officialMapProductsError && flandersScopeSelected ? ( + {officialMapProductsError && selectedProjectId ? (

{officialMapProductsError}

) : null} @@ -1877,7 +2012,12 @@ export function MapWorkspace({ (item) => item.source_name === 'digitaal_vlaanderen_dhmv' && datasetProductKey(item) === productKey - && datasetCoversSelectedArea(item, selectedMapAreaId, regionalScopeSelected), + && datasetCoversSelectedArea( + item, + selectedMapAreaId, + selectedMapArea?.name, + regionalScopeSelected, + ), ) if (dataset) { onOpenDatasetInMap(dataset) diff --git a/frontend/src/components/map/mapWorkspaceUtils.test.ts b/frontend/src/components/map/mapWorkspaceUtils.test.ts index 6d565141..eb7fade0 100644 --- a/frontend/src/components/map/mapWorkspaceUtils.test.ts +++ b/frontend/src/components/map/mapWorkspaceUtils.test.ts @@ -3,7 +3,9 @@ import { bboxesEqual, normalizeBboxFromCorners, parseBboxInput, + productCoversZones, resultMetricLabel, + selectedAreaCoverageZones, selectionAreaSquareMetres, } from './mapWorkspaceUtils' import type { VectorSelectionResponse } from '../../types' @@ -47,4 +49,18 @@ describe('map workspace selection guards', () => { } as unknown as VectorSelectionResponse expect(resultMetricLabel(result)).toBe('14,24 ha') }) + + it('maps national work areas to regional provider zones without merging sources', () => { + expect(selectedAreaCoverageZones('Belgium land')).toEqual([ + 'belgium', + 'flanders', + 'wallonia', + 'brussels', + ]) + expect(selectedAreaCoverageZones('RC Ardennes inland')).toEqual(['wallonia']) + expect(selectedAreaCoverageZones('Brussels-Capital Region')).toEqual(['brussels']) + expect(selectedAreaCoverageZones('Language boundary')).toEqual(['flanders', 'wallonia']) + expect(productCoversZones(['wallonia'], ['flanders', 'wallonia'])).toBe(true) + expect(productCoversZones(['brussels'], ['wallonia'])).toBe(false) + }) }) diff --git a/frontend/src/components/map/mapWorkspaceUtils.ts b/frontend/src/components/map/mapWorkspaceUtils.ts index 41912718..14897476 100644 --- a/frontend/src/components/map/mapWorkspaceUtils.ts +++ b/frontend/src/components/map/mapWorkspaceUtils.ts @@ -13,6 +13,28 @@ import { const MOL_PROJECT_NAME = 'Mol Municipality Workbench' const KEMPEN_PROJECT_NAME = 'Kempen Regional Workbench' +export function selectedAreaCoverageZones(areaName: string | null | undefined): string[] | null { + const normalized = String(areaName ?? '').toLowerCase() + if (!normalized) return null + if (normalized.includes('coast land-sea')) return ['flanders', 'belgian_north_sea'] + if (normalized.includes('language boundary')) return ['flanders', 'wallonia'] + if (normalized.includes('north sea')) { + return ['belgian_north_sea', 'territorial_sea', 'exclusive_economic_zone', 'continental_shelf'] + } + if (normalized.includes('territorial sea')) return ['territorial_sea'] + if (normalized.includes('exclusive economic zone')) return ['exclusive_economic_zone'] + if (normalized.includes('continental shelf')) return ['continental_shelf'] + if (normalized.includes('brussels')) return ['brussels'] + if (normalized.includes('wallonia') || normalized.includes('ardennes')) return ['wallonia'] + if (normalized.includes('flanders') || normalized.includes('mol') || normalized.includes('kempen')) return ['flanders'] + if (normalized.includes('belgium land')) return ['belgium', 'flanders', 'wallonia', 'brussels'] + return null +} + +export function productCoversZones(productZones: string[], selectedZones: string[] | null): boolean { + return selectedZones === null || selectedZones.some((zone) => productZones.includes(zone)) +} + export function operationalScopeProjectLabel(project: ProjectRead): string { if (project.name === MOL_PROJECT_NAME) { return 'Mol' diff --git a/frontend/src/hooks/useOfficialMapProducts.ts b/frontend/src/hooks/useOfficialMapProducts.ts index b5ff8a83..2e43e26b 100644 --- a/frontend/src/hooks/useOfficialMapProducts.ts +++ b/frontend/src/hooks/useOfficialMapProducts.ts @@ -1,5 +1,5 @@ -import { useEffect, useState } from 'react' -import { datasetsApi } from '../services/api' +import { useCallback, useEffect, useState } from 'react' +import { datasetsApi, externalApi } from '../services/api' import { formatError } from '../lib/formatError' import type { DhmvProductRead, @@ -9,7 +9,7 @@ import type { ThematicRasterProductRead, } from '../types' -interface OfficialMapProducts { +export interface OfficialMapProducts { thematic: ThematicRasterProductRead[] dhmv: DhmvProductRead[] floodHazard: FloodHazardProductRead[] @@ -78,5 +78,26 @@ export function useOfficialMapProducts(selectedProjectId: string | null) { } }, [selectedProjectId]) - return { products, loading, error } + const resolveCoverage = useCallback( + async (bbox: { minx: number; miny: number; maxx: number; maxy: number }) => { + if (!selectedProjectId) { + setError('Selecteer eerst een werkruimte.') + return null + } + try { + const coverage = await externalApi.resolveCoverage({ + projectId: selectedProjectId, + bbox, + }) + setError(null) + return coverage + } catch (requestError) { + setError(formatError(requestError, 'De regionale databronnen konden niet veilig worden bepaald.')) + return null + } + }, + [selectedProjectId], + ) + + return { products, loading, error, resolveCoverage } } diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 675463dd..87eeb93d 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -393,11 +393,11 @@ export interface OfficialVectorAcquireRequest { export interface OfficialVectorProductRead { key: string display_name: string - theme: 'nature_value' | 'soil' + theme: 'buildings' | 'roads' | 'water' | 'parcels' | 'nature_value' | 'soil' provider: string source_name: string reference_layer_name: string - service_type: 'OGC API Features' | 'WFS 2.0' + service_type: 'OGC API Features' | 'WFS 2.0' | 'ArcGIS REST' collection: string geometry_types: string[] source_crs: string @@ -408,6 +408,7 @@ export interface OfficialVectorProductRead { attribution: string license_note: string limitation_message: string + coverage_zones: string[] } export interface TerrainSelectionResponse { diff --git a/scripts/geographic_scopes.py b/scripts/geographic_scopes.py index f5162c13..cef6bd18 100644 --- a/scripts/geographic_scopes.py +++ b/scripts/geographic_scopes.py @@ -27,6 +27,7 @@ class GeographicScope: scope_type: str limitation_message: str members: tuple[ScopeMember, ...] + all_municipalities: bool = False @property def nis_codes(self) -> tuple[str, ...]: @@ -96,18 +97,37 @@ KEMPEN_TRANSPORT_REGION_SCOPE = GeographicScope( ), ) +BELGIUM_SCOPE = GeographicScope( + key="belgium", + display_name="België", + project_name="Belgium and North Sea Workbench", + project_region="Belgium and Belgian North Sea", + area_name="Belgium land", + authority_name="National Geographic Institute (NGI), AdminVector", + authority_url="https://www.geo.be/catalog/details/fb1e2993-2020-428c-9188-eb5f75e284b9", + scope_type="country", + limitation_message=( + "De nationale Statbel-sectorlaag dekt het Belgische landgebied. " + "Belgische maritieme zones bevatten geen statistische bevolkingssectoren." + ), + members=(), + all_municipalities=True, +) + GEOGRAPHIC_SCOPES = { scope.key: scope - for scope in (MOL_SCOPE, KEMPEN_TRANSPORT_REGION_SCOPE) + for scope in (MOL_SCOPE, KEMPEN_TRANSPORT_REGION_SCOPE, BELGIUM_SCOPE) } def validate_scope(scope: GeographicScope) -> None: if not scope.key or not scope.project_name or not scope.area_name: raise ValueError("Geographic scope identity fields must not be empty") - if len(scope.members) == 0: + if len(scope.members) == 0 and not scope.all_municipalities: raise ValueError(f"Geographic scope {scope.key} has no members") + if scope.all_municipalities and scope.scope_type != "country": + raise ValueError(f"Geographic scope {scope.key} can include all municipalities only for a country scope") names = [member.name.casefold() for member in scope.members] codes = [member.nis_code for member in scope.members] if len(names) != len(set(names)): diff --git a/scripts/manage_statbel_population_release.py b/scripts/manage_statbel_population_release.py index aabf9a5f..47a77341 100644 --- a/scripts/manage_statbel_population_release.py +++ b/scripts/manage_statbel_population_release.py @@ -50,7 +50,7 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("action", choices=("plan", "stage", "review", "apply")) parser.add_argument("--project-id", required=True, help="GeoIntel project UUID for the governed scope") - parser.add_argument("--scope", choices=(DEFAULT_SCOPE,), default=DEFAULT_SCOPE) + parser.add_argument("--scope", choices=tuple(sorted(GEOGRAPHIC_SCOPES)), default=DEFAULT_SCOPE) parser.add_argument("--api-url", default=os.environ.get("GEOINTEL_API_URL", DEFAULT_API_URL)) parser.add_argument("--confirm-edition", help="Exact official population year required after plan") parser.add_argument("--confirm-layout", choices=("standard", "new"), help="Exact planned REDEGEO layout") diff --git a/scripts/provision_belgium_north_sea_scope.py b/scripts/provision_belgium_north_sea_scope.py index be459edc..680b2575 100644 --- a/scripts/provision_belgium_north_sea_scope.py +++ b/scripts/provision_belgium_north_sea_scope.py @@ -99,6 +99,15 @@ AREA_NAMES = { "continental_shelf": "Belgian continental shelf beyond territorial sea", } +THEME_BY_LAYER = { + "belgium_land_boundary": "administrative", + "belgium_regions": "administrative", + "belgium_provinces": "administrative", + "belgium_municipalities": "administrative", + "marine_legal_scopes": "marine_environment", + "marine_spatial_plan_2026": "maritime_planning", +} + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Provision the Belgium and Belgian North Sea scope.") @@ -656,6 +665,7 @@ def _upload_dataset( source_metadata = { "provider": source_name, "authority_level": "authoritative", + "theme": THEME_BY_LAYER[reference_layer_name], "coverage_zones": coverage_zones, "reference_layer_name": reference_layer_name, "source_url": source_url, diff --git a/scripts/provision_mol_population_history.py b/scripts/provision_mol_population_history.py index 96a812f5..496712a2 100644 --- a/scripts/provision_mol_population_history.py +++ b/scripts/provision_mol_population_history.py @@ -255,6 +255,23 @@ def resolve_boundary_path(args: argparse.Namespace, scope: GeographicScope) -> P return Path(legacy) if scope.key == "mol" and DEFAULT_BOUNDARY_PATH.exists(): return DEFAULT_BOUNDARY_PATH + if scope.key == "belgium": + national_scope_dir = args.scope_output_root / "belgium-north-sea" + manifest_path = national_scope_dir / "manifest.json" + boundary_path = national_scope_dir / "belgium_land_boundary.geojson" + if not manifest_path.is_file() or not boundary_path.is_file(): + raise RuntimeError( + "Official Belgium boundary evidence is missing; " + "run provision_belgium_north_sea_scope.py --fetch-only first" + ) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + artifact = (manifest.get("artifacts") or {}).get("belgium_land_boundary") or {} + if ( + manifest.get("scope") != "belgium-and-belgian-north-sea" + or artifact.get("sha256") != sha256_path(boundary_path) + ): + raise RuntimeError("Official Belgium boundary evidence does not match the retained manifest") + return boundary_path scope_dir = args.scope_output_root / scope.key manifest_path = scope_dir / f"{scope.key.replace('-', '_')}_scope_manifest.json" if not manifest_path.exists(): @@ -330,13 +347,14 @@ def scoped_population_rows(source_rows: list[dict[str, Any]], scope: GeographicS if sector_code in seen: raise RuntimeError(f"Statbel population table contains duplicate sector {sector_code}") seen.add(sector_code) - if nis_code not in members: + if not scope.all_municipalities and nis_code not in members: continue + municipality_name = members.get(nis_code) or str(row.get("TX_DESCR_NL") or nis_code).strip() rows[sector_code] = { "population_total": int(total_raw), "sector_name_nl": row.get("TX_DESCR_SECTOR_NL"), "municipality_name_nl": row.get("TX_DESCR_NL"), - "municipality": members[nis_code], + "municipality": municipality_name, "nis_code": nis_code, } if not rows: @@ -358,7 +376,10 @@ def build_snapshot( missing_population = 0 for source_feature in sector_payload.get("features") or []: properties = source_feature.get("properties") or {} - if str(properties.get("cd_munty_refnis") or "") not in member_codes: + if ( + not scope.all_municipalities + and str(properties.get("cd_munty_refnis") or "") not in member_codes + ): continue sector_code = str(properties.get("cd_sector") or "").strip() population_values = population.get(sector_code) @@ -406,8 +427,8 @@ def build_snapshot( "features": features, "coverage_scope": scope.key, "scope_type": scope.scope_type, - "member_count": len(scope.members), - "member_nis_codes": list(scope.nis_codes), + "member_count": int(accounting.get("member_count") or len(scope.members)), + "member_nis_codes": list(accounting.get("member_nis_codes") or scope.nis_codes), "geometry_clipped_to_area": True, "observation_year": year, "missing_population_sector_count": missing_population, @@ -594,14 +615,21 @@ def upload_snapshot( observed_at = f"{year}-01-01T00:00:00Z" preflight = load_preflight_manifest(preflight_path, path, year, scope) accounting = preflight["scope_accounting"] + coverage_zones = ( + ["belgium", "flanders", "wallonia", "brussels"] + if scope.key == "belgium" + else ["flanders"] + ) source_metadata = { "provider": "Statbel", "authority_level": "authoritative", + "theme": "population", + "coverage_zones": coverage_zones, "coverage_scope": scope.key, "scope_type": scope.scope_type, "scope_display_name": scope.display_name, - "member_count": len(scope.members), - "member_nis_codes": list(scope.nis_codes), + "member_count": int(accounting.get("member_count") or len(scope.members)), + "member_nis_codes": list(accounting.get("member_nis_codes") or scope.nis_codes), "geometry_clipped_to_area": True, "attribution": ATTRIBUTION, "license": "CC BY 4.0", @@ -742,6 +770,7 @@ def main() -> int: "path": path, "manifest_path": manifest_path if manifest_path.is_file() else None, "feature_count": len(payload.get("features") or []), + "member_count": int(payload.get("member_count") or len(scope.members)), "preflight_status": preflight_status, } ) @@ -814,7 +843,10 @@ def main() -> int: "status": "ok", "scope": scope.key, "display_name": scope.display_name, - "member_count": len(scope.members), + "member_count": max( + (int(item.get("member_count") or 0) for item in prepared), + default=len(scope.members), + ), "series": series_key(scope), "snapshots": results, }, diff --git a/scripts/statbel_population_preflight.py b/scripts/statbel_population_preflight.py index 09f8b364..0c978ceb 100644 --- a/scripts/statbel_population_preflight.py +++ b/scripts/statbel_population_preflight.py @@ -509,6 +509,10 @@ def parse_geometry_archive(content: bytes, year: int) -> GeometryArchiveData: def _validate_scope(scope: GeographicScope, geometry: GeometryArchiveData, population: PopulationArchiveData) -> None: requested = set(scope.nis_codes) + if scope.all_municipalities: + if not geometry.municipality_by_sector or not population.rows: + _fail("STATBEL_SCOPE_REJECTED", "National geographic scope has no source municipalities.") + return if not requested or any(not MUNICIPALITY_CODE_PATTERN.fullmatch(value) for value in requested): _fail("STATBEL_SCOPE_REJECTED", "Approved geographic scope contains invalid NIS codes.") geometry_codes = set(geometry.municipality_by_sector.values()) @@ -540,7 +544,16 @@ def _baseline_summary(path: Path | None, *, year: int, scope: GeographicScope) - baseline_year=baseline_year, candidate_year=year, ) - if set(str(value) for value in payload.get("member_nis_codes") or []) != set(scope.nis_codes): + baseline_scope = payload.get("coverage_scope") + if ( + (scope.all_municipalities and baseline_scope != scope.key) + or (not scope.all_municipalities and baseline_scope not in {None, scope.key}) + ): + _fail("STATBEL_BASELINE_SCOPE_MISMATCH", "Baseline snapshot does not use the same approved scope.") + if ( + not scope.all_municipalities + and set(str(value) for value in payload.get("member_nis_codes") or []) != set(scope.nis_codes) + ): _fail("STATBEL_BASELINE_SCOPE_MISMATCH", "Baseline snapshot does not use the same approved scope.") features = payload.get("features") if not isinstance(features, list) or not features: @@ -625,7 +638,11 @@ def validate_statbel_release( national_unlocated_total = sum(int(population.rows[code]["TOTAL"]) for code in population_without_geometry) if national_spatial_total + national_unlocated_total != population.population_total: _fail("STATBEL_TOTAL_RECONCILIATION_FAILED", "National population accounting does not reconcile.") - requested = set(scope.nis_codes) + requested = ( + set(geometry.municipality_by_sector.values()) + if scope.all_municipalities + else set(scope.nis_codes) + ) scope_spatial_codes = { code for code in geometry_codes if geometry.municipality_by_sector[code] in requested } @@ -700,8 +717,8 @@ def validate_statbel_release( "scope_accounting": { "scope_key": scope.key, "scope_display_name": scope.display_name, - "member_count": len(scope.members), - "member_nis_codes": list(scope.nis_codes), + "member_count": len(requested), + "member_nis_codes": sorted(requested), "spatial_sector_count": len(scope_spatial_codes), "spatial_population_total": scope_spatial_total, "unlocated_row_count": len(scope_unlocated_codes),