Add Belgium and North Sea coverage foundation
This commit is contained in:
@@ -49,6 +49,19 @@
|
||||
- Completed live Detection Lab acceptance with an intentionally empty raster
|
||||
choice, disabled start action and zero browser console errors. No stale
|
||||
running Job or AnalysisRun remained after reconciliation.
|
||||
- Added a separate national coverage registry with normalized themes, legal
|
||||
land/sea zones and only four honest states: `operational`, `partial`,
|
||||
`not_configured` and `unsupported`.
|
||||
- Added canonical coverage catalog/resolve endpoints. Drawn bboxes are
|
||||
resolved against persisted Areas, cross-zone output stays split and
|
||||
`operational` requires a matching ready Dataset.
|
||||
- Added the explicit Belgium/North Sea operator with fixed official NGI/RBINS
|
||||
allowlists, strict TLS, size limits, safe archive extraction, complete WFS
|
||||
pagination, checksums and API-only persistence.
|
||||
- Added national-workspace preference and a compact selection coverage surface
|
||||
to the Map UI without removing Mol/Kempen regression workspaces.
|
||||
- Local RC-4 validation passed 986 backend tests, frontend typecheck/build,
|
||||
the full readiness gate, one Alembic head and complete offline migration SQL.
|
||||
|
||||
## Sprint 240 Operational forest, agriculture, nature and soil themes (2026-07-17)
|
||||
|
||||
|
||||
@@ -987,6 +987,30 @@ it does not create a derived dataset.
|
||||
|
||||
## Geographic scope provisioning
|
||||
|
||||
The release-candidate national foundation is provisioned explicitly:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_belgium_north_sea_scope.py
|
||||
```
|
||||
|
||||
Use `--fetch-only` to validate official NGI AdminVector, RBINS marine
|
||||
reporting units and the Belgian Marine Spatial Plan 2026-2034 without changing
|
||||
application persistence. The normal command creates or reuses
|
||||
`Belgium and North Sea Workbench`, persists Belgium, all three regions,
|
||||
territorial sea, EEZ and continental shelf as Areas, and uploads six
|
||||
checksum-bound reference Datasets through `DatasetService`.
|
||||
|
||||
The operator has a fixed URL/layer allowlist, verified TLS, archive and
|
||||
response-size limits, safe ZIP extraction, complete WFS pagination and
|
||||
immutable artifact checksums. It does not run at startup and does not write
|
||||
directly to `vector_features`.
|
||||
|
||||
`GET /api/v1/external/coverage/catalog` exposes audited national source
|
||||
contracts. `POST /api/v1/external/coverage/resolve` intersects a drawn bbox
|
||||
with persisted legal/administrative Areas and reports a split zone/theme
|
||||
matrix. `operational` requires a matching `ready` Dataset; integration without
|
||||
materialized data is only `partial`.
|
||||
|
||||
The explicit operator command below provisions the official 28-municipality
|
||||
Vlaamse vervoerregio Kempen boundary foundation:
|
||||
|
||||
|
||||
@@ -7,7 +7,8 @@ from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.models import Area, Project
|
||||
from app.providers.registry import fetch_provider_data, get_provider, import_provider_dataset, list_provider_capabilities
|
||||
from app.schemas import ExternalFetchRequest, ExternalFetchResponse, ProviderImportRequest
|
||||
from app.schemas import CoverageResolveRequest, ExternalFetchRequest, ExternalFetchResponse, ProviderImportRequest
|
||||
from app.services.coverage_registry_service import CoverageRegistryService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/external", tags=["external"])
|
||||
@@ -46,6 +47,22 @@ def list_external_providers() -> dict:
|
||||
})
|
||||
|
||||
|
||||
@router.get("/coverage/catalog")
|
||||
def get_coverage_catalog() -> dict:
|
||||
return envelope(CoverageRegistryService.catalog().model_dump())
|
||||
|
||||
|
||||
@router.post("/coverage/resolve")
|
||||
def resolve_project_coverage(payload: CoverageResolveRequest, db: Session = Depends(get_db)) -> dict:
|
||||
result = CoverageRegistryService.resolve(
|
||||
db,
|
||||
project_id=payload.project_id,
|
||||
bbox=payload.bbox,
|
||||
themes=payload.themes,
|
||||
)
|
||||
return envelope(result.model_dump())
|
||||
|
||||
|
||||
@router.get("/providers/capabilities")
|
||||
def get_external_provider_capabilities() -> dict:
|
||||
return envelope({
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .common import ApiErrorEnvelope, ApiErrorItem, Envelope, PaginationEnvelope
|
||||
from .coverage import (
|
||||
CoverageBBox,
|
||||
CoverageCatalogResponse,
|
||||
CoverageResolutionItem,
|
||||
CoverageResolveRequest,
|
||||
CoverageResolveResponse,
|
||||
CoverageSourceContract,
|
||||
)
|
||||
from .project import ProjectCreate, ProjectList, ProjectRead, ProjectUpdate
|
||||
from .area import AreaCreate, AreaList, AreaRead, AreaUpdate
|
||||
from .analysis import ChangeDetectionRequest, ChangeDetectionSummary
|
||||
@@ -145,6 +153,12 @@ __all__ = [
|
||||
"ApiErrorEnvelope",
|
||||
"ApiErrorItem",
|
||||
"PaginationEnvelope",
|
||||
"CoverageBBox",
|
||||
"CoverageCatalogResponse",
|
||||
"CoverageResolutionItem",
|
||||
"CoverageResolveRequest",
|
||||
"CoverageResolveResponse",
|
||||
"CoverageSourceContract",
|
||||
"ProjectCreate",
|
||||
"ProjectRead",
|
||||
"ProjectUpdate",
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
|
||||
CoverageStatus = Literal["operational", "partial", "not_configured", "unsupported"]
|
||||
CoverageAuthority = Literal["authoritative", "official_context", "contextual"]
|
||||
CoverageAcquisitionMode = Literal[
|
||||
"operator_archive",
|
||||
"operator_wfs",
|
||||
"bounded_api",
|
||||
"bounded_raster",
|
||||
"catalog_only",
|
||||
]
|
||||
|
||||
|
||||
class CoverageBBox(BaseModel):
|
||||
minx: float = Field(ge=-180, le=180)
|
||||
miny: float = Field(ge=-90, le=90)
|
||||
maxx: float = Field(ge=-180, le=180)
|
||||
maxy: float = Field(ge=-90, le=90)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_extent(self) -> "CoverageBBox":
|
||||
if self.maxx <= self.minx or self.maxy <= self.miny:
|
||||
raise ValueError("bbox max values must be greater than min values")
|
||||
return self
|
||||
|
||||
|
||||
class CoverageSourceContract(BaseModel):
|
||||
source_name: str
|
||||
display_name: str
|
||||
authority_level: CoverageAuthority
|
||||
coverage_zones: list[str]
|
||||
themes: list[str]
|
||||
native_layers: list[str]
|
||||
supported_geometry_types: list[str]
|
||||
acquisition_mode: CoverageAcquisitionMode
|
||||
integration_status: CoverageStatus
|
||||
source_url: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class CoverageCatalogResponse(BaseModel):
|
||||
themes: list[str]
|
||||
zones: list[str]
|
||||
statuses: list[CoverageStatus]
|
||||
sources: list[CoverageSourceContract]
|
||||
|
||||
|
||||
class CoverageResolveRequest(BaseModel):
|
||||
project_id: UUID
|
||||
bbox: CoverageBBox
|
||||
themes: list[str] = Field(default_factory=list, max_length=32)
|
||||
|
||||
|
||||
class CoverageResolutionItem(BaseModel):
|
||||
zone: str
|
||||
theme: str
|
||||
status: CoverageStatus
|
||||
source_names: list[str]
|
||||
materialized_dataset_ids: list[UUID]
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class CoverageResolveResponse(BaseModel):
|
||||
project_id: UUID
|
||||
bbox: CoverageBBox
|
||||
requested_themes: list[str]
|
||||
intersected_zones: list[str]
|
||||
outside_supported_scope: bool
|
||||
items: list[CoverageResolutionItem]
|
||||
warnings: list[str]
|
||||
@@ -0,0 +1,513 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import box
|
||||
from shapely.ops import unary_union
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.schemas.coverage import (
|
||||
CoverageBBox,
|
||||
CoverageCatalogResponse,
|
||||
CoverageResolutionItem,
|
||||
CoverageResolveResponse,
|
||||
CoverageSourceContract,
|
||||
)
|
||||
|
||||
|
||||
THEMES = (
|
||||
"admin",
|
||||
"buildings",
|
||||
"roads",
|
||||
"surface_water",
|
||||
"land_cover_use",
|
||||
"nature",
|
||||
"population",
|
||||
"parcels",
|
||||
"soil",
|
||||
"elevation",
|
||||
"orthophoto",
|
||||
"flood_climate",
|
||||
"maritime_planning",
|
||||
"marine_environment",
|
||||
"bathymetry",
|
||||
)
|
||||
|
||||
ZONES = (
|
||||
"belgium",
|
||||
"flanders",
|
||||
"wallonia",
|
||||
"brussels",
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
)
|
||||
|
||||
STATUS_ORDER = ("unsupported", "not_configured", "partial", "operational")
|
||||
STATUS_RANK = {status: index for index, status in enumerate(STATUS_ORDER)}
|
||||
|
||||
SCOPE_AREA_NAMES = {
|
||||
"belgium": "Belgium land",
|
||||
"flanders": "Flanders",
|
||||
"wallonia": "Wallonia",
|
||||
"brussels": "Brussels-Capital Region",
|
||||
"belgian_north_sea": "Belgian part of the North Sea",
|
||||
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
|
||||
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
|
||||
"continental_shelf": "Belgian continental shelf beyond territorial sea",
|
||||
}
|
||||
|
||||
DETAIL_ZONES = (
|
||||
"flanders",
|
||||
"wallonia",
|
||||
"brussels",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SourceDefinition:
|
||||
contract: CoverageSourceContract
|
||||
materialized_layer_names: tuple[str, ...] = ()
|
||||
materialized_source_names: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def _contract(
|
||||
*,
|
||||
source_name: str,
|
||||
display_name: str,
|
||||
authority_level: str,
|
||||
coverage_zones: tuple[str, ...],
|
||||
themes: tuple[str, ...],
|
||||
native_layers: tuple[str, ...],
|
||||
geometry_types: tuple[str, ...],
|
||||
acquisition_mode: str,
|
||||
integration_status: str,
|
||||
source_url: str,
|
||||
attribution: str,
|
||||
license_note: str,
|
||||
limitation_message: str,
|
||||
materialized_layer_names: tuple[str, ...] = (),
|
||||
materialized_source_names: tuple[str, ...] = (),
|
||||
) -> _SourceDefinition:
|
||||
return _SourceDefinition(
|
||||
contract=CoverageSourceContract(
|
||||
source_name=source_name,
|
||||
display_name=display_name,
|
||||
authority_level=authority_level,
|
||||
coverage_zones=list(coverage_zones),
|
||||
themes=list(themes),
|
||||
native_layers=list(native_layers),
|
||||
supported_geometry_types=list(geometry_types),
|
||||
acquisition_mode=acquisition_mode,
|
||||
integration_status=integration_status,
|
||||
source_url=source_url,
|
||||
attribution=attribution,
|
||||
license_note=license_note,
|
||||
limitation_message=limitation_message,
|
||||
),
|
||||
materialized_layer_names=materialized_layer_names,
|
||||
materialized_source_names=materialized_source_names or (source_name,),
|
||||
)
|
||||
|
||||
|
||||
SOURCE_DEFINITIONS = (
|
||||
_contract(
|
||||
source_name="ngi_adminvector",
|
||||
display_name="NGI AdminVector",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("belgium", "flanders", "wallonia", "brussels", "belgian_north_sea"),
|
||||
themes=("admin",),
|
||||
native_layers=(
|
||||
"belgianterritory",
|
||||
"belgianmaritimezone",
|
||||
"region",
|
||||
"province",
|
||||
"municipality",
|
||||
),
|
||||
geometry_types=("Polygon", "MultiPolygon"),
|
||||
acquisition_mode="operator_archive",
|
||||
integration_status="operational",
|
||||
source_url="https://www.geo.be/catalog/details/fb1e2993-2020-428c-9188-eb5f75e284b9",
|
||||
attribution="National Geographic Institute (NGI), AdminVector",
|
||||
license_note="CC BY 4.0",
|
||||
limitation_message="Administrative reference geometry; it does not provide thematic land content.",
|
||||
materialized_layer_names=(
|
||||
"belgium_land_boundary",
|
||||
"belgium_regions",
|
||||
"belgium_provinces",
|
||||
"belgium_municipalities",
|
||||
),
|
||||
),
|
||||
_contract(
|
||||
source_name="statbel",
|
||||
display_name="Statbel statistical sectors and population",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("belgium", "flanders", "wallonia", "brussels"),
|
||||
themes=("admin", "population"),
|
||||
native_layers=("statistical_sectors", "population_statistics"),
|
||||
geometry_types=("Polygon", "MultiPolygon", "Tabular"),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
source_url="https://statbel.fgov.be/en/open-data",
|
||||
attribution="Statbel",
|
||||
license_note="Consult the license of the selected Statbel release.",
|
||||
limitation_message="The catalog is audited, but no national bounded acquisition adapter is configured yet.",
|
||||
),
|
||||
_contract(
|
||||
source_name="digitaal_vlaanderen",
|
||||
display_name="Flemish authoritative services",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("flanders",),
|
||||
themes=(
|
||||
"buildings",
|
||||
"roads",
|
||||
"surface_water",
|
||||
"land_cover_use",
|
||||
"nature",
|
||||
"parcels",
|
||||
"soil",
|
||||
"elevation",
|
||||
"orthophoto",
|
||||
"flood_climate",
|
||||
),
|
||||
native_layers=("GRB", "BWK", "DHMV", "OMWRGBMRVL", "OGRK", "Mercator"),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||
acquisition_mode="bounded_api",
|
||||
integration_status="operational",
|
||||
source_url="https://www.vlaanderen.be/datavindplaats",
|
||||
attribution="Digitaal Vlaanderen and the authoritative Flemish source owners",
|
||||
license_note="Consult the license and attribution stored with each acquired dataset.",
|
||||
limitation_message="Operational only for bounded products implemented by GeoIntel and materialized in the project.",
|
||||
materialized_source_names=(
|
||||
"grb",
|
||||
"digitaal_vlaanderen_buildings_addresses_register",
|
||||
"digitaal_vlaanderen_dhmv",
|
||||
"digitaal_vlaanderen_orthophoto",
|
||||
"vmm_flood_hazard",
|
||||
"department_omgeving_thematic_raster",
|
||||
"inbo_bwk_natura2000",
|
||||
"dov_soil_map",
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels",
|
||||
),
|
||||
),
|
||||
_contract(
|
||||
source_name="spw_geoportail",
|
||||
display_name="SPW Geoportail Wallonie",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("wallonia",),
|
||||
themes=(
|
||||
"buildings",
|
||||
"roads",
|
||||
"surface_water",
|
||||
"land_cover_use",
|
||||
"nature",
|
||||
"soil",
|
||||
"elevation",
|
||||
"orthophoto",
|
||||
"flood_climate",
|
||||
"bathymetry",
|
||||
),
|
||||
native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
source_url="https://geoportail.wallonie.be/catalogue",
|
||||
attribution="Service public de Wallonie",
|
||||
license_note="Consult the license of each Geoportail Wallonie product.",
|
||||
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.",
|
||||
),
|
||||
_contract(
|
||||
source_name="urbis",
|
||||
display_name="UrbIS Brussels",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("brussels",),
|
||||
themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"),
|
||||
native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
source_url="https://datastore.brussels",
|
||||
attribution="Brussels UrbIS",
|
||||
license_note="Consult the license of the selected UrbIS dataset.",
|
||||
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.",
|
||||
),
|
||||
_contract(
|
||||
source_name="rbins_marine_reporting_units",
|
||||
display_name="RBINS marine reporting units",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=(
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
),
|
||||
themes=("admin", "marine_environment"),
|
||||
native_layers=("marine_reporting_units_2024",),
|
||||
geometry_types=("Polygon", "MultiPolygon"),
|
||||
acquisition_mode="operator_wfs",
|
||||
integration_status="operational",
|
||||
source_url=(
|
||||
"https://metadata.naturalsciences.be/geonetwork/srv/api/records/"
|
||||
"29f40b0d-2a3e-49a8-870a-e9b4acd4d1e3"
|
||||
),
|
||||
attribution="Royal Belgian Institute of Natural Sciences (RBINS), BMDC",
|
||||
license_note="Reuse conditions are retained from the source metadata with every persisted artifact.",
|
||||
limitation_message="The EEZ and continental shelf can share geometry while retaining different legal semantics.",
|
||||
materialized_layer_names=("marine_legal_scopes",),
|
||||
),
|
||||
_contract(
|
||||
source_name="rbins_msp_2026",
|
||||
display_name="Belgian Marine Spatial Plan 2026-2034",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=(
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
),
|
||||
themes=("maritime_planning", "marine_environment"),
|
||||
native_layers=("imsp26",),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon"),
|
||||
acquisition_mode="operator_wfs",
|
||||
integration_status="operational",
|
||||
source_url="https://www.health.belgium.be/en/themes/environment/marine-environment/marine-spatial-plan",
|
||||
attribution="Belgian federal Marine Environment service and RBINS",
|
||||
license_note="Official source metadata and attribution are retained with the imported snapshot.",
|
||||
limitation_message="The dataset represents the legally current 2026-2034 plan, not live maritime activity.",
|
||||
materialized_layer_names=("marine_spatial_plan_2026",),
|
||||
),
|
||||
_contract(
|
||||
source_name="mdk_bathymetry",
|
||||
display_name="MDK Belgian North Sea depth model",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=(
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
),
|
||||
themes=("bathymetry",),
|
||||
native_layers=("depth_model_20m_lat",),
|
||||
geometry_types=("Raster",),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
source_url="https://www.vlaanderen.be/datavindplaats",
|
||||
attribution="Agentschap Maritieme Dienstverlening en Kust (MDK)",
|
||||
license_note="Consult the official product license before acquisition.",
|
||||
limitation_message="Strict-TLS acquisition and vertical datum evidence are not yet sufficient; no depths are synthesized.",
|
||||
),
|
||||
)
|
||||
|
||||
FLANDERS_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = {
|
||||
"buildings": {
|
||||
"grb": ("buildings",),
|
||||
"digitaal_vlaanderen_buildings_addresses_register": (),
|
||||
},
|
||||
"roads": {"grb": ("roads",)},
|
||||
"surface_water": {"grb": ("water",)},
|
||||
"land_cover_use": {
|
||||
"department_omgeving_thematic_raster": (),
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels": (),
|
||||
},
|
||||
"nature": {"inbo_bwk_natura2000": ()},
|
||||
"parcels": {
|
||||
"grb": ("parcels",),
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels": (),
|
||||
},
|
||||
"soil": {"dov_soil_map": ()},
|
||||
"elevation": {"digitaal_vlaanderen_dhmv": ()},
|
||||
"orthophoto": {"digitaal_vlaanderen_orthophoto": ()},
|
||||
"flood_climate": {"vmm_flood_hazard": ()},
|
||||
}
|
||||
|
||||
|
||||
class CoverageRegistryService:
|
||||
@staticmethod
|
||||
def catalog() -> CoverageCatalogResponse:
|
||||
return CoverageCatalogResponse(
|
||||
themes=list(THEMES),
|
||||
zones=list(ZONES),
|
||||
statuses=list(STATUS_ORDER),
|
||||
sources=[definition.contract for definition in SOURCE_DEFINITIONS],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def normalize_themes(themes: Iterable[str]) -> list[str]:
|
||||
requested = list(dict.fromkeys(str(theme).strip().lower() for theme in themes if str(theme).strip()))
|
||||
invalid = sorted(set(requested) - set(THEMES))
|
||||
if invalid:
|
||||
raise AppError(
|
||||
code="COVERAGE_THEME_UNSUPPORTED",
|
||||
message="One or more coverage themes are unsupported",
|
||||
status_code=422,
|
||||
details={"unsupported_themes": invalid, "supported_themes": list(THEMES)},
|
||||
)
|
||||
return requested or list(THEMES)
|
||||
|
||||
@staticmethod
|
||||
def _geometry(value: Any):
|
||||
if value is None:
|
||||
return None
|
||||
return value if hasattr(value, "__geo_interface__") else to_shape(value)
|
||||
|
||||
@staticmethod
|
||||
def _intersected_zones(areas: list[Area], selection) -> tuple[list[str], bool]:
|
||||
geometries: dict[str, Any] = {}
|
||||
by_name = {area.name: area for area in areas}
|
||||
for zone, area_name in SCOPE_AREA_NAMES.items():
|
||||
area = by_name.get(area_name)
|
||||
geometry = CoverageRegistryService._geometry(area.geometry) if area else None
|
||||
if geometry is not None and not geometry.is_empty:
|
||||
geometries[zone] = geometry
|
||||
|
||||
detail_intersections = [
|
||||
zone for zone in DETAIL_ZONES if zone in geometries and geometries[zone].intersects(selection)
|
||||
]
|
||||
zones = detail_intersections
|
||||
if not any(zone in zones for zone in ("flanders", "wallonia", "brussels")):
|
||||
if "belgium" in geometries and geometries["belgium"].intersects(selection):
|
||||
zones = ["belgium", *zones]
|
||||
if not any(zone in zones for zone in ("territorial_sea", "exclusive_economic_zone", "continental_shelf")):
|
||||
if "belgian_north_sea" in geometries and geometries["belgian_north_sea"].intersects(selection):
|
||||
zones = [*zones, "belgian_north_sea"]
|
||||
|
||||
intersected_geometries = [geometries[zone].intersection(selection) for zone in zones if zone in geometries]
|
||||
covered = unary_union(intersected_geometries) if intersected_geometries else None
|
||||
outside = covered is None or covered.is_empty or not covered.covers(selection)
|
||||
return zones, outside
|
||||
|
||||
@staticmethod
|
||||
def _matching_datasets(
|
||||
datasets: list[Dataset],
|
||||
definition: _SourceDefinition,
|
||||
theme: str,
|
||||
zone: str,
|
||||
) -> list[Dataset]:
|
||||
matches: list[Dataset] = []
|
||||
for dataset in datasets:
|
||||
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
|
||||
continue
|
||||
layer_names = definition.materialized_layer_names
|
||||
if definition.contract.source_name == "digitaal_vlaanderen":
|
||||
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
|
||||
if dataset.source_name not in theme_sources:
|
||||
continue
|
||||
layer_names = theme_sources[dataset.source_name]
|
||||
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||
coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or []
|
||||
if isinstance(coverage_zones, str):
|
||||
coverage_zones = [coverage_zones]
|
||||
layer_matches = not layer_names or dataset.reference_layer_name in layer_names
|
||||
zone_matches = not coverage_zones or zone in coverage_zones or "belgium" in coverage_zones
|
||||
if layer_matches and zone_matches:
|
||||
matches.append(dataset)
|
||||
return matches
|
||||
|
||||
@staticmethod
|
||||
def _resolve_item(
|
||||
*,
|
||||
zone: str,
|
||||
theme: str,
|
||||
datasets: list[Dataset],
|
||||
) -> CoverageResolutionItem:
|
||||
definitions = [
|
||||
definition
|
||||
for definition in SOURCE_DEFINITIONS
|
||||
if zone in definition.contract.coverage_zones and theme in definition.contract.themes
|
||||
]
|
||||
if not definitions:
|
||||
return CoverageResolutionItem(
|
||||
zone=zone,
|
||||
theme=theme,
|
||||
status="unsupported",
|
||||
source_names=[],
|
||||
materialized_dataset_ids=[],
|
||||
limitation_message="No audited source contract supports this theme in the selected zone.",
|
||||
)
|
||||
|
||||
materialized: list[Dataset] = []
|
||||
source_statuses: list[str] = []
|
||||
limitations: list[str] = []
|
||||
for definition in definitions:
|
||||
matches = CoverageRegistryService._matching_datasets(
|
||||
datasets,
|
||||
definition,
|
||||
theme,
|
||||
zone,
|
||||
)
|
||||
materialized.extend(matches)
|
||||
if matches:
|
||||
source_statuses.append("operational")
|
||||
elif definition.contract.integration_status == "operational":
|
||||
source_statuses.append("partial")
|
||||
else:
|
||||
source_statuses.append(definition.contract.integration_status)
|
||||
limitations.append(definition.contract.limitation_message)
|
||||
|
||||
best_status = max(source_statuses, key=STATUS_RANK.__getitem__)
|
||||
return CoverageResolutionItem(
|
||||
zone=zone,
|
||||
theme=theme,
|
||||
status=best_status,
|
||||
source_names=[definition.contract.source_name for definition in definitions],
|
||||
materialized_dataset_ids=list(dict.fromkeys(dataset.id for dataset in materialized)),
|
||||
limitation_message=" ".join(dict.fromkeys(limitations)),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def resolve(
|
||||
db: Session,
|
||||
project_id: UUID,
|
||||
bbox: CoverageBBox,
|
||||
themes: Iterable[str],
|
||||
) -> CoverageResolveResponse:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
|
||||
requested_themes = CoverageRegistryService.normalize_themes(themes)
|
||||
selection = box(bbox.minx, bbox.miny, bbox.maxx, bbox.maxy)
|
||||
areas = db.query(Area).filter(Area.project_id == project_id).all()
|
||||
datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all()
|
||||
zones, outside_supported_scope = CoverageRegistryService._intersected_zones(areas, selection)
|
||||
if not zones:
|
||||
return CoverageResolveResponse(
|
||||
project_id=project_id,
|
||||
bbox=bbox,
|
||||
requested_themes=requested_themes,
|
||||
intersected_zones=[],
|
||||
outside_supported_scope=True,
|
||||
items=[],
|
||||
warnings=["The selection does not intersect a persisted Belgium or Belgian North Sea scope."],
|
||||
)
|
||||
|
||||
items = [
|
||||
CoverageRegistryService._resolve_item(zone=zone, theme=theme, datasets=datasets)
|
||||
for zone in zones
|
||||
for theme in requested_themes
|
||||
]
|
||||
warnings = []
|
||||
if outside_supported_scope:
|
||||
warnings.append("Part of the selection lies outside the persisted Belgium and Belgian North Sea scopes.")
|
||||
if len(zones) > 1:
|
||||
warnings.append(
|
||||
"The selection crosses coverage zones; results remain split and only semantically compatible metrics may be merged."
|
||||
)
|
||||
return CoverageResolveResponse(
|
||||
project_id=project_id,
|
||||
bbox=bbox,
|
||||
requested_themes=requested_themes,
|
||||
intersected_zones=zones,
|
||||
outside_supported_scope=outside_supported_scope,
|
||||
items=items,
|
||||
warnings=warnings,
|
||||
)
|
||||
@@ -0,0 +1,212 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.schemas.coverage import CoverageBBox
|
||||
from app.services.coverage_registry_service import CoverageRegistryService, THEMES, ZONES
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
|
||||
def filter(self, *_args, **_kwargs):
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return list(self.rows)
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, *, project, areas, datasets):
|
||||
self.project = project
|
||||
self.areas = areas
|
||||
self.datasets = datasets
|
||||
|
||||
def get(self, model, object_id):
|
||||
if model is Project and str(self.project.id) == str(object_id):
|
||||
return self.project
|
||||
return None
|
||||
|
||||
def query(self, model):
|
||||
if model is Area:
|
||||
return FakeQuery(self.areas)
|
||||
if model is Dataset:
|
||||
return FakeQuery(self.datasets)
|
||||
raise AssertionError(f"Unexpected query model: {model}")
|
||||
|
||||
|
||||
def scope_area(name: str, geometry):
|
||||
return SimpleNamespace(name=name, geometry=geometry)
|
||||
|
||||
|
||||
def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider_registry() -> None:
|
||||
catalog = CoverageRegistryService.catalog()
|
||||
|
||||
assert set(catalog.themes) == set(THEMES)
|
||||
assert set(catalog.zones) == set(ZONES)
|
||||
assert catalog.statuses == ["unsupported", "not_configured", "partial", "operational"]
|
||||
assert {source.source_name for source in catalog.sources} >= {
|
||||
"ngi_adminvector",
|
||||
"statbel",
|
||||
"digitaal_vlaanderen",
|
||||
"spw_geoportail",
|
||||
"urbis",
|
||||
"rbins_marine_reporting_units",
|
||||
"rbins_msp_2026",
|
||||
"mdk_bathymetry",
|
||||
}
|
||||
assert next(source for source in catalog.sources if source.source_name == "ngi_adminvector").license_note == "CC BY 4.0"
|
||||
assert next(source for source in catalog.sources if source.source_name == "mdk_bathymetry").integration_status == "not_configured"
|
||||
|
||||
response = TestClient(app).get("/api/v1/external/coverage/catalog")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["data"]["themes"] == list(THEMES)
|
||||
|
||||
|
||||
def test_coverage_resolver_only_reports_operational_for_materialized_ready_dataset() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
areas = [
|
||||
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
|
||||
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
|
||||
]
|
||||
bbox = CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0)
|
||||
|
||||
without_materialized = CoverageRegistryService.resolve(
|
||||
FakeSession(project=project, areas=areas, datasets=[]),
|
||||
project_id,
|
||||
bbox,
|
||||
["admin"],
|
||||
)
|
||||
assert without_materialized.intersected_zones == ["flanders"]
|
||||
assert without_materialized.items[0].status == "partial"
|
||||
assert without_materialized.items[0].materialized_dataset_ids == []
|
||||
|
||||
dataset_id = uuid4()
|
||||
materialized = SimpleNamespace(
|
||||
id=dataset_id,
|
||||
status="ready",
|
||||
source_name="ngi_adminvector",
|
||||
reference_layer_name="belgium_regions",
|
||||
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
|
||||
)
|
||||
with_materialized = CoverageRegistryService.resolve(
|
||||
FakeSession(project=project, areas=areas, datasets=[materialized]),
|
||||
project_id,
|
||||
bbox,
|
||||
["admin"],
|
||||
)
|
||||
admin_item = next(item for item in with_materialized.items if item.zone == "flanders")
|
||||
assert admin_item.status == "operational"
|
||||
assert admin_item.materialized_dataset_ids == [dataset_id]
|
||||
|
||||
|
||||
def test_mixed_land_and_north_sea_selection_remains_split() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
areas = [
|
||||
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
|
||||
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
|
||||
scope_area("Belgian part of the North Sea", box(2.2, 51.1, 3.4, 51.9)),
|
||||
scope_area("Belgian territorial sea (0-12 nautical miles)", box(2.7, 51.1, 3.4, 51.5)),
|
||||
scope_area("Belgian exclusive economic zone beyond territorial sea", box(2.2, 51.5, 3.1, 51.9)),
|
||||
scope_area("Belgian continental shelf beyond territorial sea", box(2.2, 51.5, 3.1, 51.9)),
|
||||
]
|
||||
|
||||
result = CoverageRegistryService.resolve(
|
||||
FakeSession(project=project, areas=areas, datasets=[]),
|
||||
project_id,
|
||||
CoverageBBox(minx=2.65, miny=51.05, maxx=2.85, maxy=51.2),
|
||||
["admin", "bathymetry"],
|
||||
)
|
||||
|
||||
assert result.intersected_zones == ["flanders", "territorial_sea"]
|
||||
assert len(result.items) == 4
|
||||
assert any("crosses coverage zones" in warning for warning in result.warnings)
|
||||
bathymetry = next(item for item in result.items if item.zone == "territorial_sea" and item.theme == "bathymetry")
|
||||
assert bathymetry.status == "not_configured"
|
||||
assert bathymetry.materialized_dataset_ids == []
|
||||
|
||||
|
||||
def test_flemish_materialization_is_theme_specific() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
orthophoto_id = uuid4()
|
||||
orthophoto = SimpleNamespace(
|
||||
id=orthophoto_id,
|
||||
status="ready",
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
reference_layer_name="orthophoto",
|
||||
source_metadata={"coverage_zones": ["flanders"]},
|
||||
)
|
||||
result = CoverageRegistryService.resolve(
|
||||
FakeSession(
|
||||
project=project,
|
||||
areas=[scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5))],
|
||||
datasets=[orthophoto],
|
||||
),
|
||||
project_id,
|
||||
CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0),
|
||||
["orthophoto", "roads"],
|
||||
)
|
||||
|
||||
assert next(item for item in result.items if item.theme == "orthophoto").status == "operational"
|
||||
assert next(item for item in result.items if item.theme == "roads").status == "partial"
|
||||
|
||||
|
||||
def test_outside_scope_and_unknown_theme_are_explicit() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
db = FakeSession(
|
||||
project=project,
|
||||
areas=[scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5))],
|
||||
datasets=[],
|
||||
)
|
||||
result = CoverageRegistryService.resolve(
|
||||
db,
|
||||
project_id,
|
||||
CoverageBBox(minx=7.0, miny=52.0, maxx=7.1, maxy=52.1),
|
||||
["admin"],
|
||||
)
|
||||
assert result.intersected_zones == []
|
||||
assert result.outside_supported_scope is True
|
||||
assert result.items == []
|
||||
|
||||
try:
|
||||
CoverageRegistryService.resolve(
|
||||
db,
|
||||
project_id,
|
||||
CoverageBBox(minx=4.0, miny=50.0, maxx=4.1, maxy=50.1),
|
||||
["invented_metric"],
|
||||
)
|
||||
except AppError as exc:
|
||||
assert exc.code == "COVERAGE_THEME_UNSUPPORTED"
|
||||
assert exc.status_code == 422
|
||||
assert exc.details["unsupported_themes"] == ["invented_metric"]
|
||||
else:
|
||||
raise AssertionError("Unknown coverage theme was accepted")
|
||||
|
||||
|
||||
def test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbox() -> None:
|
||||
root = Path(__file__).parents[2]
|
||||
focus = (root / "frontend" / "src" / "config" / "primaryFocus.ts").read_text(encoding="utf-8")
|
||||
workspace_hook = (root / "frontend" / "src" / "hooks" / "useProjectWorkspace.ts").read_text(encoding="utf-8")
|
||||
coverage_hook = (root / "frontend" / "src" / "hooks" / "useCoverageResolver.ts").read_text(encoding="utf-8")
|
||||
map_workspace = (root / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "Belgium and North Sea Workbench" in focus
|
||||
assert "nationalProject" in workspace_hook
|
||||
assert "data.areas.length > 0" in workspace_hook
|
||||
assert "dataset.status === 'ready'" in workspace_hook
|
||||
assert "externalApi.resolveCoverage" in coverage_hook
|
||||
assert "coverage.outside_supported_scope" in map_workspace
|
||||
assert "coverageStatusLabel" in map_workspace
|
||||
@@ -0,0 +1,161 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box, mapping, shape
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import provision_belgium_north_sea_scope as operator # noqa: E402
|
||||
|
||||
|
||||
def marine_feature(identifier: str, geometry):
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": identifier,
|
||||
"geometry": mapping(geometry),
|
||||
"properties": {"MarineReportingUnitId": identifier},
|
||||
}
|
||||
|
||||
|
||||
def test_marine_legal_scopes_are_derived_from_official_reporting_units() -> None:
|
||||
reporting_units = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
marine_feature("ANS-BE-MS-1", box(0, 0, 10, 10)),
|
||||
marine_feature("ANS-BE-AA-CW", box(0, 0, 2, 2)),
|
||||
marine_feature("ANS-BE-AA-TEW", box(0, 2, 2, 4)),
|
||||
marine_feature("ANS-BE-AA-OFFSHORE", box(2, 0, 10, 10)),
|
||||
],
|
||||
}
|
||||
|
||||
payload = operator.derive_marine_scope_payload(reporting_units)
|
||||
by_zone = {
|
||||
feature["properties"]["coverage_zone"]: feature
|
||||
for feature in payload["features"]
|
||||
}
|
||||
|
||||
assert set(by_zone) == {
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
}
|
||||
assert shape(by_zone["territorial_sea"]["geometry"]).area == pytest.approx(8.0)
|
||||
assert shape(by_zone["exclusive_economic_zone"]["geometry"]).equals(
|
||||
shape(by_zone["continental_shelf"]["geometry"])
|
||||
)
|
||||
assert (
|
||||
by_zone["exclusive_economic_zone"]["properties"]["legal_domain"]
|
||||
!= by_zone["continental_shelf"]["properties"]["legal_domain"]
|
||||
)
|
||||
assert by_zone["territorial_sea"]["properties"]["derived_from_reporting_unit_ids"] == [
|
||||
"ANS-BE-AA-CW",
|
||||
"ANS-BE-AA-TEW",
|
||||
]
|
||||
|
||||
|
||||
def test_marine_scope_derivation_fails_when_a_required_unit_is_missing() -> None:
|
||||
with pytest.raises(RuntimeError, match="ANS-BE-AA-CW"):
|
||||
operator.derive_marine_scope_payload(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
marine_feature("ANS-BE-MS-1", box(0, 0, 10, 10)),
|
||||
marine_feature("ANS-BE-AA-TEW", box(0, 2, 2, 4)),
|
||||
marine_feature("ANS-BE-AA-OFFSHORE", box(2, 0, 10, 10)),
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_archive_extraction_accepts_one_safe_geopackage_and_rejects_traversal(tmp_path: Path) -> None:
|
||||
archive = tmp_path / "adminvector.zip"
|
||||
with zipfile.ZipFile(archive, "w") as handle:
|
||||
handle.writestr("release/adminvector.gpkg", b"sqlite-bytes")
|
||||
|
||||
result = operator.extract_single_geopackage(archive, tmp_path / "output")
|
||||
assert result.read_bytes() == b"sqlite-bytes"
|
||||
|
||||
unsafe = tmp_path / "unsafe.zip"
|
||||
with zipfile.ZipFile(unsafe, "w") as handle:
|
||||
handle.writestr("../adminvector.gpkg", b"unsafe")
|
||||
with pytest.raises(RuntimeError, match="unsafe"):
|
||||
operator.extract_single_geopackage(unsafe, tmp_path / "unsafe-output")
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
self.content = json.dumps(payload).encode("utf-8")
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, pages):
|
||||
self.pages = list(pages)
|
||||
self.calls = []
|
||||
|
||||
def get(self, url, params, timeout):
|
||||
self.calls.append({"url": url, "params": params, "timeout": timeout})
|
||||
return FakeResponse(self.pages.pop(0))
|
||||
|
||||
|
||||
def test_wfs_fetch_is_allowlisted_paginated_and_complete() -> None:
|
||||
pages = [
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"numberMatched": 2,
|
||||
"features": [{"type": "Feature", "id": "unit.1", "geometry": None, "properties": {}}],
|
||||
},
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"numberMatched": 2,
|
||||
"features": [{"type": "Feature", "id": "unit.2", "geometry": None, "properties": {}}],
|
||||
},
|
||||
]
|
||||
session = FakeSession(pages)
|
||||
payload = operator.fetch_wfs_layer(
|
||||
session,
|
||||
service_url=operator.RBINS_MRU_WFS_URL,
|
||||
layer_name=operator.RBINS_MRU_LAYER,
|
||||
timeout=30,
|
||||
page_size=1,
|
||||
)
|
||||
|
||||
assert [feature["id"] for feature in payload["features"]] == ["unit.1", "unit.2"]
|
||||
assert [call["params"]["startIndex"] for call in session.calls] == [0, 1]
|
||||
assert all(call["params"]["srsName"] == "EPSG:4326" for call in session.calls)
|
||||
|
||||
with pytest.raises(RuntimeError, match="allowlist"):
|
||||
operator.fetch_wfs_layer(
|
||||
FakeSession([]),
|
||||
service_url=operator.RBINS_MSP_WFS_URL,
|
||||
layer_name="untrusted:layer",
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
|
||||
def test_operator_is_packaged_and_guarded_by_readiness() -> None:
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
source = (ROOT / "scripts" / "provision_belgium_north_sea_scope.py").read_text(encoding="utf-8")
|
||||
|
||||
assert "COPY scripts/provision_belgium_north_sea_scope.py /app/scripts/" in dockerfile
|
||||
assert "py_compile scripts/provision_belgium_north_sea_scope.py" in readiness
|
||||
assert "/datasets/upload" in source
|
||||
assert "from app.models" not in source
|
||||
assert "INSERT INTO vector_features" not in source
|
||||
@@ -113,7 +113,10 @@ def test_tower_deploy_uses_single_container_unraid_compose() -> None:
|
||||
for script in (powershell, bash):
|
||||
assert "docker compose -f docker-compose.unraid.yml config" in script
|
||||
assert "--build-arg GEOINTEL_INSTALL_AI=" in script
|
||||
assert "-f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest ." in script
|
||||
assert '--build-arg GEOINTEL_BUILD_SHA="$GEOINTEL_BUILD_SHA"' in script
|
||||
assert '--build-arg GEOINTEL_BUILD_TIME="$GEOINTEL_BUILD_TIME"' in script
|
||||
assert "-f deploy/unraid/Dockerfile.all-in-one" in script
|
||||
assert "-t geointel-all-in-one:latest" in script
|
||||
assert "docker compose -f docker-compose.unraid.yml build geointel" not in script
|
||||
assert "bash deploy/unraid/run-dockerman-container.sh" in script
|
||||
assert "LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh" in script
|
||||
|
||||
@@ -106,6 +106,7 @@ COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_bu
|
||||
COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py
|
||||
COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py
|
||||
COPY scripts/provision_geographic_scope.py /app/scripts/provision_geographic_scope.py
|
||||
COPY scripts/provision_belgium_north_sea_scope.py /app/scripts/provision_belgium_north_sea_scope.py
|
||||
COPY scripts/provision_regional_grb_buildings.py /app/scripts/provision_regional_grb_buildings.py
|
||||
COPY scripts/provision_regional_grb_context.py /app/scripts/provision_regional_grb_context.py
|
||||
COPY scripts/audit_source_freshness.py /app/scripts/audit_source_freshness.py
|
||||
|
||||
@@ -935,6 +935,49 @@ Read simplified job status payload.
|
||||
|
||||
## Provider registry
|
||||
|
||||
The legacy Sprint 7 provider registry remains the exact
|
||||
`grb|osm|manual|fixture` import abstraction. National and maritime source
|
||||
authority is exposed separately so adding official source families cannot
|
||||
silently change that import contract.
|
||||
|
||||
### GET `/api/v1/external/coverage/catalog`
|
||||
|
||||
Returns the normalized Belgium and Belgian North Sea theme vocabulary, legal
|
||||
coverage zones, allowed availability states and audited source contracts.
|
||||
Every source includes authority, native layers, geometry types, acquisition
|
||||
mode, source URL, attribution, licence and limitation text.
|
||||
|
||||
Allowed result states are exactly:
|
||||
|
||||
- `operational`: a matching `ready` Dataset is materialized in the project;
|
||||
- `partial`: the governed integration exists but matching project data is
|
||||
absent or incomplete;
|
||||
- `not_configured`: an audited source has no operational adapter;
|
||||
- `unsupported`: no audited contract supports the theme in that zone.
|
||||
|
||||
### POST `/api/v1/external/coverage/resolve`
|
||||
|
||||
Resolves a user-drawn EPSG:4326 bbox against persisted national and maritime
|
||||
Areas. Cross-region and land/sea selections remain split by zone.
|
||||
|
||||
```json
|
||||
{
|
||||
"project_id": "uuid",
|
||||
"bbox": {
|
||||
"minx": 2.65,
|
||||
"miny": 51.05,
|
||||
"maxx": 2.85,
|
||||
"maxy": 51.20
|
||||
},
|
||||
"themes": ["buildings", "surface_water", "bathymetry"]
|
||||
}
|
||||
```
|
||||
|
||||
The response returns `intersected_zones`, `outside_supported_scope`, one item
|
||||
per zone/theme combination, matching source names and IDs of actually
|
||||
materialized Datasets. An empty `themes` list requests the complete normalized
|
||||
vocabulary. Unknown themes fail with `COVERAGE_THEME_UNSUPPORTED`.
|
||||
|
||||
### GET `/api/v1/external/providers`
|
||||
|
||||
Returns all configured provider capability descriptors.
|
||||
|
||||
@@ -1,5 +1,31 @@
|
||||
## Autonomous RC program for Belgium and the Belgian North Sea (2026-07-17)
|
||||
|
||||
### RC-4 national and maritime coverage foundation
|
||||
|
||||
- Added an independent coverage registry so the exact Sprint 7
|
||||
`grb|osm|manual|fixture` provider contract remains unchanged.
|
||||
- Added `GET /api/v1/external/coverage/catalog` and
|
||||
`POST /api/v1/external/coverage/resolve` with canonical envelopes, normalized
|
||||
themes and explicit project-materialization checks.
|
||||
- Added `provision_belgium_north_sea_scope.py`. It accepts only the official
|
||||
NGI AdminVector archive, RBINS marine reporting units and the exact 15-layer
|
||||
`imsp26` WFS allowlist. It verifies archive/response limits, safe extraction,
|
||||
layer counts, pagination, geometries and SHA256 evidence.
|
||||
- Verified live source contracts from the Codex host: all 15 MSP layers were
|
||||
reachable and returned 101 features in total; the current RBINS reporting
|
||||
units yielded BPNS, territorial sea, EEZ and continental-shelf geometries.
|
||||
- The territorial sea is derived only by unioning official 0-1 nm and 1-12 nm
|
||||
units. EEZ and continental shelf retain the same official offshore geometry
|
||||
as separate records with different legal-domain provenance.
|
||||
- Frontend startup now prefers a materialized
|
||||
`Belgium and North Sea Workbench`; drawn bboxes receive a compact,
|
||||
zone-split coverage matrix. Existing Mol/Kempen workspaces remain available.
|
||||
- Local validation passed 986 backend tests, frontend typecheck/build, the
|
||||
full readiness gate, Alembic head `202607160001`, complete offline migration
|
||||
SQL and live-smoke shell syntax.
|
||||
- Tower fetch-only, persistence, live coverage API and browser acceptance are
|
||||
the remaining RC-4 exit evidence.
|
||||
|
||||
- Froze the RC geography as all Belgian land plus the separately labelled
|
||||
territorial sea, EEZ and continental shelf.
|
||||
- Retained Mol and the Kempen as validated regression areas instead of the
|
||||
|
||||
@@ -719,6 +719,42 @@ the canonical vector persistence path for Mol and Kempen. Additional Flemish
|
||||
vector themes still require governed regional operators; a catalogue record
|
||||
alone never makes them operational.
|
||||
|
||||
## Belgium and Belgian North Sea scope foundation
|
||||
|
||||
The release-candidate workbench uses two distinct registries:
|
||||
|
||||
- the fixed Sprint 7 import-provider registry for `grb`, `osm`, `manual` and
|
||||
`fixture`;
|
||||
- the national coverage registry for audited source authority and actual
|
||||
project materialization by theme and zone.
|
||||
|
||||
The explicit operator
|
||||
`scripts/provision_belgium_north_sea_scope.py` downloads the official NGI
|
||||
AdminVector EPSG:4326 GeoPackage archive, verifies size and archive safety,
|
||||
requires the territory, maritime zone, region, province and municipality
|
||||
layers, and retains archive/GeoPackage/artifact checksums. NGI output is
|
||||
licensed CC BY 4.0 and persists as reference Datasets through
|
||||
`DatasetService`.
|
||||
|
||||
The same operator obtains the RBINS/BMDC
|
||||
`od_nature:marine_reporting_units_2024` WFS layer. The territorial sea is the
|
||||
union of official 0-1 nm coastal waters and 1-12 nm territorial waters. The
|
||||
official offshore-beyond-12-nm geometry is persisted twice only to preserve
|
||||
the distinct legal semantics of the EEZ water column and continental-shelf
|
||||
seabed/subsoil. This is explicit provenance-preserving derivation, not
|
||||
invented geometry.
|
||||
|
||||
The legally current Belgian Marine Spatial Plan 2026-2034 is fetched from the
|
||||
fixed `imsp26` WFS allowlist and retained as one versioned reference Dataset.
|
||||
No live vessel activity, depth, water volume or other unsupported metric is
|
||||
inferred from these plan zones.
|
||||
|
||||
The coverage resolver reports a theme `operational` only when a matching
|
||||
`ready` Dataset exists in the selected project. Implemented acquisition
|
||||
without project materialization is `partial`; Walloon, Brussels, Statbel and
|
||||
MDK families remain `not_configured` until their own bounded adapters and
|
||||
source-specific metric contracts are implemented.
|
||||
|
||||
## Bathymetry, inland profiles and maritime scope
|
||||
|
||||
The official VHA Digital Atlas profile-point layer is the first operational
|
||||
|
||||
@@ -369,6 +369,31 @@ Ondersteund:
|
||||
- GPKG
|
||||
- WFS-resultaten
|
||||
|
||||
### National and maritime coverage contracts
|
||||
|
||||
Belgium-wide and maritime reference Datasets use the same Dataset and
|
||||
VectorFeature entities as every other vector source. No parallel national
|
||||
schema is introduced.
|
||||
|
||||
Required source metadata includes:
|
||||
|
||||
```yaml
|
||||
authority_level: authoritative|official_context|contextual
|
||||
coverage_zones:
|
||||
- belgium|flanders|wallonia|brussels
|
||||
- belgian_north_sea|territorial_sea|exclusive_economic_zone|continental_shelf
|
||||
source_url: https://...
|
||||
attribution: text
|
||||
license_note: text
|
||||
reference_layer_name: normalized_layer_key
|
||||
```
|
||||
|
||||
EEZ and continental-shelf records may have equal horizontal geometry while
|
||||
their `legal_domain` differs. They must remain distinct records. A selected
|
||||
bbox crossing regions or the coast produces zone-specific coverage items;
|
||||
only metrics with compatible source meaning, unit, time and method may be
|
||||
merged.
|
||||
|
||||
Metadata:
|
||||
|
||||
- CRS
|
||||
|
||||
@@ -291,6 +291,11 @@ errors.
|
||||
|
||||
## RC-4 - Belgium and North Sea coverage foundation
|
||||
|
||||
**State: in progress.** The national coverage registry, canonical
|
||||
catalog/resolve API, explicit NGI/RBINS operator, frontend selection matrix and
|
||||
focused safety tests are implemented. Local readiness is green. Tower
|
||||
fetch-only, persistence, live API and browser acceptance remain the phase exit.
|
||||
|
||||
### Work
|
||||
|
||||
1. Persist NGI authoritative Belgium land and administrative scopes.
|
||||
|
||||
@@ -43,8 +43,28 @@ storage/
|
||||
osm/
|
||||
sentinel/
|
||||
dhmv/
|
||||
operator-data/
|
||||
geographic-scopes/
|
||||
belgium-north-sea/
|
||||
adminvector_4326.zip
|
||||
adminvector/
|
||||
adminvector_4326.gpkg
|
||||
belgium_land_boundary.geojson
|
||||
belgium_regions.geojson
|
||||
belgium_provinces.geojson
|
||||
belgium_municipalities.geojson
|
||||
ngi_maritime_zone.geojson
|
||||
marine_legal_scopes.geojson
|
||||
marine_spatial_plan_2026.geojson
|
||||
manifest.json
|
||||
```
|
||||
|
||||
The Belgium/North Sea directory is written only by the explicit
|
||||
`provision_belgium_north_sea_scope.py` operator. The manifest binds every
|
||||
artifact to SHA256, source URL, edition/validity and feature count. Persistence
|
||||
still runs through the ordinary Dataset upload API; operator artifacts are
|
||||
evidence and replay inputs, not an alternate database.
|
||||
|
||||
## Upload policy
|
||||
|
||||
When a file is uploaded:
|
||||
|
||||
@@ -14,6 +14,12 @@ maritieme zones.
|
||||
- [x] RC-2: liveness/readiness/capabilities fail-closed en waarheidsgetrouw maken.
|
||||
- [x] RC-2: verweesde jobs en analysis runs na een procesherstart verzoenen.
|
||||
- [x] RC-3: expliciete rasterkeuze en temporeel compatibele detectie-QA afdwingen.
|
||||
- [x] RC-4: aparte nationale bron- en dekkingscontracten implementeren zonder
|
||||
het vaste Sprint 7-providerregister te wijzigen.
|
||||
- [x] RC-4: expliciete NGI/RBINS-operator en selectiegebonden coverage-API
|
||||
implementeren en lokaal valideren.
|
||||
- [ ] RC-4: Tower fetch-only, canonieke persistence en live browseracceptatie
|
||||
bewijzen.
|
||||
- [ ] RC-4: nationale basisdekking, Wallonie, Brussel en Belgische Noordzee via
|
||||
beheerde providers en golden areas operationaliseren.
|
||||
- [ ] RC-5: secrets/configuratie/uploadlimieten/immutable deploy en rollback
|
||||
|
||||
@@ -713,3 +713,16 @@ user selects a municipality or draws a bounded rectangle. Forest and
|
||||
agriculture use thematic raster analysis; nature value and soil use the
|
||||
persisted-vector GeoJSON pattern. The browser calls only the GeoIntel API and
|
||||
never contacts WCS, WFS or OGC providers directly.
|
||||
|
||||
## Belgium and Belgian North Sea coverage
|
||||
|
||||
When `Belgium and North Sea Workbench` exists with ready reference data, it is
|
||||
preferred over the legacy Mol/Kempen workspaces at startup. Mol and Kempen
|
||||
remain regression workspaces and can still be selected normally.
|
||||
|
||||
Drawing a rectangle on the Map invokes the GeoIntel coverage resolver after a
|
||||
short debounce. The map displays every intersected land or legal sea zone and
|
||||
the active theme as `Beschikbaar`, `Gedeeltelijk`, `Niet gekoppeld` or `Niet
|
||||
ondersteund`. A coastal or cross-region selection remains visibly split. The
|
||||
browser never calls NGI, SPW, UrbIS, RBINS or MDK directly and never promotes
|
||||
an audited catalog entry to operational data without a matching ready Dataset.
|
||||
|
||||
+21
-5
@@ -19,6 +19,7 @@ import { ProviderPanel } from './components/providers/ProviderPanel'
|
||||
import { SegmentationLab } from './components/segmentation/SegmentationLab'
|
||||
import { useChangeDetectionWorkflow } from './hooks/useChangeDetectionWorkflow'
|
||||
import { useDemoWorkflow } from './hooks/useDemoWorkflow'
|
||||
import { useCoverageResolver } from './hooks/useCoverageResolver'
|
||||
import { useDetectionWorkflow } from './hooks/useDetectionWorkflow'
|
||||
import { useDatasetWorkflow } from './hooks/useDatasetWorkflow'
|
||||
import { useExportWorkflow } from './hooks/useExportWorkflow'
|
||||
@@ -39,6 +40,8 @@ import { getDatasetDisplayName } from './lib/datasetDisplay'
|
||||
import {
|
||||
FLANDERS_WORKSPACE_LABEL,
|
||||
FLANDERS_WORKSPACE_PROJECT_NAME,
|
||||
NATIONAL_WORKSPACE_LABEL,
|
||||
NATIONAL_WORKSPACE_PROJECT_NAME,
|
||||
REGIONAL_WORKSPACE_LABEL,
|
||||
REGIONAL_WORKSPACE_PROJECT_NAME,
|
||||
} from './config/primaryFocus'
|
||||
@@ -475,6 +478,14 @@ function App(): JSX.Element {
|
||||
selectedDataset,
|
||||
isVectorDatasetType,
|
||||
})
|
||||
const {
|
||||
coverage: mapCoverage,
|
||||
loadingCoverage: mapCoverageLoading,
|
||||
coverageError: mapCoverageError,
|
||||
} = useCoverageResolver({
|
||||
projectId: selectedProjectId,
|
||||
bbox: mapSelectionBbox,
|
||||
})
|
||||
const {
|
||||
selectionDatasetSaving,
|
||||
selectionDatasetError,
|
||||
@@ -668,11 +679,13 @@ function App(): JSX.Element {
|
||||
|| Boolean(changeDetectionResult)
|
||||
|| detectionItems.length > 0
|
||||
|| segmentationItems.length > 0
|
||||
const projectContextLabel = selectedProject?.name === REGIONAL_WORKSPACE_PROJECT_NAME
|
||||
? REGIONAL_WORKSPACE_LABEL
|
||||
: selectedProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME
|
||||
? FLANDERS_WORKSPACE_LABEL
|
||||
: selectedProject?.name ?? 'Geen werkruimte'
|
||||
const projectContextLabel = selectedProject?.name === NATIONAL_WORKSPACE_PROJECT_NAME
|
||||
? NATIONAL_WORKSPACE_LABEL
|
||||
: selectedProject?.name === REGIONAL_WORKSPACE_PROJECT_NAME
|
||||
? REGIONAL_WORKSPACE_LABEL
|
||||
: selectedProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME
|
||||
? FLANDERS_WORKSPACE_LABEL
|
||||
: selectedProject?.name ?? 'Geen werkruimte'
|
||||
const areaContextLabel = selectedArea?.name ?? (areas.length > 0 ? 'Kies een gebied' : 'Geen gebied')
|
||||
const bathymetryContextActive = Boolean(
|
||||
activeWorkspace === 'map'
|
||||
@@ -927,6 +940,9 @@ function App(): JSX.Element {
|
||||
mapSelectionResult={mapSelectionResult}
|
||||
mapSelectionLoading={mapSelectionLoading}
|
||||
mapSelectionError={mapSelectionError}
|
||||
coverage={mapCoverage}
|
||||
coverageLoading={mapCoverageLoading}
|
||||
coverageError={mapCoverageError}
|
||||
selectionExporting={selectionExporting}
|
||||
selectionExportError={selectionExportError}
|
||||
latestSelectionExportPath={latestSelectionExport?.path ?? null}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
import GeoMap from '../GeoMap'
|
||||
import type { AreaRead, DatasetCreateResponse, DetectionQaResult, MapResultExportRequest, MapViewportState, OrthophotoAcquisitionResult, OrthophotoProductRead, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionMetric, VectorSelectionResponse } from '../../types'
|
||||
import type { AreaRead, CoverageResolveResponse, CoverageStatus, DatasetCreateResponse, DetectionQaResult, MapResultExportRequest, MapViewportState, OrthophotoAcquisitionResult, OrthophotoProductRead, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionMetric, VectorSelectionResponse } from '../../types'
|
||||
import { useMapThemeSelectionInsights, type MapThemeAcquisition, type MapThemeQuery } from '../../hooks/useMapThemeSelectionInsights'
|
||||
import { useOfficialMapProducts } from '../../hooks/useOfficialMapProducts'
|
||||
import { useTemporalComparison } from '../../hooks/useTemporalComparison'
|
||||
@@ -176,6 +176,49 @@ const DATA_THEMES: DataTheme[] = [
|
||||
},
|
||||
]
|
||||
|
||||
const COVERAGE_THEME_BY_MAP_THEME: Record<DataThemeId, string> = {
|
||||
buildings: 'buildings',
|
||||
space_occupation: 'land_cover_use',
|
||||
open_space: 'land_cover_use',
|
||||
population: 'population',
|
||||
forest: 'land_cover_use',
|
||||
nature_value: 'nature',
|
||||
agriculture: 'land_cover_use',
|
||||
soil: 'soil',
|
||||
water: 'surface_water',
|
||||
bathymetry: 'bathymetry',
|
||||
flood_hazard: 'flood_climate',
|
||||
elevation: 'elevation',
|
||||
accessibility: 'roads',
|
||||
services: 'population',
|
||||
roads: 'roads',
|
||||
parcels: 'parcels',
|
||||
}
|
||||
|
||||
function coverageStatusLabel(status: CoverageStatus): string {
|
||||
const labels: Record<CoverageStatus, string> = {
|
||||
operational: 'Beschikbaar',
|
||||
partial: 'Gedeeltelijk',
|
||||
not_configured: 'Niet gekoppeld',
|
||||
unsupported: 'Niet ondersteund',
|
||||
}
|
||||
return labels[status]
|
||||
}
|
||||
|
||||
function coverageZoneLabel(zone: string): string {
|
||||
const labels: Record<string, string> = {
|
||||
belgium: 'Belgie',
|
||||
flanders: 'Vlaanderen',
|
||||
wallonia: 'Wallonie',
|
||||
brussels: 'Brussel',
|
||||
belgian_north_sea: 'Belgische Noordzee',
|
||||
territorial_sea: 'Territoriale zee',
|
||||
exclusive_economic_zone: 'EEZ',
|
||||
continental_shelf: 'Continentaal plat',
|
||||
}
|
||||
return labels[zone] ?? zone
|
||||
}
|
||||
|
||||
const DATA_THEME_MAP_STYLES: Record<DataThemeId, { fill: string; line: string }> = {
|
||||
buildings: { fill: '#d45f3d', line: '#9f3e24' },
|
||||
space_occupation: { fill: '#be3e33', line: '#8f2c24' },
|
||||
@@ -516,6 +559,9 @@ interface MapWorkspaceProps {
|
||||
mapSelectionResult: VectorSelectionResponse | null
|
||||
mapSelectionLoading: boolean
|
||||
mapSelectionError: string | null
|
||||
coverage: CoverageResolveResponse | null
|
||||
coverageLoading: boolean
|
||||
coverageError: string | null
|
||||
selectionExporting: boolean
|
||||
selectionExportError: string | null
|
||||
latestSelectionExportPath: string | null
|
||||
@@ -602,6 +648,9 @@ export function MapWorkspace({
|
||||
mapSelectionResult,
|
||||
mapSelectionLoading,
|
||||
mapSelectionError,
|
||||
coverage,
|
||||
coverageLoading,
|
||||
coverageError,
|
||||
selectionExporting,
|
||||
selectionExportError,
|
||||
latestSelectionExportPath,
|
||||
@@ -819,6 +868,15 @@ export function MapWorkspace({
|
||||
[availableMapDatasets, regionalScopeSelected, selectedMapAreaId, themeDatasetMap],
|
||||
)
|
||||
const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0]
|
||||
const activeCoverageTheme = COVERAGE_THEME_BY_MAP_THEME[activeTheme.id]
|
||||
const activeCoverageItems = coverage?.items.filter((item) => item.theme === activeCoverageTheme) ?? []
|
||||
const coverageCounts = useMemo(
|
||||
() => coverage?.items.reduce<Record<CoverageStatus, number>>(
|
||||
(counts, item) => ({ ...counts, [item.status]: counts[item.status] + 1 }),
|
||||
{ operational: 0, partial: 0, not_configured: 0, unsupported: 0 },
|
||||
) ?? { operational: 0, partial: 0, not_configured: 0, unsupported: 0 },
|
||||
[coverage],
|
||||
)
|
||||
const onDemandProductMap = useMemo(
|
||||
() => {
|
||||
const result = new Map<DataThemeId, OnDemandMapProduct>()
|
||||
@@ -2527,6 +2585,50 @@ export function MapWorkspace({
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{mapSelectionBbox ? (
|
||||
<section className="coverage-resolution-surface" aria-label="Datadekking van de kaartselectie">
|
||||
<div className="panel-title-row">
|
||||
<div>
|
||||
<p className="eyebrow">Dekking van deze selectie</p>
|
||||
<h3>{activeTheme.label}</h3>
|
||||
</div>
|
||||
{coverageLoading ? <span className="count-pill">controleren</span> : null}
|
||||
</div>
|
||||
{coverageError ? <p className="error">{coverageError}</p> : null}
|
||||
{coverage ? (
|
||||
<>
|
||||
<div className="coverage-zone-row">
|
||||
{coverage.intersected_zones.map((zone) => (
|
||||
<span key={zone}>{coverageZoneLabel(zone)}</span>
|
||||
))}
|
||||
{coverage.outside_supported_scope ? <span className="coverage-zone-warning">deels buiten scope</span> : null}
|
||||
</div>
|
||||
<div className="coverage-active-theme-grid">
|
||||
{activeCoverageItems.map((item) => (
|
||||
<div className={`coverage-status-item coverage-status-${item.status}`} key={`${item.zone}:${item.theme}`}>
|
||||
<span>{coverageZoneLabel(item.zone)}</span>
|
||||
<strong>{coverageStatusLabel(item.status)}</strong>
|
||||
<small>{item.source_names.join(', ') || 'Geen broncontract'}</small>
|
||||
</div>
|
||||
))}
|
||||
{activeCoverageItems.length === 0 ? (
|
||||
<p className="muted">Deze selectie raakt geen bewaarde Belgische land- of zeezone.</p>
|
||||
) : null}
|
||||
</div>
|
||||
<div className="coverage-summary-row" aria-label="Samenvatting van alle themas">
|
||||
<span>{coverageCounts.operational} beschikbaar</span>
|
||||
<span>{coverageCounts.partial} gedeeltelijk</span>
|
||||
<span>{coverageCounts.not_configured} niet gekoppeld</span>
|
||||
<span>{coverageCounts.unsupported} niet ondersteund</span>
|
||||
</div>
|
||||
{coverage.warnings.map((warning) => <p className="muted" key={warning}>{warning}</p>)}
|
||||
</>
|
||||
) : !coverageLoading && !coverageError ? (
|
||||
<p className="muted">De dekkingsmatrix wordt bepaald zodra de selectie volledig is.</p>
|
||||
) : null}
|
||||
</section>
|
||||
) : null}
|
||||
|
||||
<div className="map-inspection-surface">
|
||||
<div className="gis-test-run-surface" aria-label="Operationele GIS-controle">
|
||||
<div className="panel-title-row">
|
||||
|
||||
@@ -2,6 +2,8 @@ import type { DatasetCreateResponse, ProjectRead } from '../types'
|
||||
|
||||
export const PRIMARY_FOCUS_LABEL = 'Mol'
|
||||
export const PRIMARY_FOCUS_REGION = 'Mol, Kempen'
|
||||
export const NATIONAL_WORKSPACE_PROJECT_NAME = 'Belgium and North Sea Workbench'
|
||||
export const NATIONAL_WORKSPACE_LABEL = 'Belgie en Belgische Noordzee'
|
||||
export const REGIONAL_WORKSPACE_PROJECT_NAME = 'Kempen Regional Workbench'
|
||||
export const REGIONAL_WORKSPACE_LABEL = 'Kempen (28 gemeenten)'
|
||||
export const FLANDERS_WORKSPACE_PROJECT_NAME = 'Flanders Regional Workbench'
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { externalApi } from '../services/api'
|
||||
import type { CoverageResolveResponse, VectorSelectionBBox } from '../types'
|
||||
|
||||
interface CoverageResolverOptions {
|
||||
projectId: string | null
|
||||
bbox: VectorSelectionBBox | null
|
||||
}
|
||||
|
||||
export function useCoverageResolver({ projectId, bbox }: CoverageResolverOptions) {
|
||||
const [coverage, setCoverage] = useState<CoverageResolveResponse | null>(null)
|
||||
const [loadingCoverage, setLoadingCoverage] = useState(false)
|
||||
const [coverageError, setCoverageError] = useState<string | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
if (!projectId || !bbox) {
|
||||
setCoverage(null)
|
||||
setCoverageError(null)
|
||||
setLoadingCoverage(false)
|
||||
return
|
||||
}
|
||||
let cancelled = false
|
||||
const timer = window.setTimeout(() => {
|
||||
setLoadingCoverage(true)
|
||||
setCoverageError(null)
|
||||
externalApi.resolveCoverage({
|
||||
projectId,
|
||||
bbox: {
|
||||
minx: bbox.min_x,
|
||||
miny: bbox.min_y,
|
||||
maxx: bbox.max_x,
|
||||
maxy: bbox.max_y,
|
||||
},
|
||||
})
|
||||
.then((result) => {
|
||||
if (!cancelled) {
|
||||
setCoverage(result)
|
||||
}
|
||||
})
|
||||
.catch((error) => {
|
||||
if (!cancelled) {
|
||||
setCoverage(null)
|
||||
setCoverageError(error instanceof Error ? error.message : 'Dekking kon niet worden bepaald')
|
||||
}
|
||||
})
|
||||
.finally(() => {
|
||||
if (!cancelled) {
|
||||
setLoadingCoverage(false)
|
||||
}
|
||||
})
|
||||
}, 250)
|
||||
return () => {
|
||||
cancelled = true
|
||||
window.clearTimeout(timer)
|
||||
}
|
||||
}, [projectId, bbox?.min_x, bbox?.min_y, bbox?.max_x, bbox?.max_y])
|
||||
|
||||
return {
|
||||
coverage,
|
||||
loadingCoverage,
|
||||
coverageError,
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
PRIMARY_FOCUS_AREA_GEOJSON,
|
||||
PRIMARY_FOCUS_AREA_NAME,
|
||||
PRIMARY_FOCUS_REGION,
|
||||
NATIONAL_WORKSPACE_PROJECT_NAME,
|
||||
REGIONAL_WORKSPACE_PROJECT_NAME,
|
||||
isPrimaryFocusMunicipalityBoundaryDataset,
|
||||
isPrimaryFocusMunicipalityProject,
|
||||
@@ -81,6 +82,17 @@ export function useProjectWorkspace() {
|
||||
if (selectedProjectId && items.some((project) => project.id === selectedProjectId)) {
|
||||
return selectedProjectId
|
||||
}
|
||||
const nationalProject = items.find((project) => project.name === NATIONAL_WORKSPACE_PROJECT_NAME)
|
||||
if (nationalProject) {
|
||||
try {
|
||||
const data = await fetchProjectData(nationalProject.id)
|
||||
if (data.areas.length > 0 && data.datasets.some((dataset) => dataset.status === 'ready')) {
|
||||
return nationalProject.id
|
||||
}
|
||||
} catch {
|
||||
// Continue with regional and municipality fallbacks while national data is unavailable.
|
||||
}
|
||||
}
|
||||
const regionalProject = items.find((project) => project.name === REGIONAL_WORKSPACE_PROJECT_NAME)
|
||||
if (regionalProject) {
|
||||
try {
|
||||
@@ -145,13 +157,15 @@ export function useProjectWorkspace() {
|
||||
setLoadingProjects(true)
|
||||
setErrorMessage(null)
|
||||
try {
|
||||
const [response, canonicalResponse] = await Promise.all([
|
||||
const [response, nationalResponse, canonicalResponse] = await Promise.all([
|
||||
projectsApi.list(),
|
||||
projectsApi.list({ name: NATIONAL_WORKSPACE_PROJECT_NAME, limit: 1 }),
|
||||
projectsApi.list({ name: REGIONAL_WORKSPACE_PROJECT_NAME, limit: 1 }),
|
||||
])
|
||||
const regionalItems = [...canonicalResponse.items, ...response.items]
|
||||
const items = Array.from(
|
||||
new Map(
|
||||
[...canonicalResponse.items, ...response.items].map((project) => [project.id, project]),
|
||||
[...nationalResponse.items, ...regionalItems].map((project) => [project.id, project]),
|
||||
).values(),
|
||||
)
|
||||
setProjects(items)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { apiGet, apiPost } from './client'
|
||||
import type {
|
||||
CoverageCatalogResponse,
|
||||
CoverageResolveResponse,
|
||||
ProviderCapability,
|
||||
ProviderCapabilitiesResponse,
|
||||
ProviderImportResponse,
|
||||
@@ -13,6 +15,18 @@ const normalize = (layers: string[] = []) => layers.filter((value) => value.trim
|
||||
export const externalApi = {
|
||||
listSystemCapabilities: (): Promise<SystemCapabilitiesResponse> =>
|
||||
apiGet<SystemCapabilitiesResponse>('/api/v1/system/capabilities'),
|
||||
getCoverageCatalog: (): Promise<CoverageCatalogResponse> =>
|
||||
apiGet<CoverageCatalogResponse>('/api/v1/external/coverage/catalog'),
|
||||
resolveCoverage: (payload: {
|
||||
projectId: string
|
||||
bbox: { minx: number; miny: number; maxx: number; maxy: number }
|
||||
themes?: string[]
|
||||
}): Promise<CoverageResolveResponse> =>
|
||||
apiPost<CoverageResolveResponse>('/api/v1/external/coverage/resolve', {
|
||||
project_id: payload.projectId,
|
||||
bbox: payload.bbox,
|
||||
themes: payload.themes ?? [],
|
||||
}),
|
||||
listProviders: (): Promise<ProviderCapabilitiesResponse> =>
|
||||
apiGet<ProviderCapabilitiesResponse>('/api/v1/external/providers'),
|
||||
listProviderCapabilities: (): Promise<ProviderCapabilitiesResponse> =>
|
||||
|
||||
@@ -7347,3 +7347,67 @@ section {
|
||||
.workspace-persistent-map[hidden] {
|
||||
display: none;
|
||||
}
|
||||
.coverage-resolution-surface {
|
||||
padding: 18px 20px;
|
||||
border-block: 1px solid var(--line);
|
||||
background: color-mix(in srgb, var(--surface-raised) 94%, var(--accent-soft));
|
||||
}
|
||||
|
||||
.coverage-zone-row,
|
||||
.coverage-summary-row {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.coverage-zone-row span,
|
||||
.coverage-summary-row span {
|
||||
padding: 4px 8px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 4px;
|
||||
background: var(--surface-raised);
|
||||
color: var(--muted);
|
||||
font-size: 0.78rem;
|
||||
}
|
||||
|
||||
.coverage-zone-row .coverage-zone-warning {
|
||||
border-color: var(--warning);
|
||||
color: var(--warning);
|
||||
}
|
||||
|
||||
.coverage-active-theme-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(170px, 1fr));
|
||||
gap: 8px;
|
||||
margin-top: 12px;
|
||||
}
|
||||
|
||||
.coverage-status-item {
|
||||
display: grid;
|
||||
gap: 3px;
|
||||
min-height: 76px;
|
||||
padding: 10px 12px;
|
||||
border: 1px solid var(--line);
|
||||
border-left: 3px solid var(--muted);
|
||||
border-radius: 6px;
|
||||
background: var(--surface-raised);
|
||||
}
|
||||
|
||||
.coverage-status-item span,
|
||||
.coverage-status-item small {
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.coverage-status-operational {
|
||||
border-left-color: var(--accent);
|
||||
}
|
||||
|
||||
.coverage-status-partial {
|
||||
border-left-color: var(--warning);
|
||||
}
|
||||
|
||||
.coverage-status-not_configured,
|
||||
.coverage-status-unsupported {
|
||||
border-left-color: var(--muted);
|
||||
}
|
||||
|
||||
@@ -957,6 +957,57 @@ export interface ProviderCapability {
|
||||
not_configured_reason: string | null
|
||||
}
|
||||
|
||||
export type CoverageStatus = 'operational' | 'partial' | 'not_configured' | 'unsupported'
|
||||
|
||||
export interface CoverageBBox {
|
||||
minx: number
|
||||
miny: number
|
||||
maxx: number
|
||||
maxy: number
|
||||
}
|
||||
|
||||
export interface CoverageSourceContract {
|
||||
source_name: string
|
||||
display_name: string
|
||||
authority_level: 'authoritative' | 'official_context' | 'contextual'
|
||||
coverage_zones: string[]
|
||||
themes: string[]
|
||||
native_layers: string[]
|
||||
supported_geometry_types: string[]
|
||||
acquisition_mode: 'operator_archive' | 'operator_wfs' | 'bounded_api' | 'bounded_raster' | 'catalog_only'
|
||||
integration_status: CoverageStatus
|
||||
source_url: string
|
||||
attribution: string
|
||||
license_note: string
|
||||
limitation_message: string
|
||||
}
|
||||
|
||||
export interface CoverageCatalogResponse {
|
||||
themes: string[]
|
||||
zones: string[]
|
||||
statuses: CoverageStatus[]
|
||||
sources: CoverageSourceContract[]
|
||||
}
|
||||
|
||||
export interface CoverageResolutionItem {
|
||||
zone: string
|
||||
theme: string
|
||||
status: CoverageStatus
|
||||
source_names: string[]
|
||||
materialized_dataset_ids: string[]
|
||||
limitation_message: string
|
||||
}
|
||||
|
||||
export interface CoverageResolveResponse {
|
||||
project_id: string
|
||||
bbox: CoverageBBox
|
||||
requested_themes: string[]
|
||||
intersected_zones: string[]
|
||||
outside_supported_scope: boolean
|
||||
items: CoverageResolutionItem[]
|
||||
warnings: string[]
|
||||
}
|
||||
|
||||
export interface SystemCapabilitiesResponse {
|
||||
status: string
|
||||
service: string
|
||||
|
||||
@@ -1319,6 +1319,33 @@ audits artifacts without uploading; `--force` explicitly refreshes the source
|
||||
partitions. Historical identities are declared unstable and support hectare
|
||||
comparison only, not object lineage.
|
||||
|
||||
## Belgium and Belgian North Sea foundation
|
||||
|
||||
Prepare and validate the complete national/maritime source snapshot without
|
||||
changing application persistence:
|
||||
|
||||
```bash
|
||||
docker exec -it geointel python3 /app/scripts/provision_belgium_north_sea_scope.py \
|
||||
--fetch-only
|
||||
```
|
||||
|
||||
Persist the national workbench through the canonical API:
|
||||
|
||||
```bash
|
||||
docker exec -it geointel python3 /app/scripts/provision_belgium_north_sea_scope.py
|
||||
```
|
||||
|
||||
Use `--force` only for an explicit source refresh. The operator creates or
|
||||
reuses `Belgium and North Sea Workbench`, eight legal/administrative Areas and
|
||||
six reference Datasets. It never runs on container startup, never disables
|
||||
TLS verification and never writes directly to PostGIS. Output and immutable
|
||||
checksums are retained below
|
||||
`/app/storage/operator-data/geographic-scopes/belgium-north-sea`.
|
||||
|
||||
The command imports the common administrative baseline and marine legal/use
|
||||
zones. It does not make detailed Walloon, Brussels, population, bathymetry or
|
||||
other audited source families operational.
|
||||
|
||||
## Official Kempen operational scope
|
||||
|
||||
GeoIntel defines its regional `Kempen` workspace as the official Vlaamse
|
||||
|
||||
@@ -0,0 +1,950 @@
|
||||
"""Provision Belgium and the Belgian North Sea through the canonical API.
|
||||
|
||||
This explicit operator downloads a bounded, allowlisted set of authoritative
|
||||
sources. It never runs at application startup and never writes directly to
|
||||
PostGIS or vector_features.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
|
||||
from shapely.ops import unary_union
|
||||
from shapely.validation import make_valid
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
|
||||
PROJECT_NAME = "Belgium and North Sea Workbench"
|
||||
PROJECT_REGION = "Belgium and Belgian North Sea"
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes/belgium-north-sea")
|
||||
NGI_ARCHIVE_URL = (
|
||||
"https://ac.ngi.be/remoteclient-open/ngi-standard-open/Vectordata/"
|
||||
"TerritorialDivisions/TerritorialDivisions-AdminVector/"
|
||||
"fb1e2993-2020-428c-9188-eb5f75e284b9_geopackage+sqlite3_4326.zip"
|
||||
)
|
||||
NGI_CATALOG_URL = "https://www.geo.be/catalog/details/fb1e2993-2020-428c-9188-eb5f75e284b9"
|
||||
NGI_ATTRIBUTION = "National Geographic Institute (NGI), AdminVector"
|
||||
NGI_LICENSE = "CC BY 4.0"
|
||||
RBINS_MRU_WFS_URL = "https://spatial.naturalsciences.be/geoserver/od_nature/ows"
|
||||
RBINS_MRU_METADATA_URL = (
|
||||
"https://metadata.naturalsciences.be/geonetwork/srv/api/records/"
|
||||
"29f40b0d-2a3e-49a8-870a-e9b4acd4d1e3"
|
||||
)
|
||||
RBINS_MRU_LAYER = "od_nature:marine_reporting_units_2024"
|
||||
RBINS_MSP_WFS_URL = "https://spatial.naturalsciences.be/geoserver/ows"
|
||||
RBINS_MSP_SOURCE_URL = (
|
||||
"https://www.health.belgium.be/en/themes/environment/marine-environment/marine-spatial-plan"
|
||||
)
|
||||
MSP_VALID_FROM = "2026-03-20T00:00:00Z"
|
||||
MAX_ARCHIVE_BYTES = 160 * 1024 * 1024
|
||||
MAX_EXTRACTED_BYTES = 1024 * 1024 * 1024
|
||||
MAX_WFS_RESPONSE_BYTES = 64 * 1024 * 1024
|
||||
WFS_PAGE_SIZE = 5000
|
||||
GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}}
|
||||
|
||||
ADMIN_LAYERS = {
|
||||
"belgianterritory": (1, 1),
|
||||
"belgianmaritimezone": (1, 1),
|
||||
"region": (3, 3),
|
||||
"province": (11, 11),
|
||||
"municipality": (560, 590),
|
||||
}
|
||||
|
||||
MSP_LAYERS = (
|
||||
"imsp26:bmsp_aquaculture_zone",
|
||||
"imsp26:bmsp_coastal_protection_experiment_zone",
|
||||
"imsp26:bmsp_commercial_industrial_zone",
|
||||
"imsp26:bmsp_cultural_heritage",
|
||||
"imsp26:bmsp_dredging_zone",
|
||||
"imsp26:bmsp_energy_cables_pipelines_zone",
|
||||
"imsp26:bmsp_fisheries_zone",
|
||||
"imsp26:bmsp_conservation_zone",
|
||||
"imsp26:bmsp_measuring_poles",
|
||||
"imsp26:bmsp_military_zone",
|
||||
"imsp26:bmsp_port_expansion_zone",
|
||||
"imsp26:bmsp_radar_towers",
|
||||
"imsp26:bmsp_research_recreation_zone",
|
||||
"imsp26:bmsp_extraction_zone",
|
||||
"imsp26:bmsp_shipping_ports_zone",
|
||||
)
|
||||
|
||||
MARINE_REPORTING_IDS = {
|
||||
"belgian_north_sea": "ANS-BE-MS-1",
|
||||
"territorial_1_12": "ANS-BE-AA-TEW",
|
||||
"coastal_0_1": "ANS-BE-AA-CW",
|
||||
"offshore": "ANS-BE-AA-OFFSHORE",
|
||||
}
|
||||
|
||||
AREA_NAMES = {
|
||||
"belgium": "Belgium land",
|
||||
"flanders": "Flanders",
|
||||
"wallonia": "Wallonia",
|
||||
"brussels": "Brussels-Capital Region",
|
||||
"belgian_north_sea": "Belgian part of the North Sea",
|
||||
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
|
||||
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
|
||||
"continental_shelf": "Belgian continental shelf beyond territorial sea",
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision the Belgium and Belgian North Sea scope.")
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_NATIONAL_SCOPE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
|
||||
)
|
||||
parser.add_argument("--request-timeout", type=int, default=300)
|
||||
parser.add_argument("--import-timeout", type=int, default=3600)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument("--fetch-only", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_json_atomic(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(f"{path.suffix}.partial")
|
||||
temporary.write_text(
|
||||
json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
indent=2 if pretty else None,
|
||||
separators=None if pretty else (",", ":"),
|
||||
sort_keys=pretty,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def build_session() -> requests.Session:
|
||||
retry = Retry(
|
||||
total=5,
|
||||
connect=5,
|
||||
read=5,
|
||||
status=5,
|
||||
backoff_factor=1.0,
|
||||
status_forcelist=(429, 500, 502, 503, 504),
|
||||
allowed_methods=frozenset({"GET"}),
|
||||
raise_on_status=True,
|
||||
)
|
||||
session = requests.Session()
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
session.headers.update({"User-Agent": "GeoIntel-Belgium-North-Sea-Operator/1.0"})
|
||||
return session
|
||||
|
||||
|
||||
def download_limited(session: requests.Session, url: str, target: Path, timeout: int, max_bytes: int) -> Path:
|
||||
if url != NGI_ARCHIVE_URL:
|
||||
raise RuntimeError("Archive URL is not in the fixed operator allowlist")
|
||||
temporary = target.with_suffix(f"{target.suffix}.partial")
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
size = 0
|
||||
with session.get(url, stream=True, timeout=timeout) as response:
|
||||
response.raise_for_status()
|
||||
content_length = int(response.headers.get("content-length") or 0)
|
||||
if content_length > max_bytes:
|
||||
raise RuntimeError(f"NGI archive declares {content_length} bytes, above the {max_bytes} byte limit")
|
||||
with temporary.open("wb") as handle:
|
||||
for chunk in response.iter_content(chunk_size=1024 * 1024):
|
||||
if not chunk:
|
||||
continue
|
||||
size += len(chunk)
|
||||
if size > max_bytes:
|
||||
raise RuntimeError(f"NGI archive exceeded the {max_bytes} byte limit")
|
||||
handle.write(chunk)
|
||||
if size == 0:
|
||||
raise RuntimeError("NGI archive download was empty")
|
||||
temporary.replace(target)
|
||||
return target
|
||||
|
||||
|
||||
def extract_single_geopackage(archive_path: Path, output_dir: Path) -> Path:
|
||||
extraction_root = output_dir / "adminvector"
|
||||
temporary_root = output_dir / "adminvector.partial"
|
||||
if temporary_root.exists():
|
||||
shutil.rmtree(temporary_root)
|
||||
temporary_root.mkdir(parents=True)
|
||||
with zipfile.ZipFile(archive_path) as archive:
|
||||
members = [member for member in archive.infolist() if not member.is_dir()]
|
||||
gpkg_members = [member for member in members if Path(member.filename).suffix.lower() == ".gpkg"]
|
||||
if len(gpkg_members) != 1:
|
||||
raise RuntimeError(f"Expected one GeoPackage in NGI archive, received {len(gpkg_members)}")
|
||||
member = gpkg_members[0]
|
||||
member_path = Path(member.filename)
|
||||
if member_path.is_absolute() or ".." in member_path.parts:
|
||||
raise RuntimeError("NGI archive contains an unsafe GeoPackage path")
|
||||
if member.file_size <= 0 or member.file_size > MAX_EXTRACTED_BYTES:
|
||||
raise RuntimeError("NGI GeoPackage size is empty or above the extraction limit")
|
||||
target = temporary_root / "adminvector_4326.gpkg"
|
||||
with archive.open(member) as source, target.open("wb") as destination:
|
||||
shutil.copyfileobj(source, destination, length=1024 * 1024)
|
||||
if extraction_root.exists():
|
||||
shutil.rmtree(extraction_root)
|
||||
temporary_root.replace(extraction_root)
|
||||
return extraction_root / "adminvector_4326.gpkg"
|
||||
|
||||
|
||||
def _polygonal(geometry):
|
||||
if geometry is None or geometry.is_empty:
|
||||
return None
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
if isinstance(geometry, (Polygon, MultiPolygon)):
|
||||
return geometry
|
||||
if isinstance(geometry, GeometryCollection):
|
||||
parts = [part for part in geometry.geoms if isinstance(part, (Polygon, MultiPolygon))]
|
||||
merged = unary_union(parts) if parts else None
|
||||
if isinstance(merged, (Polygon, MultiPolygon)) and not merged.is_empty:
|
||||
return merged
|
||||
return None
|
||||
|
||||
|
||||
def read_adminvector_layers(gpkg_path: Path) -> dict[str, dict[str, Any]]:
|
||||
try:
|
||||
import geopandas
|
||||
import pandas
|
||||
import pyogrio
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("GeoPandas and Pyogrio are required for the national scope operator") from exc
|
||||
|
||||
available_layers = {str(row[0]) for row in pyogrio.list_layers(gpkg_path)}
|
||||
missing = sorted(set(ADMIN_LAYERS) - available_layers)
|
||||
if missing:
|
||||
raise RuntimeError(f"NGI AdminVector is missing expected layers: {', '.join(missing)}")
|
||||
|
||||
payloads: dict[str, dict[str, Any]] = {}
|
||||
for layer_name, (minimum, maximum) in ADMIN_LAYERS.items():
|
||||
frame = geopandas.read_file(gpkg_path, layer=layer_name)
|
||||
if frame.crs is None:
|
||||
raise RuntimeError(f"NGI layer {layer_name} has no CRS")
|
||||
frame = frame.to_crs(4326)
|
||||
if not minimum <= len(frame) <= maximum:
|
||||
raise RuntimeError(
|
||||
f"NGI layer {layer_name} has {len(frame)} records; expected between {minimum} and {maximum}"
|
||||
)
|
||||
payload = json.loads(frame.to_json(drop_id=False))
|
||||
source_max_modification = None
|
||||
if "modifdate" in frame.columns:
|
||||
parsed = pandas.to_datetime(frame["modifdate"], errors="coerce", utc=True).dropna()
|
||||
if not parsed.empty:
|
||||
source_max_modification = parsed.max().isoformat()
|
||||
payload.update(
|
||||
{
|
||||
"name": f"NGI AdminVector {layer_name}",
|
||||
"crs": GEOJSON_CRS,
|
||||
"source_url": NGI_CATALOG_URL,
|
||||
"attribution": NGI_ATTRIBUTION,
|
||||
"license": NGI_LICENSE,
|
||||
"source_max_modification": source_max_modification,
|
||||
}
|
||||
)
|
||||
for feature in payload["features"]:
|
||||
properties = feature.setdefault("properties", {})
|
||||
properties.update(
|
||||
{
|
||||
"source_name": "ngi_adminvector",
|
||||
"source_layer": layer_name,
|
||||
"source_feature_id": str(feature.get("id") or properties.get("tgid") or ""),
|
||||
"authority_level": "authoritative",
|
||||
"attribution": NGI_ATTRIBUTION,
|
||||
"source_url": NGI_CATALOG_URL,
|
||||
}
|
||||
)
|
||||
payloads[layer_name] = payload
|
||||
return payloads
|
||||
|
||||
|
||||
def _response_json_limited(response: requests.Response, max_bytes: int) -> dict[str, Any]:
|
||||
response.raise_for_status()
|
||||
content = response.content
|
||||
if len(content) > max_bytes:
|
||||
raise RuntimeError(f"WFS response exceeded the {max_bytes} byte limit")
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("WFS returned non-JSON content") from exc
|
||||
if payload.get("type") != "FeatureCollection" or not isinstance(payload.get("features"), list):
|
||||
raise RuntimeError("WFS response is not a GeoJSON FeatureCollection")
|
||||
return payload
|
||||
|
||||
|
||||
def fetch_wfs_layer(
|
||||
session: requests.Session,
|
||||
*,
|
||||
service_url: str,
|
||||
layer_name: str,
|
||||
timeout: int,
|
||||
page_size: int = WFS_PAGE_SIZE,
|
||||
) -> dict[str, Any]:
|
||||
if (service_url, layer_name) not in {
|
||||
(RBINS_MRU_WFS_URL, RBINS_MRU_LAYER),
|
||||
*((RBINS_MSP_WFS_URL, layer) for layer in MSP_LAYERS),
|
||||
}:
|
||||
raise RuntimeError(f"WFS layer is not in the fixed operator allowlist: {layer_name}")
|
||||
features: list[dict[str, Any]] = []
|
||||
source_ids: set[str] = set()
|
||||
start_index = 0
|
||||
matched: int | None = None
|
||||
while True:
|
||||
response = session.get(
|
||||
service_url,
|
||||
params={
|
||||
"service": "WFS",
|
||||
"version": "2.0.0",
|
||||
"request": "GetFeature",
|
||||
"typeNames": layer_name,
|
||||
"outputFormat": "application/json",
|
||||
"srsName": "EPSG:4326",
|
||||
"count": page_size,
|
||||
"startIndex": start_index,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
payload = _response_json_limited(response, MAX_WFS_RESPONSE_BYTES)
|
||||
page = payload["features"]
|
||||
if matched is None and str(payload.get("numberMatched", "")).isdigit():
|
||||
matched = int(payload["numberMatched"])
|
||||
for feature in page:
|
||||
source_id = str(feature.get("id") or "")
|
||||
if source_id and source_id in source_ids:
|
||||
raise RuntimeError(f"WFS layer {layer_name} returned duplicate feature id {source_id}")
|
||||
if source_id:
|
||||
source_ids.add(source_id)
|
||||
features.append(feature)
|
||||
if not page or len(page) < page_size or (matched is not None and len(features) >= matched):
|
||||
break
|
||||
start_index += len(page)
|
||||
if matched is not None and len(features) != matched:
|
||||
raise RuntimeError(f"WFS layer {layer_name} returned {len(features)} of {matched} matched features")
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"name": layer_name,
|
||||
"crs": GEOJSON_CRS,
|
||||
"features": features,
|
||||
}
|
||||
|
||||
|
||||
def _reporting_id(feature: dict[str, Any]) -> str:
|
||||
properties = feature.get("properties") or {}
|
||||
return str(
|
||||
properties.get("MarineReportingUnitId")
|
||||
or properties.get("marineReportingUnitId")
|
||||
or properties.get("localId")
|
||||
or ""
|
||||
)
|
||||
|
||||
|
||||
def derive_marine_scope_payload(reporting_units: dict[str, Any]) -> dict[str, Any]:
|
||||
by_id = {_reporting_id(feature): feature for feature in reporting_units.get("features") or []}
|
||||
missing = [identifier for identifier in MARINE_REPORTING_IDS.values() if identifier not in by_id]
|
||||
if missing:
|
||||
raise RuntimeError(f"Marine reporting units are missing required identifiers: {', '.join(missing)}")
|
||||
|
||||
def geometry(identifier: str):
|
||||
result = _polygonal(shape(by_id[identifier].get("geometry")))
|
||||
if result is None:
|
||||
raise RuntimeError(f"Marine reporting unit {identifier} has no valid polygon geometry")
|
||||
return result
|
||||
|
||||
bpns = geometry(MARINE_REPORTING_IDS["belgian_north_sea"])
|
||||
territorial = _polygonal(
|
||||
unary_union(
|
||||
(
|
||||
geometry(MARINE_REPORTING_IDS["territorial_1_12"]),
|
||||
geometry(MARINE_REPORTING_IDS["coastal_0_1"]),
|
||||
)
|
||||
)
|
||||
)
|
||||
offshore = geometry(MARINE_REPORTING_IDS["offshore"])
|
||||
if territorial is None or not bpns.covers(territorial.representative_point()) or not bpns.covers(offshore.representative_point()):
|
||||
raise RuntimeError("Derived marine legal scopes do not align with the official BPNS reporting unit")
|
||||
|
||||
definitions = (
|
||||
(
|
||||
"belgian_north_sea",
|
||||
AREA_NAMES["belgian_north_sea"],
|
||||
bpns,
|
||||
"marine_water_and_seabed_scope",
|
||||
[MARINE_REPORTING_IDS["belgian_north_sea"]],
|
||||
),
|
||||
(
|
||||
"territorial_sea",
|
||||
AREA_NAMES["territorial_sea"],
|
||||
territorial,
|
||||
"territorial_water_column_and_seabed",
|
||||
[MARINE_REPORTING_IDS["coastal_0_1"], MARINE_REPORTING_IDS["territorial_1_12"]],
|
||||
),
|
||||
(
|
||||
"exclusive_economic_zone",
|
||||
AREA_NAMES["exclusive_economic_zone"],
|
||||
offshore,
|
||||
"water_column_rights_beyond_territorial_sea",
|
||||
[MARINE_REPORTING_IDS["offshore"]],
|
||||
),
|
||||
(
|
||||
"continental_shelf",
|
||||
AREA_NAMES["continental_shelf"],
|
||||
offshore,
|
||||
"seabed_and_subsoil_rights_beyond_territorial_sea",
|
||||
[MARINE_REPORTING_IDS["offshore"]],
|
||||
),
|
||||
)
|
||||
features = []
|
||||
for zone, name, legal_geometry, legal_domain, source_ids in definitions:
|
||||
features.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": f"belgian-marine-scope:{zone}",
|
||||
"geometry": mapping(legal_geometry),
|
||||
"properties": {
|
||||
"name": name,
|
||||
"coverage_zone": zone,
|
||||
"legal_domain": legal_domain,
|
||||
"derived_from_reporting_unit_ids": source_ids,
|
||||
"source_name": "rbins_marine_reporting_units",
|
||||
"source_layer": RBINS_MRU_LAYER,
|
||||
"authority_level": "authoritative",
|
||||
"attribution": "Royal Belgian Institute of Natural Sciences (RBINS), BMDC",
|
||||
"source_url": RBINS_MRU_METADATA_URL,
|
||||
"derivation": (
|
||||
"Official reporting-unit geometry reused with explicit legal semantics"
|
||||
if len(source_ids) == 1
|
||||
else "Union of official 0-1 nm coastal waters and 1-12 nm territorial waters"
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"name": "Belgian marine legal scopes",
|
||||
"crs": GEOJSON_CRS,
|
||||
"features": features,
|
||||
"source_url": RBINS_MRU_METADATA_URL,
|
||||
}
|
||||
|
||||
|
||||
def build_msp_payload(layer_payloads: dict[str, dict[str, Any]]) -> dict[str, Any]:
|
||||
if set(layer_payloads) != set(MSP_LAYERS):
|
||||
missing = sorted(set(MSP_LAYERS) - set(layer_payloads))
|
||||
raise RuntimeError(f"Marine Spatial Plan payload is incomplete: {', '.join(missing)}")
|
||||
features: list[dict[str, Any]] = []
|
||||
for layer_name in MSP_LAYERS:
|
||||
short_name = layer_name.split(":", 1)[1]
|
||||
for index, source_feature in enumerate(layer_payloads[layer_name]["features"]):
|
||||
feature = dict(source_feature)
|
||||
properties = dict(feature.get("properties") or {})
|
||||
source_id = str(feature.get("id") or f"{short_name}.{index}")
|
||||
properties.update(
|
||||
{
|
||||
"source_name": "rbins_msp_2026",
|
||||
"source_layer": layer_name,
|
||||
"source_feature_id": source_id,
|
||||
"authority_level": "authoritative",
|
||||
"valid_from": MSP_VALID_FROM,
|
||||
"valid_to": "2034-12-31T23:59:59Z",
|
||||
"attribution": "Belgian federal Marine Environment service and RBINS",
|
||||
"source_url": RBINS_MSP_SOURCE_URL,
|
||||
}
|
||||
)
|
||||
feature["id"] = f"{short_name}:{source_id}"
|
||||
feature["properties"] = properties
|
||||
features.append(feature)
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"name": "Belgian Marine Spatial Plan 2026-2034",
|
||||
"crs": GEOJSON_CRS,
|
||||
"features": features,
|
||||
"source_url": RBINS_MSP_SOURCE_URL,
|
||||
"valid_from": MSP_VALID_FROM,
|
||||
"valid_to": "2034-12-31T23:59:59Z",
|
||||
}
|
||||
|
||||
|
||||
def _max_admin_modification(payloads: dict[str, dict[str, Any]], fallback: str) -> str:
|
||||
values = [
|
||||
str(payload["source_max_modification"])
|
||||
for payload in payloads.values()
|
||||
if payload.get("source_max_modification")
|
||||
]
|
||||
value = max(values) if values else fallback
|
||||
return f"{value[:10]}T00:00:00Z"
|
||||
|
||||
|
||||
def prepare_artifacts(args: argparse.Namespace) -> tuple[dict[str, Path], dict[str, Any]]:
|
||||
output_root = args.output_root
|
||||
output_root.mkdir(parents=True, exist_ok=True)
|
||||
archive_path = output_root / "adminvector_4326.zip"
|
||||
generated_at = utc_now()
|
||||
with build_session() as session:
|
||||
if args.force or not archive_path.is_file():
|
||||
download_limited(session, NGI_ARCHIVE_URL, archive_path, args.request_timeout, MAX_ARCHIVE_BYTES)
|
||||
gpkg_path = extract_single_geopackage(archive_path, output_root)
|
||||
admin_payloads = read_adminvector_layers(gpkg_path)
|
||||
reporting_units = fetch_wfs_layer(
|
||||
session,
|
||||
service_url=RBINS_MRU_WFS_URL,
|
||||
layer_name=RBINS_MRU_LAYER,
|
||||
timeout=args.request_timeout,
|
||||
)
|
||||
marine_scopes = derive_marine_scope_payload(reporting_units)
|
||||
msp_payloads = {
|
||||
layer: fetch_wfs_layer(
|
||||
session,
|
||||
service_url=RBINS_MSP_WFS_URL,
|
||||
layer_name=layer,
|
||||
timeout=args.request_timeout,
|
||||
)
|
||||
for layer in MSP_LAYERS
|
||||
}
|
||||
msp = build_msp_payload(msp_payloads)
|
||||
|
||||
artifact_payloads = {
|
||||
"belgium_land_boundary": admin_payloads["belgianterritory"],
|
||||
"belgium_regions": admin_payloads["region"],
|
||||
"belgium_provinces": admin_payloads["province"],
|
||||
"belgium_municipalities": admin_payloads["municipality"],
|
||||
"ngi_maritime_zone": admin_payloads["belgianmaritimezone"],
|
||||
"marine_legal_scopes": marine_scopes,
|
||||
"marine_spatial_plan_2026": msp,
|
||||
}
|
||||
artifacts: dict[str, Path] = {}
|
||||
for key, payload in artifact_payloads.items():
|
||||
path = output_root / f"{key}.geojson"
|
||||
write_json_atomic(path, payload)
|
||||
artifacts[key] = path
|
||||
|
||||
admin_observed_at = _max_admin_modification(admin_payloads, generated_at)
|
||||
mru_versions = [
|
||||
str((feature.get("properties") or {}).get("beginLife") or "")
|
||||
for feature in reporting_units["features"]
|
||||
if (feature.get("properties") or {}).get("beginLife")
|
||||
]
|
||||
marine_observed_at = f"{max(mru_versions)[:10]}T00:00:00Z" if mru_versions else generated_at
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"status": "complete",
|
||||
"generated_at": generated_at,
|
||||
"project_name": PROJECT_NAME,
|
||||
"scope": "belgium-and-belgian-north-sea",
|
||||
"ngi_archive_url": NGI_ARCHIVE_URL,
|
||||
"ngi_archive_sha256": sha256_file(archive_path),
|
||||
"ngi_geopackage_sha256": sha256_file(gpkg_path),
|
||||
"admin_observed_at": admin_observed_at,
|
||||
"marine_reporting_units_url": RBINS_MRU_WFS_URL,
|
||||
"marine_reporting_units_feature_count": len(reporting_units["features"]),
|
||||
"marine_observed_at": marine_observed_at,
|
||||
"msp_wfs_url": RBINS_MSP_WFS_URL,
|
||||
"msp_layer_count": len(MSP_LAYERS),
|
||||
"msp_feature_count": len(msp["features"]),
|
||||
"msp_valid_from": MSP_VALID_FROM,
|
||||
"artifacts": {
|
||||
key: {
|
||||
"filename": path.name,
|
||||
"sha256": sha256_file(path),
|
||||
"feature_count": len(artifact_payloads[key]["features"]),
|
||||
}
|
||||
for key, path in artifacts.items()
|
||||
},
|
||||
}
|
||||
write_json_atomic(output_root / "manifest.json", manifest, pretty=True)
|
||||
return artifacts, manifest
|
||||
|
||||
|
||||
def response_data(response: requests.Response) -> Any:
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"GeoIntel API returned non-JSON ({response.status_code}): {response.text[:500]}") from exc
|
||||
if not response.ok:
|
||||
raise RuntimeError(f"GeoIntel API failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:1000]}")
|
||||
if not isinstance(payload, dict) or "data" not in payload:
|
||||
raise RuntimeError("GeoIntel API response does not use the canonical data envelope")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def list_paginated(session: requests.Session, url: str, timeout: int) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
offset = 0
|
||||
while True:
|
||||
page = response_data(session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout))
|
||||
page_items = list(page.get("items") or [])
|
||||
items.extend(page_items)
|
||||
total = int(page.get("total") or len(items))
|
||||
if not page_items or len(items) >= total:
|
||||
break
|
||||
offset += len(page_items)
|
||||
return items
|
||||
|
||||
|
||||
def _feature_geometry(payload: dict[str, Any], *, feature_id: str | None = None) -> dict[str, Any]:
|
||||
features = payload["features"]
|
||||
feature = next((item for item in features if str(item.get("id")) == feature_id), None) if feature_id else features[0]
|
||||
if not feature or not feature.get("geometry"):
|
||||
raise RuntimeError(f"Expected geometry is missing for {feature_id or payload.get('name')}")
|
||||
return feature["geometry"]
|
||||
|
||||
|
||||
def _canonical_region_name(feature: dict[str, Any]) -> str:
|
||||
properties = feature.get("properties") or {}
|
||||
nis_code = str(properties.get("niscode") or "").lstrip("0")
|
||||
if nis_code == "2000":
|
||||
return "flanders"
|
||||
if nis_code == "3000":
|
||||
return "wallonia"
|
||||
if nis_code == "4000":
|
||||
return "brussels"
|
||||
names = " ".join(str(value) for key, value in properties.items() if "name" in key.lower() or "naam" in key.lower())
|
||||
normalized = names.casefold()
|
||||
if "vlaams" in normalized or "flam" in normalized:
|
||||
return "flanders"
|
||||
if "wallon" in normalized:
|
||||
return "wallonia"
|
||||
if "brux" in normalized or "brussel" in normalized:
|
||||
return "brussels"
|
||||
raise RuntimeError(f"Cannot map NGI region feature {feature.get('id')} to a canonical region")
|
||||
|
||||
|
||||
def _upload_dataset(
|
||||
session: requests.Session,
|
||||
*,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
area_id: str,
|
||||
path: Path,
|
||||
source_name: str,
|
||||
reference_layer_name: str,
|
||||
coverage_zones: list[str],
|
||||
observed_at: str,
|
||||
valid_from: str,
|
||||
valid_to: str | None,
|
||||
source_version: str,
|
||||
artifact_sha256: str,
|
||||
source_url: str,
|
||||
attribution: str,
|
||||
license_note: str,
|
||||
timeout: int,
|
||||
) -> dict[str, Any]:
|
||||
source_metadata = {
|
||||
"provider": source_name,
|
||||
"authority_level": "authoritative",
|
||||
"coverage_zones": coverage_zones,
|
||||
"reference_layer_name": reference_layer_name,
|
||||
"source_url": source_url,
|
||||
"attribution": attribution,
|
||||
"license_note": license_note,
|
||||
}
|
||||
provenance = {
|
||||
"operator_tool": "provision_belgium_north_sea_scope.py",
|
||||
"operator_explicit_fetch": True,
|
||||
"artifact_sha256": artifact_sha256,
|
||||
"source_url": source_url,
|
||||
"direct_vector_feature_write": False,
|
||||
}
|
||||
data = {
|
||||
"dataset_type": "vector",
|
||||
"source": "operator_official_import",
|
||||
"dataset_role": "reference",
|
||||
"source_name": source_name,
|
||||
"reference_layer_name": reference_layer_name,
|
||||
"source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
|
||||
"provenance_metadata_json": json.dumps(provenance, ensure_ascii=False),
|
||||
"area_id": area_id,
|
||||
"temporal_series_key": f"{source_name}:{reference_layer_name}",
|
||||
"observed_at": observed_at,
|
||||
"valid_from": valid_from,
|
||||
"temporal_granularity": "period" if valid_to else "snapshot",
|
||||
"source_version": source_version,
|
||||
}
|
||||
if valid_to:
|
||||
data["valid_to"] = valid_to
|
||||
with path.open("rb") as handle:
|
||||
response = session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
|
||||
data=data,
|
||||
files={"file": (path.name, handle, "application/geo+json")},
|
||||
timeout=timeout,
|
||||
)
|
||||
return response_data(response)
|
||||
|
||||
|
||||
def provision(args: argparse.Namespace, artifacts: dict[str, Path], manifest: dict[str, Any]) -> dict[str, Any]:
|
||||
base_url = args.base_url.rstrip("/")
|
||||
payloads = {key: json.loads(path.read_text(encoding="utf-8")) for key, path in artifacts.items()}
|
||||
with requests.Session() as session:
|
||||
projects = list_paginated(session, f"{base_url}/api/v1/projects", args.import_timeout)
|
||||
project = next((item for item in projects if item.get("name") == PROJECT_NAME), None)
|
||||
if project is None:
|
||||
project = response_data(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects",
|
||||
json={
|
||||
"name": PROJECT_NAME,
|
||||
"description": (
|
||||
"Authoritative national and maritime scope for Belgium, its territorial sea, "
|
||||
"exclusive economic zone and continental shelf."
|
||||
),
|
||||
"region": PROJECT_REGION,
|
||||
},
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
)
|
||||
project_id = str(project["id"])
|
||||
existing_areas = list_paginated(
|
||||
session,
|
||||
f"{base_url}/api/v1/projects/{project_id}/areas",
|
||||
args.import_timeout,
|
||||
)
|
||||
areas_by_name = {str(area.get("name")): area for area in existing_areas}
|
||||
|
||||
def ensure_area(name: str, geometry: dict[str, Any]) -> dict[str, Any]:
|
||||
if name in areas_by_name:
|
||||
existing = areas_by_name[name]
|
||||
existing_geometry = existing.get("geometry")
|
||||
if not existing_geometry or not shape(existing_geometry).equals(shape(geometry)):
|
||||
raise RuntimeError(
|
||||
f"Persisted Area {name!r} differs from the current authoritative geometry; "
|
||||
"provision a fresh versioned national project instead of silently mutating it"
|
||||
)
|
||||
return existing
|
||||
area = response_data(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/areas",
|
||||
json={"name": name, "crs": "EPSG:4326", "geometry": geometry},
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
)
|
||||
areas_by_name[name] = area
|
||||
return area
|
||||
|
||||
land_area = ensure_area(AREA_NAMES["belgium"], _feature_geometry(payloads["belgium_land_boundary"]))
|
||||
region_features = {_canonical_region_name(feature): feature for feature in payloads["belgium_regions"]["features"]}
|
||||
for zone in ("flanders", "wallonia", "brussels"):
|
||||
ensure_area(AREA_NAMES[zone], region_features[zone]["geometry"])
|
||||
marine_features = {
|
||||
str((feature.get("properties") or {}).get("coverage_zone")): feature
|
||||
for feature in payloads["marine_legal_scopes"]["features"]
|
||||
}
|
||||
for zone in ("belgian_north_sea", "territorial_sea", "exclusive_economic_zone", "continental_shelf"):
|
||||
ensure_area(AREA_NAMES[zone], marine_features[zone]["geometry"])
|
||||
|
||||
datasets = list_paginated(
|
||||
session,
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets",
|
||||
args.import_timeout,
|
||||
)
|
||||
specs = (
|
||||
(
|
||||
"belgium_land_boundary",
|
||||
"ngi_adminvector",
|
||||
"belgium_land_boundary",
|
||||
["belgium"],
|
||||
str(land_area["id"]),
|
||||
manifest["admin_observed_at"],
|
||||
manifest["admin_observed_at"],
|
||||
None,
|
||||
manifest["admin_observed_at"][:10],
|
||||
NGI_CATALOG_URL,
|
||||
NGI_ATTRIBUTION,
|
||||
NGI_LICENSE,
|
||||
),
|
||||
(
|
||||
"belgium_regions",
|
||||
"ngi_adminvector",
|
||||
"belgium_regions",
|
||||
["belgium", "flanders", "wallonia", "brussels"],
|
||||
str(land_area["id"]),
|
||||
manifest["admin_observed_at"],
|
||||
manifest["admin_observed_at"],
|
||||
None,
|
||||
manifest["admin_observed_at"][:10],
|
||||
NGI_CATALOG_URL,
|
||||
NGI_ATTRIBUTION,
|
||||
NGI_LICENSE,
|
||||
),
|
||||
(
|
||||
"belgium_provinces",
|
||||
"ngi_adminvector",
|
||||
"belgium_provinces",
|
||||
["belgium", "flanders", "wallonia"],
|
||||
str(land_area["id"]),
|
||||
manifest["admin_observed_at"],
|
||||
manifest["admin_observed_at"],
|
||||
None,
|
||||
manifest["admin_observed_at"][:10],
|
||||
NGI_CATALOG_URL,
|
||||
NGI_ATTRIBUTION,
|
||||
NGI_LICENSE,
|
||||
),
|
||||
(
|
||||
"belgium_municipalities",
|
||||
"ngi_adminvector",
|
||||
"belgium_municipalities",
|
||||
["belgium", "flanders", "wallonia", "brussels"],
|
||||
str(land_area["id"]),
|
||||
manifest["admin_observed_at"],
|
||||
manifest["admin_observed_at"],
|
||||
None,
|
||||
manifest["admin_observed_at"][:10],
|
||||
NGI_CATALOG_URL,
|
||||
NGI_ATTRIBUTION,
|
||||
NGI_LICENSE,
|
||||
),
|
||||
(
|
||||
"marine_legal_scopes",
|
||||
"rbins_marine_reporting_units",
|
||||
"marine_legal_scopes",
|
||||
[
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
],
|
||||
str(areas_by_name[AREA_NAMES["belgian_north_sea"]]["id"]),
|
||||
manifest["marine_observed_at"],
|
||||
manifest["marine_observed_at"],
|
||||
None,
|
||||
manifest["marine_observed_at"][:10],
|
||||
RBINS_MRU_METADATA_URL,
|
||||
"Royal Belgian Institute of Natural Sciences (RBINS), BMDC",
|
||||
"See source metadata",
|
||||
),
|
||||
(
|
||||
"marine_spatial_plan_2026",
|
||||
"rbins_msp_2026",
|
||||
"marine_spatial_plan_2026",
|
||||
[
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
],
|
||||
str(areas_by_name[AREA_NAMES["belgian_north_sea"]]["id"]),
|
||||
MSP_VALID_FROM,
|
||||
MSP_VALID_FROM,
|
||||
"2034-12-31T23:59:59Z",
|
||||
"2026-2034",
|
||||
RBINS_MSP_SOURCE_URL,
|
||||
"Belgian federal Marine Environment service and RBINS",
|
||||
"See source metadata",
|
||||
),
|
||||
)
|
||||
persisted = []
|
||||
for (
|
||||
artifact_key,
|
||||
source_name,
|
||||
layer_name,
|
||||
coverage_zones,
|
||||
area_id,
|
||||
observed_at,
|
||||
valid_from,
|
||||
valid_to,
|
||||
source_version,
|
||||
source_url,
|
||||
attribution,
|
||||
license_note,
|
||||
) in specs:
|
||||
checksum = manifest["artifacts"][artifact_key]["sha256"]
|
||||
existing = next(
|
||||
(
|
||||
dataset
|
||||
for dataset in datasets
|
||||
if dataset.get("source_name") == source_name
|
||||
and dataset.get("reference_layer_name") == layer_name
|
||||
and dataset.get("source_version") == source_version
|
||||
),
|
||||
None,
|
||||
)
|
||||
if existing:
|
||||
persisted_checksum = (existing.get("provenance_metadata") or {}).get("artifact_sha256")
|
||||
if persisted_checksum != checksum:
|
||||
raise RuntimeError(
|
||||
f"Existing immutable dataset {layer_name} has checksum {persisted_checksum}, expected {checksum}"
|
||||
)
|
||||
persisted.append(existing)
|
||||
continue
|
||||
created = _upload_dataset(
|
||||
session,
|
||||
base_url=base_url,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
path=artifacts[artifact_key],
|
||||
source_name=source_name,
|
||||
reference_layer_name=layer_name,
|
||||
coverage_zones=coverage_zones,
|
||||
observed_at=observed_at,
|
||||
valid_from=valid_from,
|
||||
valid_to=valid_to,
|
||||
source_version=source_version,
|
||||
artifact_sha256=checksum,
|
||||
source_url=source_url,
|
||||
attribution=attribution,
|
||||
license_note=license_note,
|
||||
timeout=args.import_timeout,
|
||||
)
|
||||
persisted.append(created)
|
||||
|
||||
return {
|
||||
"project_id": project_id,
|
||||
"project_name": PROJECT_NAME,
|
||||
"area_count": len(AREA_NAMES),
|
||||
"dataset_ids": [str(dataset["id"]) for dataset in persisted],
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
try:
|
||||
artifacts, manifest = prepare_artifacts(args)
|
||||
workspace = None if args.fetch_only else provision(args, artifacts, manifest)
|
||||
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException, zipfile.BadZipFile) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"mode": "fetch_only" if args.fetch_only else "provisioned",
|
||||
"scope": manifest["scope"],
|
||||
"manifest_path": str(args.output_root / "manifest.json"),
|
||||
"artifact_count": len(artifacts),
|
||||
"msp_feature_count": manifest["msp_feature_count"],
|
||||
"workspace": workspace,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -78,6 +78,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_regional_soil_map.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_timeseries.py
|
||||
${PYTHON_BIN} -m py_compile scripts/geographic_scopes.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_geographic_scope.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_belgium_north_sea_scope.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_grb_buildings.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_grb_context.py
|
||||
${PYTHON_BIN} -m py_compile scripts/audit_source_freshness.py
|
||||
|
||||
Reference in New Issue
Block a user