Add Belgium and North Sea coverage foundation
This commit is contained in:
@@ -987,6 +987,30 @@ it does not create a derived dataset.
|
||||
|
||||
## Geographic scope provisioning
|
||||
|
||||
The release-candidate national foundation is provisioned explicitly:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_belgium_north_sea_scope.py
|
||||
```
|
||||
|
||||
Use `--fetch-only` to validate official NGI AdminVector, RBINS marine
|
||||
reporting units and the Belgian Marine Spatial Plan 2026-2034 without changing
|
||||
application persistence. The normal command creates or reuses
|
||||
`Belgium and North Sea Workbench`, persists Belgium, all three regions,
|
||||
territorial sea, EEZ and continental shelf as Areas, and uploads six
|
||||
checksum-bound reference Datasets through `DatasetService`.
|
||||
|
||||
The operator has a fixed URL/layer allowlist, verified TLS, archive and
|
||||
response-size limits, safe ZIP extraction, complete WFS pagination and
|
||||
immutable artifact checksums. It does not run at startup and does not write
|
||||
directly to `vector_features`.
|
||||
|
||||
`GET /api/v1/external/coverage/catalog` exposes audited national source
|
||||
contracts. `POST /api/v1/external/coverage/resolve` intersects a drawn bbox
|
||||
with persisted legal/administrative Areas and reports a split zone/theme
|
||||
matrix. `operational` requires a matching `ready` Dataset; integration without
|
||||
materialized data is only `partial`.
|
||||
|
||||
The explicit operator command below provisions the official 28-municipality
|
||||
Vlaamse vervoerregio Kempen boundary foundation:
|
||||
|
||||
|
||||
@@ -7,7 +7,8 @@ from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.models import Area, Project
|
||||
from app.providers.registry import fetch_provider_data, get_provider, import_provider_dataset, list_provider_capabilities
|
||||
from app.schemas import ExternalFetchRequest, ExternalFetchResponse, ProviderImportRequest
|
||||
from app.schemas import CoverageResolveRequest, ExternalFetchRequest, ExternalFetchResponse, ProviderImportRequest
|
||||
from app.services.coverage_registry_service import CoverageRegistryService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/external", tags=["external"])
|
||||
@@ -46,6 +47,22 @@ def list_external_providers() -> dict:
|
||||
})
|
||||
|
||||
|
||||
@router.get("/coverage/catalog")
|
||||
def get_coverage_catalog() -> dict:
|
||||
return envelope(CoverageRegistryService.catalog().model_dump())
|
||||
|
||||
|
||||
@router.post("/coverage/resolve")
|
||||
def resolve_project_coverage(payload: CoverageResolveRequest, db: Session = Depends(get_db)) -> dict:
|
||||
result = CoverageRegistryService.resolve(
|
||||
db,
|
||||
project_id=payload.project_id,
|
||||
bbox=payload.bbox,
|
||||
themes=payload.themes,
|
||||
)
|
||||
return envelope(result.model_dump())
|
||||
|
||||
|
||||
@router.get("/providers/capabilities")
|
||||
def get_external_provider_capabilities() -> dict:
|
||||
return envelope({
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .common import ApiErrorEnvelope, ApiErrorItem, Envelope, PaginationEnvelope
|
||||
from .coverage import (
|
||||
CoverageBBox,
|
||||
CoverageCatalogResponse,
|
||||
CoverageResolutionItem,
|
||||
CoverageResolveRequest,
|
||||
CoverageResolveResponse,
|
||||
CoverageSourceContract,
|
||||
)
|
||||
from .project import ProjectCreate, ProjectList, ProjectRead, ProjectUpdate
|
||||
from .area import AreaCreate, AreaList, AreaRead, AreaUpdate
|
||||
from .analysis import ChangeDetectionRequest, ChangeDetectionSummary
|
||||
@@ -145,6 +153,12 @@ __all__ = [
|
||||
"ApiErrorEnvelope",
|
||||
"ApiErrorItem",
|
||||
"PaginationEnvelope",
|
||||
"CoverageBBox",
|
||||
"CoverageCatalogResponse",
|
||||
"CoverageResolutionItem",
|
||||
"CoverageResolveRequest",
|
||||
"CoverageResolveResponse",
|
||||
"CoverageSourceContract",
|
||||
"ProjectCreate",
|
||||
"ProjectRead",
|
||||
"ProjectUpdate",
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
|
||||
CoverageStatus = Literal["operational", "partial", "not_configured", "unsupported"]
|
||||
CoverageAuthority = Literal["authoritative", "official_context", "contextual"]
|
||||
CoverageAcquisitionMode = Literal[
|
||||
"operator_archive",
|
||||
"operator_wfs",
|
||||
"bounded_api",
|
||||
"bounded_raster",
|
||||
"catalog_only",
|
||||
]
|
||||
|
||||
|
||||
class CoverageBBox(BaseModel):
|
||||
minx: float = Field(ge=-180, le=180)
|
||||
miny: float = Field(ge=-90, le=90)
|
||||
maxx: float = Field(ge=-180, le=180)
|
||||
maxy: float = Field(ge=-90, le=90)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_extent(self) -> "CoverageBBox":
|
||||
if self.maxx <= self.minx or self.maxy <= self.miny:
|
||||
raise ValueError("bbox max values must be greater than min values")
|
||||
return self
|
||||
|
||||
|
||||
class CoverageSourceContract(BaseModel):
|
||||
source_name: str
|
||||
display_name: str
|
||||
authority_level: CoverageAuthority
|
||||
coverage_zones: list[str]
|
||||
themes: list[str]
|
||||
native_layers: list[str]
|
||||
supported_geometry_types: list[str]
|
||||
acquisition_mode: CoverageAcquisitionMode
|
||||
integration_status: CoverageStatus
|
||||
source_url: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class CoverageCatalogResponse(BaseModel):
|
||||
themes: list[str]
|
||||
zones: list[str]
|
||||
statuses: list[CoverageStatus]
|
||||
sources: list[CoverageSourceContract]
|
||||
|
||||
|
||||
class CoverageResolveRequest(BaseModel):
|
||||
project_id: UUID
|
||||
bbox: CoverageBBox
|
||||
themes: list[str] = Field(default_factory=list, max_length=32)
|
||||
|
||||
|
||||
class CoverageResolutionItem(BaseModel):
|
||||
zone: str
|
||||
theme: str
|
||||
status: CoverageStatus
|
||||
source_names: list[str]
|
||||
materialized_dataset_ids: list[UUID]
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class CoverageResolveResponse(BaseModel):
|
||||
project_id: UUID
|
||||
bbox: CoverageBBox
|
||||
requested_themes: list[str]
|
||||
intersected_zones: list[str]
|
||||
outside_supported_scope: bool
|
||||
items: list[CoverageResolutionItem]
|
||||
warnings: list[str]
|
||||
@@ -0,0 +1,513 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import box
|
||||
from shapely.ops import unary_union
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.schemas.coverage import (
|
||||
CoverageBBox,
|
||||
CoverageCatalogResponse,
|
||||
CoverageResolutionItem,
|
||||
CoverageResolveResponse,
|
||||
CoverageSourceContract,
|
||||
)
|
||||
|
||||
|
||||
THEMES = (
|
||||
"admin",
|
||||
"buildings",
|
||||
"roads",
|
||||
"surface_water",
|
||||
"land_cover_use",
|
||||
"nature",
|
||||
"population",
|
||||
"parcels",
|
||||
"soil",
|
||||
"elevation",
|
||||
"orthophoto",
|
||||
"flood_climate",
|
||||
"maritime_planning",
|
||||
"marine_environment",
|
||||
"bathymetry",
|
||||
)
|
||||
|
||||
ZONES = (
|
||||
"belgium",
|
||||
"flanders",
|
||||
"wallonia",
|
||||
"brussels",
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
)
|
||||
|
||||
STATUS_ORDER = ("unsupported", "not_configured", "partial", "operational")
|
||||
STATUS_RANK = {status: index for index, status in enumerate(STATUS_ORDER)}
|
||||
|
||||
SCOPE_AREA_NAMES = {
|
||||
"belgium": "Belgium land",
|
||||
"flanders": "Flanders",
|
||||
"wallonia": "Wallonia",
|
||||
"brussels": "Brussels-Capital Region",
|
||||
"belgian_north_sea": "Belgian part of the North Sea",
|
||||
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
|
||||
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
|
||||
"continental_shelf": "Belgian continental shelf beyond territorial sea",
|
||||
}
|
||||
|
||||
DETAIL_ZONES = (
|
||||
"flanders",
|
||||
"wallonia",
|
||||
"brussels",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SourceDefinition:
|
||||
contract: CoverageSourceContract
|
||||
materialized_layer_names: tuple[str, ...] = ()
|
||||
materialized_source_names: tuple[str, ...] = ()
|
||||
|
||||
|
||||
def _contract(
|
||||
*,
|
||||
source_name: str,
|
||||
display_name: str,
|
||||
authority_level: str,
|
||||
coverage_zones: tuple[str, ...],
|
||||
themes: tuple[str, ...],
|
||||
native_layers: tuple[str, ...],
|
||||
geometry_types: tuple[str, ...],
|
||||
acquisition_mode: str,
|
||||
integration_status: str,
|
||||
source_url: str,
|
||||
attribution: str,
|
||||
license_note: str,
|
||||
limitation_message: str,
|
||||
materialized_layer_names: tuple[str, ...] = (),
|
||||
materialized_source_names: tuple[str, ...] = (),
|
||||
) -> _SourceDefinition:
|
||||
return _SourceDefinition(
|
||||
contract=CoverageSourceContract(
|
||||
source_name=source_name,
|
||||
display_name=display_name,
|
||||
authority_level=authority_level,
|
||||
coverage_zones=list(coverage_zones),
|
||||
themes=list(themes),
|
||||
native_layers=list(native_layers),
|
||||
supported_geometry_types=list(geometry_types),
|
||||
acquisition_mode=acquisition_mode,
|
||||
integration_status=integration_status,
|
||||
source_url=source_url,
|
||||
attribution=attribution,
|
||||
license_note=license_note,
|
||||
limitation_message=limitation_message,
|
||||
),
|
||||
materialized_layer_names=materialized_layer_names,
|
||||
materialized_source_names=materialized_source_names or (source_name,),
|
||||
)
|
||||
|
||||
|
||||
SOURCE_DEFINITIONS = (
|
||||
_contract(
|
||||
source_name="ngi_adminvector",
|
||||
display_name="NGI AdminVector",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("belgium", "flanders", "wallonia", "brussels", "belgian_north_sea"),
|
||||
themes=("admin",),
|
||||
native_layers=(
|
||||
"belgianterritory",
|
||||
"belgianmaritimezone",
|
||||
"region",
|
||||
"province",
|
||||
"municipality",
|
||||
),
|
||||
geometry_types=("Polygon", "MultiPolygon"),
|
||||
acquisition_mode="operator_archive",
|
||||
integration_status="operational",
|
||||
source_url="https://www.geo.be/catalog/details/fb1e2993-2020-428c-9188-eb5f75e284b9",
|
||||
attribution="National Geographic Institute (NGI), AdminVector",
|
||||
license_note="CC BY 4.0",
|
||||
limitation_message="Administrative reference geometry; it does not provide thematic land content.",
|
||||
materialized_layer_names=(
|
||||
"belgium_land_boundary",
|
||||
"belgium_regions",
|
||||
"belgium_provinces",
|
||||
"belgium_municipalities",
|
||||
),
|
||||
),
|
||||
_contract(
|
||||
source_name="statbel",
|
||||
display_name="Statbel statistical sectors and population",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("belgium", "flanders", "wallonia", "brussels"),
|
||||
themes=("admin", "population"),
|
||||
native_layers=("statistical_sectors", "population_statistics"),
|
||||
geometry_types=("Polygon", "MultiPolygon", "Tabular"),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
source_url="https://statbel.fgov.be/en/open-data",
|
||||
attribution="Statbel",
|
||||
license_note="Consult the license of the selected Statbel release.",
|
||||
limitation_message="The catalog is audited, but no national bounded acquisition adapter is configured yet.",
|
||||
),
|
||||
_contract(
|
||||
source_name="digitaal_vlaanderen",
|
||||
display_name="Flemish authoritative services",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("flanders",),
|
||||
themes=(
|
||||
"buildings",
|
||||
"roads",
|
||||
"surface_water",
|
||||
"land_cover_use",
|
||||
"nature",
|
||||
"parcels",
|
||||
"soil",
|
||||
"elevation",
|
||||
"orthophoto",
|
||||
"flood_climate",
|
||||
),
|
||||
native_layers=("GRB", "BWK", "DHMV", "OMWRGBMRVL", "OGRK", "Mercator"),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||
acquisition_mode="bounded_api",
|
||||
integration_status="operational",
|
||||
source_url="https://www.vlaanderen.be/datavindplaats",
|
||||
attribution="Digitaal Vlaanderen and the authoritative Flemish source owners",
|
||||
license_note="Consult the license and attribution stored with each acquired dataset.",
|
||||
limitation_message="Operational only for bounded products implemented by GeoIntel and materialized in the project.",
|
||||
materialized_source_names=(
|
||||
"grb",
|
||||
"digitaal_vlaanderen_buildings_addresses_register",
|
||||
"digitaal_vlaanderen_dhmv",
|
||||
"digitaal_vlaanderen_orthophoto",
|
||||
"vmm_flood_hazard",
|
||||
"department_omgeving_thematic_raster",
|
||||
"inbo_bwk_natura2000",
|
||||
"dov_soil_map",
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels",
|
||||
),
|
||||
),
|
||||
_contract(
|
||||
source_name="spw_geoportail",
|
||||
display_name="SPW Geoportail Wallonie",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("wallonia",),
|
||||
themes=(
|
||||
"buildings",
|
||||
"roads",
|
||||
"surface_water",
|
||||
"land_cover_use",
|
||||
"nature",
|
||||
"soil",
|
||||
"elevation",
|
||||
"orthophoto",
|
||||
"flood_climate",
|
||||
"bathymetry",
|
||||
),
|
||||
native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
source_url="https://geoportail.wallonie.be/catalogue",
|
||||
attribution="Service public de Wallonie",
|
||||
license_note="Consult the license of each Geoportail Wallonie product.",
|
||||
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.",
|
||||
),
|
||||
_contract(
|
||||
source_name="urbis",
|
||||
display_name="UrbIS Brussels",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=("brussels",),
|
||||
themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"),
|
||||
native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
source_url="https://datastore.brussels",
|
||||
attribution="Brussels UrbIS",
|
||||
license_note="Consult the license of the selected UrbIS dataset.",
|
||||
limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.",
|
||||
),
|
||||
_contract(
|
||||
source_name="rbins_marine_reporting_units",
|
||||
display_name="RBINS marine reporting units",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=(
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
),
|
||||
themes=("admin", "marine_environment"),
|
||||
native_layers=("marine_reporting_units_2024",),
|
||||
geometry_types=("Polygon", "MultiPolygon"),
|
||||
acquisition_mode="operator_wfs",
|
||||
integration_status="operational",
|
||||
source_url=(
|
||||
"https://metadata.naturalsciences.be/geonetwork/srv/api/records/"
|
||||
"29f40b0d-2a3e-49a8-870a-e9b4acd4d1e3"
|
||||
),
|
||||
attribution="Royal Belgian Institute of Natural Sciences (RBINS), BMDC",
|
||||
license_note="Reuse conditions are retained from the source metadata with every persisted artifact.",
|
||||
limitation_message="The EEZ and continental shelf can share geometry while retaining different legal semantics.",
|
||||
materialized_layer_names=("marine_legal_scopes",),
|
||||
),
|
||||
_contract(
|
||||
source_name="rbins_msp_2026",
|
||||
display_name="Belgian Marine Spatial Plan 2026-2034",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=(
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
),
|
||||
themes=("maritime_planning", "marine_environment"),
|
||||
native_layers=("imsp26",),
|
||||
geometry_types=("Point", "LineString", "Polygon", "MultiPolygon"),
|
||||
acquisition_mode="operator_wfs",
|
||||
integration_status="operational",
|
||||
source_url="https://www.health.belgium.be/en/themes/environment/marine-environment/marine-spatial-plan",
|
||||
attribution="Belgian federal Marine Environment service and RBINS",
|
||||
license_note="Official source metadata and attribution are retained with the imported snapshot.",
|
||||
limitation_message="The dataset represents the legally current 2026-2034 plan, not live maritime activity.",
|
||||
materialized_layer_names=("marine_spatial_plan_2026",),
|
||||
),
|
||||
_contract(
|
||||
source_name="mdk_bathymetry",
|
||||
display_name="MDK Belgian North Sea depth model",
|
||||
authority_level="authoritative",
|
||||
coverage_zones=(
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
),
|
||||
themes=("bathymetry",),
|
||||
native_layers=("depth_model_20m_lat",),
|
||||
geometry_types=("Raster",),
|
||||
acquisition_mode="catalog_only",
|
||||
integration_status="not_configured",
|
||||
source_url="https://www.vlaanderen.be/datavindplaats",
|
||||
attribution="Agentschap Maritieme Dienstverlening en Kust (MDK)",
|
||||
license_note="Consult the official product license before acquisition.",
|
||||
limitation_message="Strict-TLS acquisition and vertical datum evidence are not yet sufficient; no depths are synthesized.",
|
||||
),
|
||||
)
|
||||
|
||||
FLANDERS_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = {
|
||||
"buildings": {
|
||||
"grb": ("buildings",),
|
||||
"digitaal_vlaanderen_buildings_addresses_register": (),
|
||||
},
|
||||
"roads": {"grb": ("roads",)},
|
||||
"surface_water": {"grb": ("water",)},
|
||||
"land_cover_use": {
|
||||
"department_omgeving_thematic_raster": (),
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels": (),
|
||||
},
|
||||
"nature": {"inbo_bwk_natura2000": ()},
|
||||
"parcels": {
|
||||
"grb": ("parcels",),
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels": (),
|
||||
},
|
||||
"soil": {"dov_soil_map": ()},
|
||||
"elevation": {"digitaal_vlaanderen_dhmv": ()},
|
||||
"orthophoto": {"digitaal_vlaanderen_orthophoto": ()},
|
||||
"flood_climate": {"vmm_flood_hazard": ()},
|
||||
}
|
||||
|
||||
|
||||
class CoverageRegistryService:
|
||||
@staticmethod
|
||||
def catalog() -> CoverageCatalogResponse:
|
||||
return CoverageCatalogResponse(
|
||||
themes=list(THEMES),
|
||||
zones=list(ZONES),
|
||||
statuses=list(STATUS_ORDER),
|
||||
sources=[definition.contract for definition in SOURCE_DEFINITIONS],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def normalize_themes(themes: Iterable[str]) -> list[str]:
|
||||
requested = list(dict.fromkeys(str(theme).strip().lower() for theme in themes if str(theme).strip()))
|
||||
invalid = sorted(set(requested) - set(THEMES))
|
||||
if invalid:
|
||||
raise AppError(
|
||||
code="COVERAGE_THEME_UNSUPPORTED",
|
||||
message="One or more coverage themes are unsupported",
|
||||
status_code=422,
|
||||
details={"unsupported_themes": invalid, "supported_themes": list(THEMES)},
|
||||
)
|
||||
return requested or list(THEMES)
|
||||
|
||||
@staticmethod
|
||||
def _geometry(value: Any):
|
||||
if value is None:
|
||||
return None
|
||||
return value if hasattr(value, "__geo_interface__") else to_shape(value)
|
||||
|
||||
@staticmethod
|
||||
def _intersected_zones(areas: list[Area], selection) -> tuple[list[str], bool]:
|
||||
geometries: dict[str, Any] = {}
|
||||
by_name = {area.name: area for area in areas}
|
||||
for zone, area_name in SCOPE_AREA_NAMES.items():
|
||||
area = by_name.get(area_name)
|
||||
geometry = CoverageRegistryService._geometry(area.geometry) if area else None
|
||||
if geometry is not None and not geometry.is_empty:
|
||||
geometries[zone] = geometry
|
||||
|
||||
detail_intersections = [
|
||||
zone for zone in DETAIL_ZONES if zone in geometries and geometries[zone].intersects(selection)
|
||||
]
|
||||
zones = detail_intersections
|
||||
if not any(zone in zones for zone in ("flanders", "wallonia", "brussels")):
|
||||
if "belgium" in geometries and geometries["belgium"].intersects(selection):
|
||||
zones = ["belgium", *zones]
|
||||
if not any(zone in zones for zone in ("territorial_sea", "exclusive_economic_zone", "continental_shelf")):
|
||||
if "belgian_north_sea" in geometries and geometries["belgian_north_sea"].intersects(selection):
|
||||
zones = [*zones, "belgian_north_sea"]
|
||||
|
||||
intersected_geometries = [geometries[zone].intersection(selection) for zone in zones if zone in geometries]
|
||||
covered = unary_union(intersected_geometries) if intersected_geometries else None
|
||||
outside = covered is None or covered.is_empty or not covered.covers(selection)
|
||||
return zones, outside
|
||||
|
||||
@staticmethod
|
||||
def _matching_datasets(
|
||||
datasets: list[Dataset],
|
||||
definition: _SourceDefinition,
|
||||
theme: str,
|
||||
zone: str,
|
||||
) -> list[Dataset]:
|
||||
matches: list[Dataset] = []
|
||||
for dataset in datasets:
|
||||
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
|
||||
continue
|
||||
layer_names = definition.materialized_layer_names
|
||||
if definition.contract.source_name == "digitaal_vlaanderen":
|
||||
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
|
||||
if dataset.source_name not in theme_sources:
|
||||
continue
|
||||
layer_names = theme_sources[dataset.source_name]
|
||||
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||
coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or []
|
||||
if isinstance(coverage_zones, str):
|
||||
coverage_zones = [coverage_zones]
|
||||
layer_matches = not layer_names or dataset.reference_layer_name in layer_names
|
||||
zone_matches = not coverage_zones or zone in coverage_zones or "belgium" in coverage_zones
|
||||
if layer_matches and zone_matches:
|
||||
matches.append(dataset)
|
||||
return matches
|
||||
|
||||
@staticmethod
|
||||
def _resolve_item(
|
||||
*,
|
||||
zone: str,
|
||||
theme: str,
|
||||
datasets: list[Dataset],
|
||||
) -> CoverageResolutionItem:
|
||||
definitions = [
|
||||
definition
|
||||
for definition in SOURCE_DEFINITIONS
|
||||
if zone in definition.contract.coverage_zones and theme in definition.contract.themes
|
||||
]
|
||||
if not definitions:
|
||||
return CoverageResolutionItem(
|
||||
zone=zone,
|
||||
theme=theme,
|
||||
status="unsupported",
|
||||
source_names=[],
|
||||
materialized_dataset_ids=[],
|
||||
limitation_message="No audited source contract supports this theme in the selected zone.",
|
||||
)
|
||||
|
||||
materialized: list[Dataset] = []
|
||||
source_statuses: list[str] = []
|
||||
limitations: list[str] = []
|
||||
for definition in definitions:
|
||||
matches = CoverageRegistryService._matching_datasets(
|
||||
datasets,
|
||||
definition,
|
||||
theme,
|
||||
zone,
|
||||
)
|
||||
materialized.extend(matches)
|
||||
if matches:
|
||||
source_statuses.append("operational")
|
||||
elif definition.contract.integration_status == "operational":
|
||||
source_statuses.append("partial")
|
||||
else:
|
||||
source_statuses.append(definition.contract.integration_status)
|
||||
limitations.append(definition.contract.limitation_message)
|
||||
|
||||
best_status = max(source_statuses, key=STATUS_RANK.__getitem__)
|
||||
return CoverageResolutionItem(
|
||||
zone=zone,
|
||||
theme=theme,
|
||||
status=best_status,
|
||||
source_names=[definition.contract.source_name for definition in definitions],
|
||||
materialized_dataset_ids=list(dict.fromkeys(dataset.id for dataset in materialized)),
|
||||
limitation_message=" ".join(dict.fromkeys(limitations)),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def resolve(
|
||||
db: Session,
|
||||
project_id: UUID,
|
||||
bbox: CoverageBBox,
|
||||
themes: Iterable[str],
|
||||
) -> CoverageResolveResponse:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
|
||||
requested_themes = CoverageRegistryService.normalize_themes(themes)
|
||||
selection = box(bbox.minx, bbox.miny, bbox.maxx, bbox.maxy)
|
||||
areas = db.query(Area).filter(Area.project_id == project_id).all()
|
||||
datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all()
|
||||
zones, outside_supported_scope = CoverageRegistryService._intersected_zones(areas, selection)
|
||||
if not zones:
|
||||
return CoverageResolveResponse(
|
||||
project_id=project_id,
|
||||
bbox=bbox,
|
||||
requested_themes=requested_themes,
|
||||
intersected_zones=[],
|
||||
outside_supported_scope=True,
|
||||
items=[],
|
||||
warnings=["The selection does not intersect a persisted Belgium or Belgian North Sea scope."],
|
||||
)
|
||||
|
||||
items = [
|
||||
CoverageRegistryService._resolve_item(zone=zone, theme=theme, datasets=datasets)
|
||||
for zone in zones
|
||||
for theme in requested_themes
|
||||
]
|
||||
warnings = []
|
||||
if outside_supported_scope:
|
||||
warnings.append("Part of the selection lies outside the persisted Belgium and Belgian North Sea scopes.")
|
||||
if len(zones) > 1:
|
||||
warnings.append(
|
||||
"The selection crosses coverage zones; results remain split and only semantically compatible metrics may be merged."
|
||||
)
|
||||
return CoverageResolveResponse(
|
||||
project_id=project_id,
|
||||
bbox=bbox,
|
||||
requested_themes=requested_themes,
|
||||
intersected_zones=zones,
|
||||
outside_supported_scope=outside_supported_scope,
|
||||
items=items,
|
||||
warnings=warnings,
|
||||
)
|
||||
@@ -0,0 +1,212 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.schemas.coverage import CoverageBBox
|
||||
from app.services.coverage_registry_service import CoverageRegistryService, THEMES, ZONES
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
|
||||
def filter(self, *_args, **_kwargs):
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return list(self.rows)
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, *, project, areas, datasets):
|
||||
self.project = project
|
||||
self.areas = areas
|
||||
self.datasets = datasets
|
||||
|
||||
def get(self, model, object_id):
|
||||
if model is Project and str(self.project.id) == str(object_id):
|
||||
return self.project
|
||||
return None
|
||||
|
||||
def query(self, model):
|
||||
if model is Area:
|
||||
return FakeQuery(self.areas)
|
||||
if model is Dataset:
|
||||
return FakeQuery(self.datasets)
|
||||
raise AssertionError(f"Unexpected query model: {model}")
|
||||
|
||||
|
||||
def scope_area(name: str, geometry):
|
||||
return SimpleNamespace(name=name, geometry=geometry)
|
||||
|
||||
|
||||
def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider_registry() -> None:
|
||||
catalog = CoverageRegistryService.catalog()
|
||||
|
||||
assert set(catalog.themes) == set(THEMES)
|
||||
assert set(catalog.zones) == set(ZONES)
|
||||
assert catalog.statuses == ["unsupported", "not_configured", "partial", "operational"]
|
||||
assert {source.source_name for source in catalog.sources} >= {
|
||||
"ngi_adminvector",
|
||||
"statbel",
|
||||
"digitaal_vlaanderen",
|
||||
"spw_geoportail",
|
||||
"urbis",
|
||||
"rbins_marine_reporting_units",
|
||||
"rbins_msp_2026",
|
||||
"mdk_bathymetry",
|
||||
}
|
||||
assert next(source for source in catalog.sources if source.source_name == "ngi_adminvector").license_note == "CC BY 4.0"
|
||||
assert next(source for source in catalog.sources if source.source_name == "mdk_bathymetry").integration_status == "not_configured"
|
||||
|
||||
response = TestClient(app).get("/api/v1/external/coverage/catalog")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["data"]["themes"] == list(THEMES)
|
||||
|
||||
|
||||
def test_coverage_resolver_only_reports_operational_for_materialized_ready_dataset() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
areas = [
|
||||
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
|
||||
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
|
||||
]
|
||||
bbox = CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0)
|
||||
|
||||
without_materialized = CoverageRegistryService.resolve(
|
||||
FakeSession(project=project, areas=areas, datasets=[]),
|
||||
project_id,
|
||||
bbox,
|
||||
["admin"],
|
||||
)
|
||||
assert without_materialized.intersected_zones == ["flanders"]
|
||||
assert without_materialized.items[0].status == "partial"
|
||||
assert without_materialized.items[0].materialized_dataset_ids == []
|
||||
|
||||
dataset_id = uuid4()
|
||||
materialized = SimpleNamespace(
|
||||
id=dataset_id,
|
||||
status="ready",
|
||||
source_name="ngi_adminvector",
|
||||
reference_layer_name="belgium_regions",
|
||||
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
|
||||
)
|
||||
with_materialized = CoverageRegistryService.resolve(
|
||||
FakeSession(project=project, areas=areas, datasets=[materialized]),
|
||||
project_id,
|
||||
bbox,
|
||||
["admin"],
|
||||
)
|
||||
admin_item = next(item for item in with_materialized.items if item.zone == "flanders")
|
||||
assert admin_item.status == "operational"
|
||||
assert admin_item.materialized_dataset_ids == [dataset_id]
|
||||
|
||||
|
||||
def test_mixed_land_and_north_sea_selection_remains_split() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
areas = [
|
||||
scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)),
|
||||
scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)),
|
||||
scope_area("Belgian part of the North Sea", box(2.2, 51.1, 3.4, 51.9)),
|
||||
scope_area("Belgian territorial sea (0-12 nautical miles)", box(2.7, 51.1, 3.4, 51.5)),
|
||||
scope_area("Belgian exclusive economic zone beyond territorial sea", box(2.2, 51.5, 3.1, 51.9)),
|
||||
scope_area("Belgian continental shelf beyond territorial sea", box(2.2, 51.5, 3.1, 51.9)),
|
||||
]
|
||||
|
||||
result = CoverageRegistryService.resolve(
|
||||
FakeSession(project=project, areas=areas, datasets=[]),
|
||||
project_id,
|
||||
CoverageBBox(minx=2.65, miny=51.05, maxx=2.85, maxy=51.2),
|
||||
["admin", "bathymetry"],
|
||||
)
|
||||
|
||||
assert result.intersected_zones == ["flanders", "territorial_sea"]
|
||||
assert len(result.items) == 4
|
||||
assert any("crosses coverage zones" in warning for warning in result.warnings)
|
||||
bathymetry = next(item for item in result.items if item.zone == "territorial_sea" and item.theme == "bathymetry")
|
||||
assert bathymetry.status == "not_configured"
|
||||
assert bathymetry.materialized_dataset_ids == []
|
||||
|
||||
|
||||
def test_flemish_materialization_is_theme_specific() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
orthophoto_id = uuid4()
|
||||
orthophoto = SimpleNamespace(
|
||||
id=orthophoto_id,
|
||||
status="ready",
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
reference_layer_name="orthophoto",
|
||||
source_metadata={"coverage_zones": ["flanders"]},
|
||||
)
|
||||
result = CoverageRegistryService.resolve(
|
||||
FakeSession(
|
||||
project=project,
|
||||
areas=[scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5))],
|
||||
datasets=[orthophoto],
|
||||
),
|
||||
project_id,
|
||||
CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0),
|
||||
["orthophoto", "roads"],
|
||||
)
|
||||
|
||||
assert next(item for item in result.items if item.theme == "orthophoto").status == "operational"
|
||||
assert next(item for item in result.items if item.theme == "roads").status == "partial"
|
||||
|
||||
|
||||
def test_outside_scope_and_unknown_theme_are_explicit() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
db = FakeSession(
|
||||
project=project,
|
||||
areas=[scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5))],
|
||||
datasets=[],
|
||||
)
|
||||
result = CoverageRegistryService.resolve(
|
||||
db,
|
||||
project_id,
|
||||
CoverageBBox(minx=7.0, miny=52.0, maxx=7.1, maxy=52.1),
|
||||
["admin"],
|
||||
)
|
||||
assert result.intersected_zones == []
|
||||
assert result.outside_supported_scope is True
|
||||
assert result.items == []
|
||||
|
||||
try:
|
||||
CoverageRegistryService.resolve(
|
||||
db,
|
||||
project_id,
|
||||
CoverageBBox(minx=4.0, miny=50.0, maxx=4.1, maxy=50.1),
|
||||
["invented_metric"],
|
||||
)
|
||||
except AppError as exc:
|
||||
assert exc.code == "COVERAGE_THEME_UNSUPPORTED"
|
||||
assert exc.status_code == 422
|
||||
assert exc.details["unsupported_themes"] == ["invented_metric"]
|
||||
else:
|
||||
raise AssertionError("Unknown coverage theme was accepted")
|
||||
|
||||
|
||||
def test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbox() -> None:
|
||||
root = Path(__file__).parents[2]
|
||||
focus = (root / "frontend" / "src" / "config" / "primaryFocus.ts").read_text(encoding="utf-8")
|
||||
workspace_hook = (root / "frontend" / "src" / "hooks" / "useProjectWorkspace.ts").read_text(encoding="utf-8")
|
||||
coverage_hook = (root / "frontend" / "src" / "hooks" / "useCoverageResolver.ts").read_text(encoding="utf-8")
|
||||
map_workspace = (root / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "Belgium and North Sea Workbench" in focus
|
||||
assert "nationalProject" in workspace_hook
|
||||
assert "data.areas.length > 0" in workspace_hook
|
||||
assert "dataset.status === 'ready'" in workspace_hook
|
||||
assert "externalApi.resolveCoverage" in coverage_hook
|
||||
assert "coverage.outside_supported_scope" in map_workspace
|
||||
assert "coverageStatusLabel" in map_workspace
|
||||
@@ -0,0 +1,161 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box, mapping, shape
|
||||
|
||||
|
||||
ROOT = Path(__file__).parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import provision_belgium_north_sea_scope as operator # noqa: E402
|
||||
|
||||
|
||||
def marine_feature(identifier: str, geometry):
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": identifier,
|
||||
"geometry": mapping(geometry),
|
||||
"properties": {"MarineReportingUnitId": identifier},
|
||||
}
|
||||
|
||||
|
||||
def test_marine_legal_scopes_are_derived_from_official_reporting_units() -> None:
|
||||
reporting_units = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
marine_feature("ANS-BE-MS-1", box(0, 0, 10, 10)),
|
||||
marine_feature("ANS-BE-AA-CW", box(0, 0, 2, 2)),
|
||||
marine_feature("ANS-BE-AA-TEW", box(0, 2, 2, 4)),
|
||||
marine_feature("ANS-BE-AA-OFFSHORE", box(2, 0, 10, 10)),
|
||||
],
|
||||
}
|
||||
|
||||
payload = operator.derive_marine_scope_payload(reporting_units)
|
||||
by_zone = {
|
||||
feature["properties"]["coverage_zone"]: feature
|
||||
for feature in payload["features"]
|
||||
}
|
||||
|
||||
assert set(by_zone) == {
|
||||
"belgian_north_sea",
|
||||
"territorial_sea",
|
||||
"exclusive_economic_zone",
|
||||
"continental_shelf",
|
||||
}
|
||||
assert shape(by_zone["territorial_sea"]["geometry"]).area == pytest.approx(8.0)
|
||||
assert shape(by_zone["exclusive_economic_zone"]["geometry"]).equals(
|
||||
shape(by_zone["continental_shelf"]["geometry"])
|
||||
)
|
||||
assert (
|
||||
by_zone["exclusive_economic_zone"]["properties"]["legal_domain"]
|
||||
!= by_zone["continental_shelf"]["properties"]["legal_domain"]
|
||||
)
|
||||
assert by_zone["territorial_sea"]["properties"]["derived_from_reporting_unit_ids"] == [
|
||||
"ANS-BE-AA-CW",
|
||||
"ANS-BE-AA-TEW",
|
||||
]
|
||||
|
||||
|
||||
def test_marine_scope_derivation_fails_when_a_required_unit_is_missing() -> None:
|
||||
with pytest.raises(RuntimeError, match="ANS-BE-AA-CW"):
|
||||
operator.derive_marine_scope_payload(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
marine_feature("ANS-BE-MS-1", box(0, 0, 10, 10)),
|
||||
marine_feature("ANS-BE-AA-TEW", box(0, 2, 2, 4)),
|
||||
marine_feature("ANS-BE-AA-OFFSHORE", box(2, 0, 10, 10)),
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_archive_extraction_accepts_one_safe_geopackage_and_rejects_traversal(tmp_path: Path) -> None:
|
||||
archive = tmp_path / "adminvector.zip"
|
||||
with zipfile.ZipFile(archive, "w") as handle:
|
||||
handle.writestr("release/adminvector.gpkg", b"sqlite-bytes")
|
||||
|
||||
result = operator.extract_single_geopackage(archive, tmp_path / "output")
|
||||
assert result.read_bytes() == b"sqlite-bytes"
|
||||
|
||||
unsafe = tmp_path / "unsafe.zip"
|
||||
with zipfile.ZipFile(unsafe, "w") as handle:
|
||||
handle.writestr("../adminvector.gpkg", b"unsafe")
|
||||
with pytest.raises(RuntimeError, match="unsafe"):
|
||||
operator.extract_single_geopackage(unsafe, tmp_path / "unsafe-output")
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
self.content = json.dumps(payload).encode("utf-8")
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, pages):
|
||||
self.pages = list(pages)
|
||||
self.calls = []
|
||||
|
||||
def get(self, url, params, timeout):
|
||||
self.calls.append({"url": url, "params": params, "timeout": timeout})
|
||||
return FakeResponse(self.pages.pop(0))
|
||||
|
||||
|
||||
def test_wfs_fetch_is_allowlisted_paginated_and_complete() -> None:
|
||||
pages = [
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"numberMatched": 2,
|
||||
"features": [{"type": "Feature", "id": "unit.1", "geometry": None, "properties": {}}],
|
||||
},
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"numberMatched": 2,
|
||||
"features": [{"type": "Feature", "id": "unit.2", "geometry": None, "properties": {}}],
|
||||
},
|
||||
]
|
||||
session = FakeSession(pages)
|
||||
payload = operator.fetch_wfs_layer(
|
||||
session,
|
||||
service_url=operator.RBINS_MRU_WFS_URL,
|
||||
layer_name=operator.RBINS_MRU_LAYER,
|
||||
timeout=30,
|
||||
page_size=1,
|
||||
)
|
||||
|
||||
assert [feature["id"] for feature in payload["features"]] == ["unit.1", "unit.2"]
|
||||
assert [call["params"]["startIndex"] for call in session.calls] == [0, 1]
|
||||
assert all(call["params"]["srsName"] == "EPSG:4326" for call in session.calls)
|
||||
|
||||
with pytest.raises(RuntimeError, match="allowlist"):
|
||||
operator.fetch_wfs_layer(
|
||||
FakeSession([]),
|
||||
service_url=operator.RBINS_MSP_WFS_URL,
|
||||
layer_name="untrusted:layer",
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
|
||||
def test_operator_is_packaged_and_guarded_by_readiness() -> None:
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
source = (ROOT / "scripts" / "provision_belgium_north_sea_scope.py").read_text(encoding="utf-8")
|
||||
|
||||
assert "COPY scripts/provision_belgium_north_sea_scope.py /app/scripts/" in dockerfile
|
||||
assert "py_compile scripts/provision_belgium_north_sea_scope.py" in readiness
|
||||
assert "/datasets/upload" in source
|
||||
assert "from app.models" not in source
|
||||
assert "INSERT INTO vector_features" not in source
|
||||
@@ -113,7 +113,10 @@ def test_tower_deploy_uses_single_container_unraid_compose() -> None:
|
||||
for script in (powershell, bash):
|
||||
assert "docker compose -f docker-compose.unraid.yml config" in script
|
||||
assert "--build-arg GEOINTEL_INSTALL_AI=" in script
|
||||
assert "-f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest ." in script
|
||||
assert '--build-arg GEOINTEL_BUILD_SHA="$GEOINTEL_BUILD_SHA"' in script
|
||||
assert '--build-arg GEOINTEL_BUILD_TIME="$GEOINTEL_BUILD_TIME"' in script
|
||||
assert "-f deploy/unraid/Dockerfile.all-in-one" in script
|
||||
assert "-t geointel-all-in-one:latest" in script
|
||||
assert "docker compose -f docker-compose.unraid.yml build geointel" not in script
|
||||
assert "bash deploy/unraid/run-dockerman-container.sh" in script
|
||||
assert "LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh" in script
|
||||
|
||||
Reference in New Issue
Block a user