Add bounded GRB map acquisition
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-17 21:29:26 +02:00
parent 47cde21e17
commit 488da4cc83
28 changed files with 1644 additions and 134 deletions
+11
View File
@@ -14,6 +14,17 @@ ORTHOPHOTO_MAX_SIDE_M=1024
ORTHOPHOTO_CACHE_TTL_HOURS=24
SOURCE_CATALOG_PROBE_ENABLED=true
SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs
GRB_ENABLED=true
GRB_OGC_API_URL=https://geo.api.vlaanderen.be/GRB/ogc/features/v1
GRB_MIN_SIDE_M=10
GRB_MAX_SIDE_M=20000
GRB_PAGE_SIZE=1000
GRB_MAX_PAGES=200
GRB_MAX_FEATURES=150000
GRB_TIMEOUT_SECONDS=180
GRB_MAX_RESPONSE_MB=20
GRB_MAX_TOTAL_RESPONSE_MB=256
GRB_CACHE_TTL_HOURS=24
SOURCE_CATALOG_STATBEL_DCAT_URL=https://doc.statbel.be/publications/DCAT/DCAT_opendata_datasets.ttl
SOURCE_CATALOG_STATBEL_MAX_RESPONSE_MB=5
SOURCE_CATALOG_ALZ_RELEASE_URL=https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen
+17
View File
@@ -7,6 +7,23 @@
# Changelog
## Sprint 239 Governed bounded GRB map acquisition (2026-07-17)
- Added a fixed four-product GRB registry for buildings, roads, water and
administrative parcels through the official OGC API Features service.
- Added bounded acquisition with exact `bbox ∩ Area` clipping, complete
allowlisted pagination, response/feature limits, request and artifact
checksums, 24-hour identity cache and canonical Job/Dataset/VectorFeature
persistence.
- Added semantic selection metrics for building footprint, road length, water
area/supporting line length and parcel area without fabricating traffic,
legal-boundary or water-volume data.
- Extended the Flanders map catalog and all-theme analysis so the four GRB
products are usable on demand without browser-side external calls.
- Updated the provider registry to report the bounded GRB integration as
configured while keeping the old generic import/fetch contract non-fetching
and directing callers to the governed project endpoint.
## Sprint 238 Governed Flanders terrain and flood selection (2026-07-17)
- Unified the Flanders map selection flow behind one official-raster
+32
View File
@@ -1360,6 +1360,38 @@ idempotent for the exact plan/raster checksum, creates a new immutable raster
Dataset and DatasetVersion with the official edition, and retains every older
snapshot. No command is scheduled or invoked by startup or browser actions.
## Governed bounded GRB acquisition
`GET /api/v1/projects/{project_id}/datasets/grb/products` exposes four fixed
official vector products: building footprints, road segments, water
surfaces/lines and administrative parcels. `POST .../datasets/grb/acquire`
accepts an EPSG:4326 rectangle, optional project Area, one product key and an
explicit refresh flag.
The service queries only the allowlisted GRB OGC API collection paths, follows
complete same-host pagination and clips every geometry to `bbox ∩ Area`.
It explicitly requests OGC CRS84 GeoJSON for bbox and output; the native
EPSG:31370 storage CRS remains provenance rather than being guessed from raw
coordinates.
Requests fail closed above 20 km per side, 200 pages, 150,000 retained
features, 20 MiB per page or 256 MiB total. No partial Dataset is persisted
when a limit is exceeded. Official ids, request URLs, page checksums and the
final artifact checksum are retained as provenance.
Persistence uses the existing synchronous Job plus
`DatasetService.import_vector_bytes`, so Dataset, DatasetVersion and
VectorFeature rows remain one canonical flow. The browser never contacts the
provider directly. Exact request identities are reused for 24 hours. Buildings
return footprint area in hectares, roads return line length in kilometres,
water returns surface area plus supporting water-line length, and parcels
return mapped area. GRB cannot provide water volume, legal parcel boundaries
or traffic information.
Settings: `GRB_ENABLED`, `GRB_OGC_API_URL`, `GRB_MIN_SIDE_M`,
`GRB_MAX_SIDE_M`, `GRB_PAGE_SIZE`, `GRB_MAX_PAGES`, `GRB_MAX_FEATURES`,
`GRB_TIMEOUT_SECONDS`, `GRB_MAX_RESPONSE_MB`,
`GRB_MAX_TOTAL_RESPONSE_MB` and `GRB_CACHE_TTL_HOURS`.
## Governed DHMV terrain acquisition
`GET /api/v1/projects/{project_id}/datasets/dhmv/products` exposes the fixed
+26
View File
@@ -31,6 +31,7 @@ from app.schemas import (
BathymetryProfileAcquireRequest,
ThematicRasterAcquireRequest,
ThematicRasterSelectionRequest,
GrbAcquireRequest,
VectorBBoxResponse,
VectorBufferRequest,
VectorClipRequest,
@@ -49,6 +50,7 @@ from app.services.dataset_service import DatasetService
from app.services.source_freshness_service import SourceFreshnessService
from app.services.source_catalog_probe_service import SourceCatalogProbeService
from app.services.grb_refresh_plan_service import GrbRefreshPlanService
from app.services.grb_acquisition_service import GrbAcquisitionService
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.terrain_analysis_service import TerrainAnalysisService
@@ -193,6 +195,30 @@ def list_dhmv_products(project_id: UUID, db: Session = Depends(get_db)):
return envelope({"items": items, "total": len(items)})
@router.post("/datasets/grb/acquire", response_model=dict)
def acquire_bounded_grb(
project_id: UUID,
payload: GrbAcquireRequest,
db: Session = Depends(get_db),
):
job = JobService.run_sync_job(
db=db,
project_id=project_id,
job_type="vector.grb.acquire",
parameters=payload.model_dump(mode="json"),
operation=lambda: GrbAcquisitionService.acquire(db, project_id, payload),
)
return envelope(job)
@router.get("/datasets/grb/products", response_model=dict)
def list_grb_products(project_id: UUID, db: Session = Depends(get_db)):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
items = GrbAcquisitionService.list_products()
return envelope({"items": items, "total": len(items)})
@router.post("/datasets/flood-hazard/acquire", response_model=dict)
def acquire_bounded_flood_hazard(
project_id: UUID,
+1 -1
View File
@@ -39,7 +39,7 @@ def capabilities() -> dict:
geopandas=_dependency_enabled("geopandas"),
yolo=False,
sam=False,
grb="planned",
grb="bounded",
sentinel="planned",
providers=providers,
).model_dump()}
+19
View File
@@ -68,6 +68,25 @@ class Settings(BaseSettings):
le=86_400,
validation_alias="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS",
)
grb_enabled: bool = Field(default=True, validation_alias="GRB_ENABLED")
grb_ogc_api_url: str = Field(
default="https://geo.api.vlaanderen.be/GRB/ogc/features/v1",
validation_alias="GRB_OGC_API_URL",
)
grb_min_side_m: float = Field(default=10.0, gt=0, validation_alias="GRB_MIN_SIDE_M")
grb_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="GRB_MAX_SIDE_M")
grb_page_size: int = Field(default=1000, ge=1, le=1000, validation_alias="GRB_PAGE_SIZE")
grb_max_pages: int = Field(default=200, ge=1, le=1000, validation_alias="GRB_MAX_PAGES")
grb_max_features: int = Field(default=150_000, ge=1, validation_alias="GRB_MAX_FEATURES")
grb_timeout_seconds: int = Field(default=180, ge=1, le=600, validation_alias="GRB_TIMEOUT_SECONDS")
grb_max_response_mb: int = Field(default=20, ge=1, le=100, validation_alias="GRB_MAX_RESPONSE_MB")
grb_max_total_response_mb: int = Field(
default=256,
ge=1,
le=2048,
validation_alias="GRB_MAX_TOTAL_RESPONSE_MB",
)
grb_cache_ttl_hours: int = Field(default=24, ge=0, le=8760, validation_alias="GRB_CACHE_TTL_HOURS")
dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED")
dhmv_wcs_url: str = Field(
default="https://geo.api.vlaanderen.be/DHMV/wcs",
+21 -7
View File
@@ -9,12 +9,26 @@ class GRBProvider(BaseReferenceProvider):
provider_name="grb",
display_name="GRB",
authority_level="authoritative",
supported_layers=["buildings", "roads", "parcels"],
supported_layers=["buildings", "roads", "water", "parcels"],
supported_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"],
supported_query_modes=["area"],
fetch_signature="POST /api/v1/external/grb/fetch",
limitation_message="GRB live WFS/download integration is not configured in Sprint 7B.",
attribution="Digitaal Vlaanderen - Basiskaart Vlaanderen (GRB)",
license_note="Use must follow Digitaal Vlaanderen open data and attribution terms.",
configured=False,
supported_query_modes=["bbox", "persisted_area"],
fetch_signature="POST /api/v1/projects/{project_id}/datasets/grb/acquire",
limitation_message=(
"Alleen expliciet begrensde selecties tot 20 km per zijde worden opgehaald. "
"Volledige providerdownloads en onbeperkte queries zijn niet toegestaan."
),
attribution="Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen",
license_note="Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen.",
configured=True,
)
def fetch(self, project_id: str, area_id: str | None, layers: list[str]) -> dict:
del project_id, area_id, layers
return {
"provider": self.provider_name,
"status": "bounded_request_required",
"message": (
"Use POST /api/v1/projects/{project_id}/datasets/grb/acquire with an EPSG:4326 "
"bounding box and one governed product key."
),
}
+14 -2
View File
@@ -78,11 +78,23 @@ class ExternalProviderRegistry:
del project_id, area_id
provider = self.get(provider_name)
mapping = self.dataset_mapping(provider.provider_name, requested_dataset_role=requested_dataset_role)
if provider.provider_name in {"grb", "osm"}:
if provider.provider_name == "grb":
return ProviderImportResult(
provider_name="grb",
status="bounded_request_required",
message=(
"Use the governed project GRB acquisition endpoint with an EPSG:4326 bounding box "
"and one supported layer."
),
requested_layers=layers,
dataset_role=mapping.dataset_role,
source_name=mapping.source_name,
)
if provider.provider_name == "osm":
return ProviderImportResult(
provider_name=provider.provider_name,
status="not_configured",
message=f"No live {provider.display_name} import is configured in Sprint 7B.",
message=f"No live {provider.display_name} import is configured.",
requested_layers=layers,
dataset_role=mapping.dataset_role,
source_name=mapping.source_name,
+4
View File
@@ -17,6 +17,7 @@ from .source_catalog import (
SourceCatalogProbeSummary,
)
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
from .detection import (
DetectionListResponse,
DetectionModelCapability,
@@ -161,6 +162,9 @@ __all__ = [
"GrbRefreshLayerPlan",
"GrbRefreshPlan",
"GrbRefreshPlanSummary",
"GrbAcquireRequest",
"GrbAcquisitionResult",
"GrbProductRead",
"DetectionListResponse",
"DetectionModelCapability",
"DetectionModelsResponse",
+45
View File
@@ -0,0 +1,45 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel
from .operations import VectorSelectionBBox
class GrbAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str = "buildings"
force_refresh: bool = False
class GrbProductRead(BaseModel):
key: str
display_name: str
reference_layer_name: str
collections: list[str]
geometry_types: list[str]
source_crs: str
authority_level: str
catalog_url: str
attribution: str
license_note: str
limitation_message: str
class GrbAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
reference_layer_name: str
collections: list[str]
feature_count: int
candidate_feature_count: int
page_count: int
bbox_epsg4326: list[float]
source_version: str
attribution: str
limitation_message: str
@@ -0,0 +1,779 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
import hashlib
import json
import math
from pathlib import Path
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
from urllib.request import HTTPRedirectHandler, Request, build_opener
from uuid import UUID
from geoalchemy2.shape import to_shape
from pyproj import Transformer
from shapely.geometry import LineString, MultiLineString, MultiPolygon, Polygon, box, mapping, shape
from shapely.ops import unary_union
from shapely.validation import make_valid
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import Area, Dataset, Project
from app.schemas.grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
from app.services.dataset_service import DatasetService
class _RejectRedirects(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
del req, fp, code, msg, headers, newurl
return None
_NO_REDIRECT_OPENER = build_opener(_RejectRedirects())
@dataclass(frozen=True)
class GrbCollection:
name: str
geometry_dimension: int
@dataclass(frozen=True)
class GrbProduct:
key: str
display_name: str
reference_layer_name: str
layer_type: str
collections: tuple[GrbCollection, ...]
geometry_types: tuple[str, ...]
metric_key: str
metric_method: str
metric_label: str
metric_unit: str
metric_dimension: int
metric_warning: str
limitation_message: str
class GrbAcquisitionService:
PROVIDER = "grb"
SOURCE_CRS = "EPSG:4326"
OGC_CRS84_URI = "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
AUTHORITY_LEVEL = "authoritative"
ATTRIBUTION = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen"
LICENSE_NOTE = "Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen."
CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/basiskaart-vlaanderen-grb"
@staticmethod
def _products() -> dict[str, GrbProduct]:
products = (
GrbProduct(
key="buildings",
display_name="GRB gebouwcontouren",
reference_layer_name="buildings",
layer_type="building",
collections=(GrbCollection("GBG", 2),),
geometry_types=("Polygon", "MultiPolygon"),
metric_key="footprint_area",
metric_method="intersection_area",
metric_label="Bebouwde grondoppervlakte",
metric_unit="ha",
metric_dimension=2,
metric_warning=(
"Dit is de grondoppervlakte van gebouwcontouren, niet de totale vloeroppervlakte "
"of het gebouwvolume."
),
limitation_message=(
"GRB GBG bevat gebouwcontouren uit de basiskaart. Registratie en fysieke verandering "
"kunnen in tijd verschillen."
),
),
GrbProduct(
key="roads",
display_name="GRB wegsegmenten",
reference_layer_name="roads",
layer_type="road",
collections=(GrbCollection("Wegsegment", 1),),
geometry_types=("LineString", "MultiLineString"),
metric_key="road_length",
metric_method="intersection_length",
metric_label="Totale weglengte",
metric_unit="km",
metric_dimension=1,
metric_warning=(
"De lengte volgt GRB-wegsegmenten en zegt niets over rijstroken, verkeersvolume "
"of verhardingsoppervlakte."
),
limitation_message=(
"GRB Wegsegment beschrijft netwerkgeometrie en is geen verkeersmodel of routeadvies."
),
),
GrbProduct(
key="water",
display_name="GRB wateroppervlakken en waterlijnen",
reference_layer_name="water",
layer_type="water",
collections=(
GrbCollection("WTZ", 2),
GrbCollection("WLAS", 1),
GrbCollection("WGR", 1),
),
geometry_types=("LineString", "MultiLineString", "Polygon", "MultiPolygon"),
metric_key="water_area",
metric_method="intersection_area",
metric_label="Wateroppervlakte",
metric_unit="ha",
metric_dimension=2,
metric_warning=(
"Watervolume is niet berekenbaar zonder betrouwbare diepte- of bathymetrische gegevens. "
"De GRB-bron levert alleen oppervlakte- en lijngeometrie."
),
limitation_message=(
"GRB-water combineert wateroppervlakken en watergerelateerde lijnen. Objectaantallen en "
"oppervlakte zijn geen actueel waterpeil of watervolume."
),
),
GrbProduct(
key="parcels",
display_name="GRB administratieve percelen",
reference_layer_name="parcels",
layer_type="parcel",
collections=(GrbCollection("ADP", 2),),
geometry_types=("Polygon", "MultiPolygon"),
metric_key="parcel_area",
metric_method="intersection_area",
metric_label="Perceeloppervlakte",
metric_unit="ha",
metric_dimension=2,
metric_warning=(
"GRB-percelen zijn een grafische referentie en vormen geen juridische grensopmeting."
),
limitation_message=(
"GRB ADP toont de vermoedelijke ligging van kadastrale percelen en is geen juridische grens."
),
),
)
return {product.key: product for product in products}
@staticmethod
def list_products() -> list[dict[str, Any]]:
return [
GrbProductRead(
key=product.key,
display_name=product.display_name,
reference_layer_name=product.reference_layer_name,
collections=[collection.name for collection in product.collections],
geometry_types=list(product.geometry_types),
source_crs=GrbAcquisitionService.SOURCE_CRS,
authority_level=GrbAcquisitionService.AUTHORITY_LEVEL,
catalog_url=GrbAcquisitionService.CATALOG_URL,
attribution=GrbAcquisitionService.ATTRIBUTION,
license_note=GrbAcquisitionService.LICENSE_NOTE,
limitation_message=product.limitation_message,
).model_dump()
for product in GrbAcquisitionService._products().values()
]
@staticmethod
def _product(product_key: str) -> GrbProduct:
product = GrbAcquisitionService._products().get(product_key.strip().lower())
if product is None:
raise AppError(
code="GRB_PRODUCT_NOT_SUPPORTED",
message="Select buildings, roads, water or parcels from the governed GRB product registry",
details={"product_key": product_key},
status_code=422,
)
return product
@staticmethod
def _validate_scope(
db,
project_id: UUID,
payload: GrbAcquireRequest,
settings: Settings,
) -> tuple[Any, list[float], list[float]]:
if not settings.grb_enabled:
raise AppError(code="GRB_NOT_CONFIGURED", message="Bounded GRB acquisition is disabled", status_code=503)
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
if payload.bbox.crs.upper() != "EPSG:4326":
raise AppError(code="GRB_INVALID_CRS", message="GRB acquisition requires EPSG:4326", status_code=400)
values = (payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
if not all(math.isfinite(value) for value in values):
raise AppError(code="GRB_INVALID_BBOX", message="Bounding box values must be finite", status_code=400)
if values[0] >= values[2] or values[1] >= values[3]:
raise AppError(code="GRB_INVALID_BBOX", message="Bounding box has no area", status_code=400)
if values[0] < -180 or values[2] > 180 or values[1] < -90 or values[3] > 90:
raise AppError(code="GRB_INVALID_BBOX", message="Bounding box is outside EPSG:4326", status_code=400)
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
metric_bounds = transformer.transform_bounds(*values, densify_pts=21)
width_m = float(metric_bounds[2] - metric_bounds[0])
height_m = float(metric_bounds[3] - metric_bounds[1])
if width_m < settings.grb_min_side_m or height_m < settings.grb_min_side_m:
raise AppError(
code="GRB_SELECTION_TOO_SMALL",
message=f"Select an area of at least {settings.grb_min_side_m:g} by {settings.grb_min_side_m:g} metres",
status_code=422,
)
if width_m > settings.grb_max_side_m or height_m > settings.grb_max_side_m:
raise AppError(
code="GRB_SELECTION_TOO_LARGE",
message=f"Select an area no larger than {settings.grb_max_side_m:g} by {settings.grb_max_side_m:g} metres",
details={"width_m": width_m, "height_m": height_m},
status_code=422,
)
selection = box(*values)
if payload.area_id is None:
scope_geometry = selection
else:
area = db.get(Area, payload.area_id)
if area is None:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
scope_geometry = to_shape(area.geometry).intersection(selection)
if scope_geometry.is_empty:
raise AppError(
code="GRB_SCOPE_EMPTY",
message="The requested bounding box does not intersect the selected area",
status_code=400,
)
return scope_geometry, [float(value) for value in values], [float(value) for value in metric_bounds]
@staticmethod
def _geometry_dimension(geometry: Any) -> int:
if geometry is None or geometry.is_empty:
return -1
if "Polygon" in geometry.geom_type:
return 2
if "LineString" in geometry.geom_type or geometry.geom_type == "LinearRing":
return 1
if "Point" in geometry.geom_type:
return 0
if hasattr(geometry, "geoms"):
return max((GrbAcquisitionService._geometry_dimension(item) for item in geometry.geoms), default=-1)
return -1
@staticmethod
def _extract_dimension(geometry: Any, expected_dimension: int) -> Any | None:
if geometry is None or geometry.is_empty:
return None
if not geometry.is_valid:
geometry = make_valid(geometry)
parts: list[Any] = []
def collect(candidate: Any) -> None:
if candidate is None or candidate.is_empty:
return
if expected_dimension == 2:
if isinstance(candidate, Polygon):
parts.append(candidate)
return
if isinstance(candidate, MultiPolygon):
parts.extend(item for item in candidate.geoms if not item.is_empty)
return
if expected_dimension == 1:
if isinstance(candidate, LineString):
parts.append(candidate)
return
if isinstance(candidate, MultiLineString):
parts.extend(item for item in candidate.geoms if not item.is_empty)
return
if hasattr(candidate, "geoms"):
for item in candidate.geoms:
collect(item)
collect(geometry)
if not parts:
return None
normalized = unary_union(parts)
if normalized.is_empty:
return None
if not normalized.is_valid:
normalized = make_valid(normalized)
if (
normalized.is_empty
or not normalized.is_valid
or GrbAcquisitionService._geometry_dimension(normalized) != expected_dimension
):
return None
return normalized
@staticmethod
def _collection_url(settings: Settings, collection: GrbCollection, bbox_values: tuple[float, ...]) -> str:
base = settings.grb_ogc_api_url.rstrip("/")
query = urlencode(
{
"f": "application/geo+json",
"limit": str(settings.grb_page_size),
"bbox": ",".join(f"{value:.8f}" for value in bbox_values),
"bbox-crs": GrbAcquisitionService.OGC_CRS84_URI,
"crs": GrbAcquisitionService.OGC_CRS84_URI,
}
)
return f"{base}/collections/{collection.name}/items?{query}"
@staticmethod
def _validated_page_url(
url: str,
settings: Settings,
product: GrbProduct,
bbox_values: tuple[float, ...],
) -> str:
parsed = urlparse(url)
base = urlparse(settings.grb_ogc_api_url)
allowed_paths = {
f"{base.path.rstrip('/')}/collections/{collection.name}/items"
for collection in product.collections
}
if (
parsed.scheme != "https"
or base.scheme != "https"
or parsed.netloc.casefold() != base.netloc.casefold()
or parsed.path not in allowed_paths
):
raise AppError(
code="GRB_PROVIDER_INVALID_PAGINATION",
message="GRB returned a pagination URL outside the governed OGC API allowlist",
status_code=502,
)
query = dict(parse_qsl(parsed.query, keep_blank_values=True))
query.update(
{
"f": "application/geo+json",
"limit": str(settings.grb_page_size),
"bbox": ",".join(f"{value:.8f}" for value in bbox_values),
"bbox-crs": GrbAcquisitionService.OGC_CRS84_URI,
"crs": GrbAcquisitionService.OGC_CRS84_URI,
}
)
return urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", urlencode(query), ""))
@staticmethod
def _read_page(
url: str,
settings: Settings,
opener: Callable[..., Any] | None,
) -> tuple[dict[str, Any], str, int]:
request = Request(
url,
headers={
"Accept": "application/geo+json, application/json",
"User-Agent": "GeoIntel/1.0 bounded-grb-acquisition",
},
)
try:
with (opener or _NO_REDIRECT_OPENER.open)(
request,
timeout=settings.grb_timeout_seconds,
) as response:
limit = settings.grb_max_response_mb * 1024 * 1024
content = response.read(limit + 1)
except HTTPError as exc:
raise AppError(
code="GRB_PROVIDER_HTTP_ERROR",
message="The GRB OGC API returned an HTTP error",
details={"status_code": exc.code},
status_code=502,
) from exc
except (TimeoutError, URLError, OSError) as exc:
raise AppError(
code="GRB_PROVIDER_UNAVAILABLE",
message="The GRB OGC API is unavailable",
status_code=502,
) from exc
if len(content) > limit:
raise AppError(
code="GRB_PROVIDER_RESPONSE_TOO_LARGE",
message="A GRB response page exceeded the configured size limit",
status_code=502,
)
try:
payload = json.loads(content.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AppError(
code="GRB_PROVIDER_INVALID_RESPONSE",
message="The GRB OGC API returned invalid GeoJSON",
status_code=502,
) from exc
if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection":
raise AppError(
code="GRB_PROVIDER_INVALID_RESPONSE",
message="The GRB OGC API returned a non-FeatureCollection response",
status_code=502,
)
return payload, hashlib.sha256(content).hexdigest(), len(content)
@staticmethod
def _next_url(payload: dict[str, Any], current_url: str) -> str | None:
links = payload.get("links")
if not isinstance(links, list):
return None
for link in links:
if isinstance(link, dict) and link.get("rel") == "next" and link.get("href"):
return urljoin(current_url, str(link["href"]))
return None
@staticmethod
def _fetch_features(
product: GrbProduct,
scope_geometry: Any,
bbox_values: tuple[float, ...],
coverage_scope: str,
settings: Settings,
opener: Callable[..., Any] | None,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
retained: list[dict[str, Any]] = []
seen_ids: set[str] = set()
request_urls: list[str] = []
response_sha256: list[str] = []
candidate_feature_count = 0
total_response_bytes = 0
geometry_types: dict[str, int] = {}
collection_counts: dict[str, int] = {item.name: 0 for item in product.collections}
for collection in product.collections:
url: str | None = GrbAcquisitionService._collection_url(settings, collection, bbox_values)
seen_pages: set[str] = set()
while url:
url = GrbAcquisitionService._validated_page_url(url, settings, product, bbox_values)
if url in seen_pages:
raise AppError(
code="GRB_PROVIDER_PAGINATION_LOOP",
message="The GRB OGC API repeated a pagination URL",
status_code=502,
)
if len(request_urls) >= settings.grb_max_pages:
raise AppError(
code="GRB_SELECTION_TOO_LARGE",
message="GRB acquisition exceeded the configured page limit",
details={"max_pages": settings.grb_max_pages},
status_code=422,
)
seen_pages.add(url)
payload, response_hash, response_size = GrbAcquisitionService._read_page(url, settings, opener)
request_urls.append(url)
response_sha256.append(response_hash)
total_response_bytes += response_size
if total_response_bytes > settings.grb_max_total_response_mb * 1024 * 1024:
raise AppError(
code="GRB_PROVIDER_RESPONSE_TOO_LARGE",
message="The complete GRB response exceeded the configured transfer limit",
status_code=502,
)
source_features = payload.get("features")
if not isinstance(source_features, list):
raise AppError(
code="GRB_PROVIDER_INVALID_RESPONSE",
message="The GRB FeatureCollection has no valid feature list",
status_code=502,
)
for source_feature in source_features:
candidate_feature_count += 1
if not isinstance(source_feature, dict):
continue
raw_id = str(source_feature.get("id") or "").strip()
if not raw_id:
raise AppError(
code="GRB_PROVIDER_INVALID_RESPONSE",
message=f"GRB {collection.name} returned a feature without an official identity",
status_code=502,
)
feature_id = f"{collection.name}:{raw_id}"
if feature_id in seen_ids:
continue
seen_ids.add(feature_id)
try:
source_geometry = GrbAcquisitionService._extract_dimension(
shape(source_feature.get("geometry")),
collection.geometry_dimension,
)
except Exception as exc:
raise AppError(
code="GRB_PROVIDER_INVALID_GEOMETRY",
message=f"GRB {collection.name} returned invalid geometry",
status_code=502,
) from exc
if source_geometry is None or not source_geometry.intersects(scope_geometry):
continue
retained_geometry = GrbAcquisitionService._extract_dimension(
source_geometry.intersection(scope_geometry),
collection.geometry_dimension,
)
if retained_geometry is None:
continue
if len(retained) >= settings.grb_max_features:
raise AppError(
code="GRB_SELECTION_TOO_LARGE",
message="GRB selection exceeds the configured feature limit; draw a smaller rectangle",
details={"max_features": settings.grb_max_features},
status_code=422,
)
properties = dict(source_feature.get("properties") or {})
properties.update(
{
"source_name": GrbAcquisitionService.PROVIDER,
"source_collection": collection.name,
"source_feature_id": feature_id,
"reference_layer_name": product.reference_layer_name,
"layer_type": product.layer_type,
"theme": product.key,
"authority_level": GrbAcquisitionService.AUTHORITY_LEVEL,
"coverage_scope": coverage_scope,
"geometry_clipped_to_selection": not scope_geometry.covers(source_geometry),
"attribution": GrbAcquisitionService.ATTRIBUTION,
}
)
retained.append(
{
"type": "Feature",
"id": feature_id,
"geometry": mapping(retained_geometry),
"properties": properties,
}
)
collection_counts[collection.name] += 1
geometry_types[retained_geometry.geom_type] = geometry_types.get(retained_geometry.geom_type, 0) + 1
url = GrbAcquisitionService._next_url(payload, url)
return retained, {
"candidate_feature_count": candidate_feature_count,
"feature_count": len(retained),
"page_count": len(request_urls),
"request_urls": request_urls,
"response_sha256": response_sha256,
"response_size_bytes": total_response_bytes,
"collection_feature_counts": collection_counts,
"geometry_types": geometry_types,
"output_crs": GrbAcquisitionService.OGC_CRS84_URI,
"reference_truncated": False,
}
@staticmethod
def _cached_dataset(
db,
project_id: UUID,
product: GrbProduct,
request_hash: str,
settings: Settings,
) -> Dataset | None:
if settings.grb_cache_ttl_hours <= 0:
return None
candidates = (
db.query(Dataset)
.filter(
Dataset.project_id == project_id,
Dataset.source_name == GrbAcquisitionService.PROVIDER,
Dataset.reference_layer_name == product.reference_layer_name,
Dataset.status == "ready",
)
.order_by(Dataset.imported_at.desc())
.all()
)
cutoff = datetime.now(UTC) - timedelta(hours=settings.grb_cache_ttl_hours)
for candidate in candidates:
provenance = candidate.provenance_metadata if isinstance(candidate.provenance_metadata, dict) else {}
imported_at = candidate.imported_at
if (
provenance.get("request_hash") == request_hash
and candidate.storage_path
and Path(candidate.storage_path).is_file()
and imported_at is not None
and imported_at >= cutoff
):
return candidate
return None
@staticmethod
def _result(
dataset: Dataset,
product: GrbProduct,
*,
reused: bool,
bbox_values: list[float],
) -> dict[str, Any]:
source_metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
provenance = dataset.provenance_metadata if isinstance(dataset.provenance_metadata, dict) else {}
metadata = dataset.metadata_json if isinstance(dataset.metadata_json, dict) else {}
return GrbAcquisitionResult(
output_dataset_id=dataset.id,
reused=reused,
provider=GrbAcquisitionService.PROVIDER,
product_key=product.key,
display_name=product.display_name,
reference_layer_name=product.reference_layer_name,
collections=[collection.name for collection in product.collections],
feature_count=int(metadata.get("feature_count", source_metadata.get("feature_count", 0))),
candidate_feature_count=int(provenance.get("candidate_feature_count", 0)),
page_count=int(provenance.get("page_count", 0)),
bbox_epsg4326=bbox_values,
source_version=str(dataset.source_version or ""),
attribution=GrbAcquisitionService.ATTRIBUTION,
limitation_message=product.limitation_message,
).model_dump(mode="json")
@staticmethod
def acquire(
db,
project_id: UUID,
payload: GrbAcquireRequest,
*,
settings: Settings | None = None,
opener: Callable[..., Any] | None = None,
) -> dict[str, Any]:
resolved_settings = settings or get_settings()
product = GrbAcquisitionService._product(payload.product_key)
scope_geometry, bbox_values, metric_bounds = GrbAcquisitionService._validate_scope(
db,
project_id,
payload,
resolved_settings,
)
request_identity = {
"provider": GrbAcquisitionService.PROVIDER,
"product_key": product.key,
"bbox_epsg4326": [round(value, 8) for value in bbox_values],
"area_id": str(payload.area_id) if payload.area_id else None,
}
request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode()).hexdigest()
if not payload.force_refresh:
cached = GrbAcquisitionService._cached_dataset(
db,
project_id,
product,
request_hash,
resolved_settings,
)
if cached is not None:
return GrbAcquisitionService._result(cached, product, reused=True, bbox_values=bbox_values)
area = db.get(Area, payload.area_id) if payload.area_id else None
coverage_scope = (
"municipality"
if area is not None and area.name.strip().lower().startswith("gemeente ")
else "bounded_selection"
)
features, transfer = GrbAcquisitionService._fetch_features(
product,
scope_geometry,
tuple(scope_geometry.bounds),
coverage_scope,
resolved_settings,
opener,
)
acquired_at = datetime.now(UTC)
source_version = acquired_at.date().isoformat()
feature_collection = {
"type": "FeatureCollection",
"name": product.display_name,
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
"features": features,
}
artifact = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
filename = f"grb_{product.key}_{source_version}_{request_hash[:12]}.geojson"
source_metadata: dict[str, Any] = {
"provider": "Digitaal Vlaanderen",
"service": "OGC API Features",
"product_key": product.key,
"product_display_name": product.display_name,
"collections": [collection.name for collection in product.collections],
"authority_level": GrbAcquisitionService.AUTHORITY_LEVEL,
"theme": product.key,
"layer_type": product.layer_type,
"coverage_scope": coverage_scope,
"geometry_clipped_to_area": payload.area_id is not None,
"geometry_clipped_to_selection": True,
"bbox_epsg4326": bbox_values,
"bbox_epsg31370": metric_bounds,
"feature_count": len(features),
"collection_feature_counts": transfer["collection_feature_counts"],
"identity_stable": True,
"identity_scheme": "grb_ogc_feature_id",
"source_storage_crs": "EPSG:31370",
"requested_output_crs": GrbAcquisitionService.OGC_CRS84_URI,
"selection_aggregation": {
"metric_key": product.metric_key,
"method": product.metric_method,
"label": product.metric_label,
"unit": product.metric_unit,
"geometry_dimension": product.metric_dimension,
"is_estimate": False,
"warning": product.metric_warning,
},
"attribution": GrbAcquisitionService.ATTRIBUTION,
"license_note": GrbAcquisitionService.LICENSE_NOTE,
"catalog_url": GrbAcquisitionService.CATALOG_URL,
"limitation_message": product.limitation_message,
}
if product.key == "water":
source_metadata["selection_metrics"] = [
{
"metric_key": "water_length",
"method": "intersection_length",
"label": "Lengte watergerelateerde lijnen",
"unit": "km",
"geometry_dimension": 1,
"is_estimate": False,
}
]
try:
dataset_response = DatasetService.import_vector_bytes(
db,
project_id=project_id,
area_id=payload.area_id,
filename=filename,
content=artifact,
source="Digitaal Vlaanderen GRB OGC API Features",
source_name=GrbAcquisitionService.PROVIDER,
dataset_role="reference",
reference_layer_name=product.reference_layer_name,
temporal_series_key=f"grb:{product.key}:{request_hash[:24]}",
observed_at=datetime(
acquired_at.year,
acquired_at.month,
acquired_at.day,
tzinfo=UTC,
),
temporal_granularity="snapshot",
source_version=source_version,
source_metadata=source_metadata,
provenance_metadata={
"acquisition": "explicit_bounded_ogc_api_features",
"acquired_at": acquired_at.isoformat(),
"request_hash": request_hash,
"request_urls": transfer["request_urls"],
"response_sha256": transfer["response_sha256"],
"response_size_bytes": transfer["response_size_bytes"],
"page_count": transfer["page_count"],
"candidate_feature_count": transfer["candidate_feature_count"],
"exact_feature_count": transfer["feature_count"],
"reference_truncated": False,
"artifact_sha256": hashlib.sha256(artifact).hexdigest(),
"clipped_to_area_id": str(payload.area_id) if payload.area_id else None,
"scope_geometry_type": scope_geometry.geom_type,
"limitation_message": product.limitation_message,
},
)
except AppError:
raise
except Exception as exc:
raise AppError(
code="GRB_PERSISTENCE_FAILED",
message="The validated GRB selection could not be persisted",
details={"reason": str(exc)},
status_code=500,
) from exc
persisted = db.get(Dataset, dataset_response.id)
if persisted is None:
raise AppError(
code="GRB_PERSISTENCE_FAILED",
message="The persisted GRB dataset could not be reloaded",
status_code=500,
)
return GrbAcquisitionService._result(persisted, product, reused=False, bbox_values=bbox_values)
@@ -54,4 +54,5 @@ def test_detection_lab_surfaces_local_model_asset_selection() -> None:
assert "onSelectModelAsset" in lab
assert "modelAssets={modelAssets}" in app
assert "Officiële referentiebronnen" in provider_panel
assert "live GRB- en OSM-koppelingen staan uit" in provider_panel
assert "GRB is beschikbaar voor expliciet begrensde kaartselecties" in provider_panel
assert "OSM blijft uitgeschakeld" in provider_panel
@@ -10,12 +10,12 @@ def read(path: str) -> str:
def test_flanders_workspace_exposes_governed_thematic_products_on_demand() -> None:
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
product_hook = read("frontend/src/hooks/useOfficialRasterProducts.ts")
product_hook = read("frontend/src/hooks/useOfficialMapProducts.ts")
selection_hook = read("frontend/src/hooks/useMapThemeSelectionInsights.ts")
api = read("frontend/src/services/api/datasets.ts")
assert "activeScopeProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME" in workspace
assert "new Map<DataThemeId, OnDemandRasterProduct>" in workspace
assert "new Map<DataThemeId, OnDemandMapProduct>" in workspace
assert "'Op aanvraag'" in workspace
assert "theme.id === 'space_occupation'" in workspace
assert "setActiveThemeId(fallbackTheme.id)" in workspace
@@ -36,23 +36,23 @@ def test_selection_runs_all_available_themes_and_refreshes_persisted_datasets()
assert "for (const theme of DATA_THEMES)" in workspace
assert "await loadThemeInsights(bbox, availableThemes, areaId)" in workspace
assert "!regionalPartitionedThemeActive && !onDemandRasterThemeActive" in workspace
assert "!regionalPartitionedThemeActive && !onDemandThemeActive" in workspace
assert "onRefreshProjectData" in workspace
assert "selectedProjectId ? loadProjectData(selectedProjectId)" in app
assert "successful.some((item) => item.acquisition)" in selection_hook
assert "await onDatasetsChanged()" in selection_hook
def test_regional_on_demand_rasters_require_a_bounded_drawn_selection() -> None:
def test_regional_on_demand_sources_require_a_bounded_drawn_selection() -> None:
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
assert "regionalOnDemandRasterThemeActive" in workspace
assert "regionalRasterThemeActive || regionalOnDemandRasterThemeActive" in workspace
assert "Teken een begrensde rechthoek voor een regionale rasteranalyse." in workspace
assert "officiële Vlaamse rasters worden begrensd opgehaald, bewaard en hergebruikt" in workspace
assert "regionalOnDemandThemeActive" in workspace
assert "regionalRasterThemeActive || regionalOnDemandThemeActive" in workspace
assert "Teken een begrensde rechthoek voor deze regionale analyse." in workspace
assert "Vlaamse kaartbronnen worden begrensd opgehaald, bewaard en hergebruikt" in workspace
def test_frontend_does_not_contact_the_external_wcs_directly() -> None:
def test_frontend_does_not_contact_external_map_services_directly() -> None:
frontend_sources = "\n".join(
path.read_text(encoding="utf-8")
for path in (ROOT / "frontend/src").rglob("*")
@@ -60,4 +60,5 @@ def test_frontend_does_not_contact_the_external_wcs_directly() -> None:
)
assert "mercatornet.be" not in frontend_sources.casefold()
assert "geo.api.vlaanderen.be" not in frontend_sources.casefold()
assert "GetCoverage" not in frontend_sources
@@ -11,12 +11,13 @@ def read(path: str) -> str:
return (ROOT / path).read_text(encoding="utf-8")
def test_official_raster_catalog_hook_loads_all_governed_registries() -> None:
hook = read("frontend/src/hooks/useOfficialRasterProducts.ts")
def test_official_map_catalog_hook_loads_all_governed_registries() -> None:
hook = read("frontend/src/hooks/useOfficialMapProducts.ts")
assert "datasetsApi.listThematicRasterProducts" in hook
assert "datasetsApi.listDhmvProducts" in hook
assert "datasetsApi.listFloodHazardProducts" in hook
assert "datasetsApi.listGrbProducts" in hook
assert "Promise.all([" in hook
@@ -24,7 +25,7 @@ def test_map_selection_can_acquire_dhmv_and_flood_hazard_products() -> None:
selection_hook = read("frontend/src/hooks/useMapThemeSelectionInsights.ts")
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
assert "'thematic_raster' | 'dhmv' | 'flood_hazard'" in selection_hook
assert "'thematic_raster' | 'dhmv' | 'flood_hazard' | 'grb'" in selection_hook
assert "datasetsApi.acquireDhmv" in selection_hook
assert "datasetsApi.acquireFloodHazard" in selection_hook
assert "datasetsApi.acquireThematicRaster" in selection_hook
@@ -0,0 +1,392 @@
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import parse_qs, urlparse
from uuid import uuid4
import pytest
from fastapi.testclient import TestClient
from geoalchemy2.shape import from_shape
from shapely.geometry import MultiPolygon, Polygon
from app.core.config import Settings
from app.core.errors import AppError
from app.db.session import get_db
from app.main import app
from app.models import Area, Dataset, Job, Project
from app.schemas.grb import GrbAcquireRequest
from app.services.dataset_service import DatasetService
from app.services.grb_acquisition_service import GrbAcquisitionService
ROOT = Path(__file__).resolve().parents[2]
class FakeQuery:
def __init__(self, result=None):
self.result = result
def filter(self, *_args):
return self
def order_by(self, *_args):
return self
def all(self):
return self.result if isinstance(self.result, list) else []
class FakeSession:
def __init__(self, rows=None, query_result=None):
self.rows = rows or {}
self.query_result = query_result
self.added = []
def get(self, model, row_id):
row = self.rows.get((model, row_id))
if row is not None:
return row
return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
def add(self, row):
self.added.append(row)
def commit(self):
return None
def rollback(self):
return None
def refresh(self, row):
return row
def query(self, _model):
return FakeQuery(self.query_result)
class JsonResponse:
def __init__(self, payload):
self.content = json.dumps(payload).encode("utf-8")
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, size=-1):
return self.content if size < 0 else self.content[:size]
def request(*, product_key="buildings", area_id=None, force_refresh=True) -> GrbAcquireRequest:
return GrbAcquireRequest(
bbox={
"min_x": 5.15,
"min_y": 51.18,
"max_x": 5.17,
"max_y": 51.20,
"crs": "EPSG:4326",
},
area_id=area_id,
product_key=product_key,
force_refresh=force_refresh,
)
def polygon_feature(feature_id: str, coordinates) -> dict:
return {
"type": "Feature",
"id": feature_id,
"geometry": {"type": "Polygon", "coordinates": [coordinates]},
"properties": {"source_field": feature_id},
}
def test_grb_registry_exposes_four_governed_products() -> None:
products = {item["key"]: item for item in GrbAcquisitionService.list_products()}
assert set(products) == {"buildings", "roads", "water", "parcels"}
assert products["buildings"]["collections"] == ["GBG"]
assert products["roads"]["collections"] == ["Wegsegment"]
assert products["water"]["collections"] == ["WTZ", "WLAS", "WGR"]
assert products["parcels"]["collections"] == ["ADP"]
assert all(item["authority_level"] == "authoritative" for item in products.values())
def test_grb_fetch_follows_pagination_clips_geometry_and_preserves_official_identity() -> None:
product = GrbAcquisitionService._product("buildings")
settings = Settings(_env_file=None)
pages = []
first = polygon_feature(
"GBG.1",
[(5.155, 51.185), (5.175, 51.185), (5.175, 51.195), (5.155, 51.195), (5.155, 51.185)],
)
second = polygon_feature(
"GBG.2",
[(5.151, 51.181), (5.152, 51.181), (5.152, 51.182), (5.151, 51.182), (5.151, 51.181)],
)
outside = polygon_feature(
"GBG.3",
[(5.3, 51.3), (5.31, 51.3), (5.31, 51.31), (5.3, 51.31), (5.3, 51.3)],
)
def opener(raw_request, timeout):
assert timeout == settings.grb_timeout_seconds
parsed = urlparse(raw_request.full_url)
query = parse_qs(parsed.query)
pages.append(raw_request.full_url)
assert query["bbox-crs"] == [GrbAcquisitionService.OGC_CRS84_URI]
assert query["crs"] == [GrbAcquisitionService.OGC_CRS84_URI]
if query.get("cursor") == ["next"]:
return JsonResponse({"type": "FeatureCollection", "features": [second, outside], "links": []})
return JsonResponse(
{
"type": "FeatureCollection",
"features": [first],
"links": [
{
"rel": "next",
"href": (
"https://geo.api.vlaanderen.be/GRB/ogc/features/v1/"
"collections/GBG/items?cursor=next"
),
}
],
}
)
scope = Polygon(
[(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)]
)
features, transfer = GrbAcquisitionService._fetch_features(
product,
scope,
scope.bounds,
"bounded_selection",
settings,
opener,
)
assert len(pages) == 2
assert transfer["page_count"] == 2
assert transfer["candidate_feature_count"] == 3
assert transfer["feature_count"] == 2
assert transfer["reference_truncated"] is False
assert {feature["id"] for feature in features} == {"GBG:GBG.1", "GBG:GBG.2"}
clipped = next(feature for feature in features if feature["id"] == "GBG:GBG.1")
assert clipped["properties"]["source_feature_id"] == "GBG:GBG.1"
assert clipped["properties"]["geometry_clipped_to_selection"] is True
assert clipped["properties"]["coverage_scope"] == "bounded_selection"
def test_grb_fetch_rejects_untrusted_pagination_and_unbounded_feature_volume() -> None:
product = GrbAcquisitionService._product("buildings")
settings = Settings(_env_file=None)
feature = polygon_feature(
"GBG.1",
[(5.151, 51.181), (5.152, 51.181), (5.152, 51.182), (5.151, 51.182), (5.151, 51.181)],
)
scope = Polygon(
[(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)]
)
def hostile_opener(_request, timeout):
del timeout
return JsonResponse(
{
"type": "FeatureCollection",
"features": [feature],
"links": [{"rel": "next", "href": "https://example.test/private"}],
}
)
with pytest.raises(AppError) as invalid_next:
GrbAcquisitionService._fetch_features(
product,
scope,
scope.bounds,
"bounded_selection",
settings,
hostile_opener,
)
assert invalid_next.value.code == "GRB_PROVIDER_INVALID_PAGINATION"
def oversized_opener(_request, timeout):
del timeout
return JsonResponse(
{
"type": "FeatureCollection",
"features": [
feature,
{**feature, "id": "GBG.2"},
],
"links": [],
}
)
with pytest.raises(AppError) as oversized:
GrbAcquisitionService._fetch_features(
product,
scope,
scope.bounds,
"bounded_selection",
Settings(_env_file=None, GRB_MAX_FEATURES=1),
oversized_opener,
)
assert oversized.value.code == "GRB_SELECTION_TOO_LARGE"
def test_grb_acquisition_rejects_large_scope_before_network_access() -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Vlaanderen")})
payload = GrbAcquireRequest(
bbox={"min_x": 4.0, "min_y": 50.7, "max_x": 5.0, "max_y": 51.7, "crs": "EPSG:4326"},
product_key="buildings",
)
with pytest.raises(AppError) as exc_info:
GrbAcquisitionService.acquire(db, project_id, payload, settings=Settings(_env_file=None))
assert exc_info.value.code == "GRB_SELECTION_TOO_LARGE"
def test_grb_acquisition_persists_via_dataset_service_with_selection_metrics(monkeypatch) -> None:
project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4()
municipality = MultiPolygon(
[
Polygon(
[(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)]
)
]
)
project = Project(id=project_id, name="Vlaanderen")
area = Area(
id=area_id,
project_id=project_id,
name="Gemeente Mol - officieel",
geometry=from_shape(municipality, srid=4326),
)
db = FakeSession({(Project, project_id): project, (Area, area_id): area})
captured = {}
def opener(_request, timeout):
del timeout
parsed = urlparse(_request.full_url)
collection = parsed.path.split("/")[-2]
if collection == "WTZ":
features = [
polygon_feature(
"WTZ.1",
[(5.151, 51.181), (5.16, 51.181), (5.16, 51.19), (5.151, 51.19), (5.151, 51.181)],
)
]
else:
features = []
return JsonResponse({"type": "FeatureCollection", "features": features, "links": []})
def persist(_db, **kwargs):
captured.update(kwargs)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
area_id=area_id,
name=kwargs["filename"],
dataset_type="vector",
source=kwargs["source"],
dataset_role=kwargs["dataset_role"],
source_name=kwargs["source_name"],
reference_layer_name=kwargs["reference_layer_name"],
temporal_series_key=kwargs["temporal_series_key"],
observed_at=kwargs["observed_at"],
source_version=kwargs["source_version"],
source_metadata=kwargs["source_metadata"],
provenance_metadata=kwargs["provenance_metadata"],
metadata_json={"feature_count": 1},
status="ready",
)
db.rows[(Dataset, dataset_id)] = dataset
return SimpleNamespace(id=dataset_id)
monkeypatch.setattr(DatasetService, "import_vector_bytes", persist)
result = GrbAcquisitionService.acquire(
db,
project_id,
request(product_key="water", area_id=area_id),
settings=Settings(_env_file=None),
opener=opener,
)
assert result["output_dataset_id"] == str(dataset_id)
assert result["feature_count"] == 1
assert captured["dataset_role"] == "reference"
assert captured["source_name"] == "grb"
assert captured["reference_layer_name"] == "water"
assert captured["source_metadata"]["coverage_scope"] == "municipality"
assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == "water_area"
assert captured["source_metadata"]["selection_metrics"][0]["metric_key"] == "water_length"
assert captured["provenance_metadata"]["reference_truncated"] is False
collection = json.loads(captured["content"])
assert collection["features"][0]["properties"]["coverage_scope"] == "municipality"
def test_grb_routes_use_canonical_envelopes_and_existing_job_contract(monkeypatch) -> None:
project_id, dataset_id = uuid4(), uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
monkeypatch.setattr(
GrbAcquisitionService,
"acquire",
lambda *_args, **_kwargs: {
"output_dataset_id": str(dataset_id),
"provider": "grb",
"product_key": "buildings",
"feature_count": 2,
},
)
app.dependency_overrides[get_db] = lambda: db
try:
client = TestClient(app)
products_response = client.get(f"/api/v1/projects/{project_id}/datasets/grb/products")
acquire_response = client.post(
f"/api/v1/projects/{project_id}/datasets/grb/acquire",
json=request().model_dump(mode="json"),
)
finally:
app.dependency_overrides.clear()
assert products_response.status_code == 200
assert set(products_response.json()) == {"data"}
assert products_response.json()["data"]["total"] == 4
assert acquire_response.status_code == 200
assert set(acquire_response.json()) == {"data"}
assert acquire_response.json()["data"]["job_type"] == "vector.grb.acquire"
assert acquire_response.json()["data"]["output_dataset_id"] == str(dataset_id)
assert any(isinstance(item, Job) for item in db.added)
def test_system_capabilities_reports_bounded_grb_integration() -> None:
response = TestClient(app).get("/api/v1/system/capabilities")
assert response.status_code == 200
assert response.json()["data"]["grb"] == "bounded"
grb = next(
item for item in response.json()["data"]["providers"]
if item["provider_name"] == "grb"
)
assert grb["status"] == "configured"
assert grb["fetch_signature"].endswith("/datasets/grb/acquire")
def test_grb_frontend_and_contracts_use_only_the_governed_backend_path() -> None:
selection_hook = (ROOT / "frontend/src/hooks/useMapThemeSelectionInsights.ts").read_text(encoding="utf-8")
catalog_hook = (ROOT / "frontend/src/hooks/useOfficialMapProducts.ts").read_text(encoding="utf-8")
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
contracts = (ROOT / "docs/API_CONTRACTS.md").read_text(encoding="utf-8")
assert "datasetsApi.acquireGrb" in selection_hook
assert "datasetsApi.listGrbProducts" in catalog_hook
assert "officialMapProducts.grb" in workspace
assert "/datasets/grb/acquire" in contracts
assert "geo.api.vlaanderen.be" not in workspace
@@ -286,10 +286,10 @@ def test_provider_capabilities_expose_sprint7a_contract() -> None:
assert capabilities["osm"]["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"]
assert capabilities["osm"]["supported_query_modes"] == ["area"]
assert capabilities["osm"]["status"] == "not_configured"
assert capabilities["grb"]["supported_layers"] == ["buildings", "roads", "parcels"]
assert capabilities["grb"]["supported_layers"] == ["buildings", "roads", "water", "parcels"]
assert capabilities["grb"]["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"]
assert capabilities["grb"]["supported_query_modes"] == ["area"]
assert capabilities["grb"]["status"] == "not_configured"
assert capabilities["grb"]["supported_query_modes"] == ["bbox", "persisted_area"]
assert capabilities["grb"]["status"] == "configured"
def test_sprint7a_migration_declares_foundation_tables_and_indexes() -> None:
@@ -16,8 +16,8 @@ def test_provider_registry_lists_sprint7b_providers() -> None:
assert set(providers) == {"grb", "osm", "manual", "fixture"}
assert providers["grb"].authority_level == "authoritative"
assert providers["grb"].configured is False
assert providers["grb"].status == "not_configured"
assert providers["grb"].configured is True
assert providers["grb"].status == "configured"
assert providers["osm"].authority_level == "contextual"
assert providers["osm"].configured is False
assert providers["osm"].status == "not_configured"
@@ -34,9 +34,9 @@ def test_provider_capabilities_include_required_metadata() -> None:
assert grb["provider_name"] == "grb"
assert grb["display_name"] == "GRB"
assert grb["supported_layers"] == ["buildings", "roads", "parcels"]
assert grb["supported_layers"] == ["buildings", "roads", "water", "parcels"]
assert grb["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"]
assert grb["supported_query_modes"] == ["area"]
assert grb["supported_query_modes"] == ["bbox", "persisted_area"]
assert grb["limitation_message"]
assert grb["attribution"]
assert grb["license_note"]
@@ -56,13 +56,13 @@ def test_provider_to_dataset_mapping_is_enforced() -> None:
assert get_provider_dataset_mapping("osm", requested_dataset_role="reference").dataset_role == "reference"
def test_grb_osm_import_contract_returns_not_configured_without_fetching() -> None:
def test_grb_import_contract_requires_bounded_request_and_osm_remains_not_configured() -> None:
grb = import_provider_dataset("grb", project_id="project", area_id="area", layers=["buildings"])
osm = import_provider_dataset("osm", project_id="project", area_id="area", layers=["buildings"])
assert grb.status == "not_configured"
assert grb.status == "bounded_request_required"
assert grb.dataset_id is None
assert "No live" in grb.message
assert "bounding box" in grb.message
assert osm.status == "not_configured"
assert osm.dataset_id is None
@@ -107,7 +107,7 @@ def test_provider_api_envelopes_and_invalid_provider() -> None:
assert invalid_response.json()["message"] == "Provider not found"
def test_provider_import_api_returns_clear_not_configured_response() -> None:
def test_provider_import_api_points_grb_to_governed_bounded_endpoint() -> None:
client = TestClient(app)
response = client.post(
@@ -121,7 +121,7 @@ def test_provider_import_api_returns_clear_not_configured_response() -> None:
assert response.status_code == 200
assert response.json()["data"]["provider_name"] == "grb"
assert response.json()["data"]["status"] == "not_configured"
assert response.json()["data"]["status"] == "bounded_request_required"
assert response.json()["data"]["dataset_id"] is None
+11
View File
@@ -27,6 +27,17 @@ services:
ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24}
SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true}
SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}
GRB_ENABLED: ${GRB_ENABLED:-true}
GRB_OGC_API_URL: ${GRB_OGC_API_URL:-https://geo.api.vlaanderen.be/GRB/ogc/features/v1}
GRB_MIN_SIDE_M: ${GRB_MIN_SIDE_M:-10}
GRB_MAX_SIDE_M: ${GRB_MAX_SIDE_M:-20000}
GRB_PAGE_SIZE: ${GRB_PAGE_SIZE:-1000}
GRB_MAX_PAGES: ${GRB_MAX_PAGES:-200}
GRB_MAX_FEATURES: ${GRB_MAX_FEATURES:-150000}
GRB_TIMEOUT_SECONDS: ${GRB_TIMEOUT_SECONDS:-180}
GRB_MAX_RESPONSE_MB: ${GRB_MAX_RESPONSE_MB:-20}
GRB_MAX_TOTAL_RESPONSE_MB: ${GRB_MAX_TOTAL_RESPONSE_MB:-256}
GRB_CACHE_TTL_HOURS: ${GRB_CACHE_TTL_HOURS:-24}
SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10}
SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2}
SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900}
+11
View File
@@ -32,6 +32,17 @@ services:
ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24}
SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true}
SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs}
GRB_ENABLED: ${GRB_ENABLED:-true}
GRB_OGC_API_URL: ${GRB_OGC_API_URL:-https://geo.api.vlaanderen.be/GRB/ogc/features/v1}
GRB_MIN_SIDE_M: ${GRB_MIN_SIDE_M:-10}
GRB_MAX_SIDE_M: ${GRB_MAX_SIDE_M:-20000}
GRB_PAGE_SIZE: ${GRB_PAGE_SIZE:-1000}
GRB_MAX_PAGES: ${GRB_MAX_PAGES:-200}
GRB_MAX_FEATURES: ${GRB_MAX_FEATURES:-150000}
GRB_TIMEOUT_SECONDS: ${GRB_TIMEOUT_SECONDS:-180}
GRB_MAX_RESPONSE_MB: ${GRB_MAX_RESPONSE_MB:-20}
GRB_MAX_TOTAL_RESPONSE_MB: ${GRB_MAX_TOTAL_RESPONSE_MB:-256}
GRB_CACHE_TTL_HOURS: ${GRB_CACHE_TTL_HOURS:-24}
SOURCE_CATALOG_STATBEL_DCAT_URL: ${SOURCE_CATALOG_STATBEL_DCAT_URL:-https://doc.statbel.be/publications/DCAT/DCAT_opendata_datasets.ttl}
SOURCE_CATALOG_STATBEL_MAX_RESPONSE_MB: ${SOURCE_CATALOG_STATBEL_MAX_RESPONSE_MB:-5}
SOURCE_CATALOG_ALZ_RELEASE_URL: ${SOURCE_CATALOG_ALZ_RELEASE_URL:-https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen}
+66 -18
View File
@@ -64,23 +64,23 @@ Returns enabled feature flags and tool availability.
"geopandas": true,
"yolo": false,
"sam": false,
"grb": "planned",
"grb": "bounded",
"sentinel": "planned",
"providers": [
{
"provider_name": "grb",
"display_name": "GRB",
"authority_level": "authoritative",
"supported_layers": ["buildings", "roads", "parcels"],
"supported_layers": ["buildings", "roads", "water", "parcels"],
"supported_geometry_types": ["Polygon", "MultiPolygon", "LineString", "MultiLineString"],
"supported_query_modes": ["area"],
"fetch_signature": "POST /api/v1/external/grb/fetch",
"configured": false,
"status": "not_configured",
"limitation_message": "GRB live WFS/download integration is not configured in Sprint 7B.",
"attribution": "Digitaal Vlaanderen - Basiskaart Vlaanderen (GRB)",
"license_note": "Use must follow Digitaal Vlaanderen open data and attribution terms.",
"not_configured_reason": "Provider integration is not configured yet"
"supported_query_modes": ["bbox", "persisted_area"],
"fetch_signature": "POST /api/v1/projects/{project_id}/datasets/grb/acquire",
"configured": true,
"status": "configured",
"limitation_message": "Alleen expliciet begrensde selecties tot 20 km per zijde worden opgehaald.",
"attribution": "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen",
"license_note": "Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen.",
"not_configured_reason": null
}
]
}
@@ -233,6 +233,45 @@ dimensions, EPSG:4326/EPSG:31370 bounds, sampling resolution, attribution,
cache reuse and limitation text. Historical products also persist their
observation/validity period and a spatially scoped temporal-series key.
### GET `/api/v1/projects/{project_id}/datasets/grb/products`
Returns the fixed official GRB vector registry in the canonical envelope.
Only `buildings` (`GBG`), `roads` (`Wegsegment`), `water`
(`WTZ`/`WLAS`/`WGR`) and `parcels` (`ADP`) are exposed. Each product reports
its exact source collections, supported geometry types, attribution, licence
and semantic limitation.
### POST `/api/v1/projects/{project_id}/datasets/grb/acquire`
Acquires one explicitly bounded GRB product from the allowlisted Digitaal
Vlaanderen OGC API Features service behind the synchronous Job abstraction:
```json
{
"bbox": {"min_x": 5.12, "min_y": 51.18, "max_x": 5.17, "max_y": 51.22, "crs": "EPSG:4326"},
"area_id": "optional-project-area-uuid",
"product_key": "buildings",
"force_refresh": false
}
```
The backend requires EPSG:4326, intersects the rectangle with the optional
persisted Area, rejects sides below 10 m or above 20 km, follows only
same-host allowlisted collection pagination and fails instead of truncating
when page, transfer or feature limits are reached. Every official feature id,
request URL, response checksum and artifact checksum remains provenance.
Although GRB is stored natively in EPSG:31370, the service explicitly
negotiates OGC CRS84 longitude/latitude GeoJSON for both bbox and output before
performing the exact geometry intersection.
Output is an ordinary reference Dataset persisted through `DatasetService`,
`DatasetVersion` and `VectorFeatureService`; providers never write directly
to `vector_features`. Exact requests are reused for 24 hours unless
`force_refresh=true`. Selection metrics are footprint hectares for buildings,
line kilometres for roads, hectares plus supporting line kilometres for
water, and hectares for parcels. Water volume is unsupported because GRB
contains no depth or bathymetry.
### GET `/api/v1/projects/{project_id}/datasets/dhmv/products`
Returns the fixed official DHMV II product registry in the canonical envelope.
@@ -898,7 +937,8 @@ Returns configured/status/limitation fields.
### POST `/api/v1/external/providers/{provider_name}/import`
Defines the future provider import contract. Sprint 7B does not perform live imports or write datasets.
Compatibility contract for provider-specific import flows. It never writes
datasets itself.
Request:
@@ -911,13 +951,13 @@ Request:
}
```
GRB/OSM response:
GRB response:
```json
{
"provider_name": "grb",
"status": "not_configured",
"message": "No live GRB import is configured in Sprint 7B.",
"status": "bounded_request_required",
"message": "Use the governed project GRB acquisition endpoint with an EPSG:4326 bounding box and one supported layer.",
"requested_layers": ["buildings"],
"dataset_id": null,
"dataset_role": "reference",
@@ -925,7 +965,10 @@ GRB/OSM response:
}
```
Manual and fixture providers point callers to existing upload/fixture flows. No provider writes directly to `vector_features`; all future provider output must flow through `DatasetService` and `VectorFeatureService`.
OSM still returns `not_configured`. Manual and fixture providers point callers
to existing upload/fixture flows. No provider writes directly to
`vector_features`; the bounded GRB integration and all future provider output
flow through `DatasetService` and `VectorFeatureService`.
## External data fetchers
@@ -953,7 +996,10 @@ Request:
}
```
V1 may initially implement this as a service interface with a clear `not_configured` response until the exact WFS endpoint is wired.
This legacy compatibility endpoint never fetches. It returns
`bounded_request_required` and directs callers to
`POST /api/v1/projects/{project_id}/datasets/grb/acquire`, where bbox, Area,
product allowlist, limits and persistence are enforceable.
Sprint 7B provider contract responses expose capabilities only. Providers must report:
@@ -973,7 +1019,8 @@ Sprint 7B provider contract responses expose capabilities only. Providers must r
}
```
No GRB WFS, OSM Overpass or provider downloads are implemented in Sprint 7B.
No unbounded GRB WFS download or OSM Overpass integration is implemented.
Bounded GRB OGC API Features acquisition is documented under Datasets.
## Demo workflow
@@ -1518,7 +1565,8 @@ GeoJSON feature properties include:
Limitations:
- No live GRB/OSM/Sentinel fetching.
- Change detection itself performs no provider fetch. Bounded GRB acquisition
is a separate explicit Dataset operation; OSM and Sentinel remain disabled.
- No fake object lifecycle classification.
- No `changed` classification without durable object ids/versioning.
- No first-class change table yet; the current output is stored in job
+32
View File
@@ -10222,3 +10222,35 @@ Validation:
(49 tests).
- Frontend TypeScript typecheck and production build passed before the full
release gate.
## Sprint 239 - Governed bounded GRB map acquisition (2026-07-17)
Implemented:
- Added a server-allowlisted GRB OGC API Features registry for `GBG`,
`Wegsegment`, `WTZ`/`WLAS`/`WGR` and `ADP`, exposed as buildings, roads,
water and parcels.
- Added canonical product and acquisition endpoints. Acquisition validates
EPSG:4326 and metric side lengths, intersects the request with the persisted
Area, follows only trusted pagination, fails instead of truncating and
retains response/artifact checksums.
- Routed all output through the existing synchronous Job and
`DatasetService.import_vector_bytes` flow. The resulting reference Datasets,
DatasetVersions and PostGIS VectorFeatures use the same persistence boundary
as uploaded data.
- Added explicit semantic selection metadata for hectares and kilometres.
Water volume, legal parcel certainty and traffic semantics remain
unsupported.
- Extended the Flanders map product catalog with the four GRB products and
kept all provider traffic behind the backend.
Validation:
- Focused provider, acquisition, route-envelope and frontend contract tests
cover complete pagination, exact clipping, official identity, hostile next
links, feature limits, persistent metadata and the canonical Job response.
- A read-only live provider probe over one small Mol rectangle returned 8
`GBG` buildings, 23 `Wegsegment` roads, 20 water features across
`WTZ`/`WLAS`/`WGR` and 96 `ADP` parcels. Every collection completed without
truncation using explicit OGC CRS84 bbox and output negotiation.
- The complete readiness gate passed 943 backend tests, backend compilation,
the 118-route API contract audit, Alembic head `202607160001`, frontend
TypeScript typecheck and the production Vite build.
+10
View File
@@ -607,6 +607,16 @@ browser never contacts WCS directly. An exact repeat reuses the persisted
request. A complete-Flanders raster request remains disabled by the regional
size guard, while a complete municipality remains a valid bounded analysis.
The same explicit-selection workflow exposes GRB buildings, roads, water and
parcels. These are vector products rather than rasters: selecting a
municipality or drawing a rectangle starts a bounded backend OGC API request,
persists the complete clipped result and analyzes the persisted
`vector_features`. Cards show `Op aanvraag` before first use and measured
hectares or kilometres afterwards. Full-Flanders acquisition remains disabled
by the 20 km side limit; users select a municipality or smaller rectangle.
The map never calls Digitaal Vlaanderen directly and never displays a
silently truncated reference layer.
Selecting `Hoogte & reliëf` shows the source distinction before analysis:
DTM represents terrain after removal of buildings and other objects; DSM
represents the visible surface including buildings and vegetation. Selecting
+67 -60
View File
@@ -2,7 +2,7 @@ import { useEffect, useMemo, useState } from 'react'
import GeoMap from '../GeoMap'
import type { AreaRead, DatasetCreateResponse, DetectionQaResult, MapResultExportRequest, MapViewportState, OrthophotoAcquisitionResult, OrthophotoProductRead, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionMetric, VectorSelectionResponse } from '../../types'
import { useMapThemeSelectionInsights, type MapThemeAcquisition, type MapThemeQuery } from '../../hooks/useMapThemeSelectionInsights'
import { useOfficialRasterProducts } from '../../hooks/useOfficialRasterProducts'
import { useOfficialMapProducts } from '../../hooks/useOfficialMapProducts'
import { useTemporalComparison } from '../../hooks/useTemporalComparison'
import { getDatasetDisplayName, getDatasetSourceDisplayName } from '../../lib/datasetDisplay'
import { TemporalTrendChart } from './TemporalTrendChart'
@@ -54,10 +54,9 @@ interface TemporalSeriesGroup {
items: DatasetCreateResponse[]
}
interface OnDemandRasterProduct extends MapThemeAcquisition {
interface OnDemandMapProduct extends MapThemeAcquisition {
theme: DataThemeId
nativeResolutionM: number
referenceLabel: string
availabilityLabel: string
attribution: string
limitationMessage: string
}
@@ -669,10 +668,10 @@ export function MapWorkspace({
clearThemeInsights,
} = useMapThemeSelectionInsights<DataThemeId>(selectedProjectId, onRefreshProjectData)
const {
products: officialRasterProducts,
loading: officialRasterProductsLoading,
error: officialRasterProductsError,
} = useOfficialRasterProducts(flandersScopeSelected ? selectedProjectId : null)
products: officialMapProducts,
loading: officialMapProductsLoading,
error: officialMapProductsError,
} = useOfficialMapProducts(flandersScopeSelected ? selectedProjectId : null)
const {
temporalComparison,
temporalComparisonLoading,
@@ -760,10 +759,10 @@ export function MapWorkspace({
)
if (selectedFloodHazard) {
result.flood_hazard = selectedFloodHazard
} else if (flandersScopeSelected && officialRasterProducts.floodHazard.length > 0) {
} else if (flandersScopeSelected && officialMapProducts.floodHazard.length > 0) {
result.flood_hazard = null
}
if (flandersScopeSelected && officialRasterProducts.dhmv.length > 0) {
if (flandersScopeSelected && officialMapProducts.dhmv.length > 0) {
result.elevation = availableMapDatasets.find(
(dataset) =>
dataset.source_name === 'digitaal_vlaanderen_dhmv'
@@ -776,8 +775,8 @@ export function MapWorkspace({
availableMapDatasets,
flandersScopeSelected,
floodHazardDatasets,
officialRasterProducts.dhmv.length,
officialRasterProducts.floodHazard.length,
officialMapProducts.dhmv.length,
officialMapProducts.floodHazard.length,
regionalScopeSelected,
selectedDhmvProductKey,
selectedFloodHazardDatasetId,
@@ -800,38 +799,47 @@ export function MapWorkspace({
[availableMapDatasets, regionalScopeSelected, selectedMapAreaId, themeDatasetMap],
)
const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0]
const onDemandRasterProductMap = useMemo(
const onDemandProductMap = useMemo(
() => {
const result = new Map<DataThemeId, OnDemandRasterProduct>()
const result = new Map<DataThemeId, OnDemandMapProduct>()
if (!flandersScopeSelected) {
return result
}
for (const product of officialRasterProducts.thematic) {
for (const product of officialMapProducts.thematic) {
result.set(product.theme, {
kind: 'thematic_raster',
productKey: product.key,
displayName: product.display_name,
theme: product.theme,
nativeResolutionM: product.native_resolution_m,
referenceLabel: String(product.observation_year),
availabilityLabel: `${product.native_resolution_m} m · ${product.observation_year} · laad bij selectie`,
attribution: product.attribution,
limitationMessage: product.limitation_message,
})
}
const dhmvProduct = officialRasterProducts.dhmv.find((product) => product.key === selectedDhmvProductKey)
for (const product of officialMapProducts.grb) {
result.set(product.key, {
kind: 'grb',
productKey: product.key,
displayName: product.display_name,
theme: product.key,
availabilityLabel: 'officiële vectorbron · laad bij selectie',
attribution: product.attribution,
limitationMessage: product.limitation_message,
})
}
const dhmvProduct = officialMapProducts.dhmv.find((product) => product.key === selectedDhmvProductKey)
if (dhmvProduct) {
result.set('elevation', {
kind: 'dhmv',
productKey: dhmvProduct.key,
displayName: dhmvProduct.display_name,
theme: 'elevation',
nativeResolutionM: dhmvProduct.native_resolution_m,
referenceLabel: dhmvProduct.acquisition_period,
availabilityLabel: `${dhmvProduct.native_resolution_m} m · ${dhmvProduct.acquisition_period} · laad bij selectie`,
attribution: dhmvProduct.attribution,
limitationMessage: dhmvProduct.limitation_message,
})
}
const floodProduct = officialRasterProducts.floodHazard.find(
const floodProduct = officialMapProducts.floodHazard.find(
(product) => product.key === selectedFloodHazardProductKey,
)
if (floodProduct) {
@@ -840,8 +848,7 @@ export function MapWorkspace({
productKey: floodProduct.key,
displayName: floodProduct.display_name,
theme: 'flood_hazard',
nativeResolutionM: floodProduct.native_resolution_m,
referenceLabel: `${floodProduct.climate_context} · T${floodProduct.return_period_years}`,
availabilityLabel: `${floodProduct.native_resolution_m} m · ${floodProduct.climate_context} · T${floodProduct.return_period_years} · laad bij selectie`,
attribution: floodProduct.attribution,
limitationMessage: floodProduct.limitation_message,
})
@@ -850,12 +857,12 @@ export function MapWorkspace({
},
[
flandersScopeSelected,
officialRasterProducts,
officialMapProducts,
selectedDhmvProductKey,
selectedFloodHazardProductKey,
],
)
const activeOnDemandRasterProduct = onDemandRasterProductMap.get(activeTheme.id) ?? null
const activeOnDemandMapProduct = onDemandProductMap.get(activeTheme.id) ?? null
const activeThemeMapStyle = DATA_THEME_MAP_STYLES[activeTheme.id]
const analysisOverlayActive = mapContentMode === 'analysis' && analysisLayerAvailable && Boolean(mapFeatureCollection)
const selectedOrthophotoProduct = orthophotoProducts.find((item) => item.key === selectedOrthophotoProductKey) ?? null
@@ -875,24 +882,24 @@ export function MapWorkspace({
const regionalRasterThemeActive = regionalScopeSelected && isPartitionedRaster(activeThemeDataset)
const regionalBathymetryThemeActive = regionalScopeSelected && isPartitionedBathymetry(activeThemeDataset)
const regionalPartitionedThemeActive = regionalRasterThemeActive || regionalBathymetryThemeActive
const onDemandRasterThemeActive = analysisMode === 'current' && Boolean(activeOnDemandRasterProduct)
const regionalOnDemandRasterThemeActive = regionalScopeSelected && onDemandRasterThemeActive
const activeThemeAvailable = Boolean(activeThemeDataset) || onDemandRasterThemeActive
const onDemandThemeActive = analysisMode === 'current' && Boolean(activeOnDemandMapProduct)
const regionalOnDemandThemeActive = regionalScopeSelected && onDemandThemeActive
const activeThemeAvailable = Boolean(activeThemeDataset) || onDemandThemeActive
useEffect(() => {
if (
!flandersScopeSelected
|| analysisMode !== 'current'
|| activeThemeAvailable
|| (onDemandRasterProductMap.size === 0 && !officialRasterProductsError)
|| (onDemandProductMap.size === 0 && !officialMapProductsError)
) {
return
}
const fallbackTheme = DATA_THEMES.find((theme) =>
theme.id === 'space_occupation'
&& Boolean(themeDatasetMap[theme.id] || onDemandRasterProductMap.get(theme.id)),
&& Boolean(themeDatasetMap[theme.id] || onDemandProductMap.get(theme.id)),
) ?? DATA_THEMES.find((theme) =>
Boolean(themeDatasetMap[theme.id] || onDemandRasterProductMap.get(theme.id)),
Boolean(themeDatasetMap[theme.id] || onDemandProductMap.get(theme.id)),
)
if (!fallbackTheme) {
return
@@ -906,8 +913,8 @@ export function MapWorkspace({
activeThemeAvailable,
analysisMode,
flandersScopeSelected,
officialRasterProductsError,
onDemandRasterProductMap,
officialMapProductsError,
onDemandProductMap,
onOpenDatasetInMap,
themeDatasetMap,
])
@@ -999,12 +1006,12 @@ export function MapWorkspace({
useEffect(() => {
const contextSourceLabel = analysisMode === 'evolution' ? activeTemporalSeriesGroup?.label ?? null
: regionalBathymetryThemeActive ? 'VHA-dwarsprofielen Vlaanderen'
: activeOnDemandRasterProduct?.displayName ?? null
: activeOnDemandMapProduct?.displayName ?? null
onSetContextSourceLabel(contextSourceLabel)
return () => onSetContextSourceLabel(null)
}, [
activeTemporalSeriesGroup?.label,
activeOnDemandRasterProduct?.displayName,
activeOnDemandMapProduct?.displayName,
analysisMode,
onSetContextSourceLabel,
regionalBathymetryThemeActive,
@@ -1112,7 +1119,7 @@ export function MapWorkspace({
}
return
}
if (flandersScopeSelected && officialRasterProducts.floodHazard.length > 0) {
if (flandersScopeSelected && officialMapProducts.floodHazard.length > 0) {
setSelectedFloodHazardDatasetId('')
return
}
@@ -1126,7 +1133,7 @@ export function MapWorkspace({
}, [
flandersScopeSelected,
floodHazardDatasets,
officialRasterProducts.floodHazard.length,
officialMapProducts.floodHazard.length,
selectedFloodHazardDatasetId,
selectedFloodHazardProductKey,
])
@@ -1350,7 +1357,7 @@ export function MapWorkspace({
? temporalGroup?.items[temporalGroup.items.length - 1] ?? null
: themeDatasetMap[theme.id]
const onDemandProduct = analysisMode === 'current'
? onDemandRasterProductMap.get(theme.id)
? onDemandProductMap.get(theme.id)
: null
if (!dataset && !onDemandProduct) {
return
@@ -1381,7 +1388,7 @@ export function MapWorkspace({
const availableThemes: Array<MapThemeQuery<DataThemeId>> = []
for (const theme of DATA_THEMES) {
const onDemandProduct = analysisMode === 'current'
? onDemandRasterProductMap.get(theme.id)
? onDemandProductMap.get(theme.id)
: null
if (onDemandProduct) {
availableThemes.push({
@@ -1410,7 +1417,7 @@ export function MapWorkspace({
const analyzeSelection = async (bbox: VectorSelectionBBox, areaId?: string) => {
setSelectionBbox(bbox)
const tasks: Array<Promise<unknown>> = [loadAllThemeResults(bbox, areaId)]
if (!regionalPartitionedThemeActive && !onDemandRasterThemeActive) {
if (!regionalPartitionedThemeActive && !onDemandThemeActive) {
tasks.push(onRunMapSelectionExtract(bbox, areaId))
}
if (analysisMode === 'evolution' && earlierDatasetId && laterDatasetId) {
@@ -1571,7 +1578,7 @@ export function MapWorkspace({
<div className="geo-theme-list">
{DATA_THEMES.map((theme) => {
const dataset = themeDatasetMap[theme.id]
const onDemandProduct = onDemandRasterProductMap.get(theme.id)
const onDemandProduct = onDemandProductMap.get(theme.id)
const partitions = themePartitionMap[theme.id]
const temporalGroups = themeTemporalSeriesMap[theme.id]
const temporalGroup = temporalGroups[0]
@@ -1603,7 +1610,7 @@ export function MapWorkspace({
: dataset
? datasetAvailabilityLabel(dataset, partitions)
: onDemandProduct
? `${onDemandProduct.nativeResolutionM} m · ${onDemandProduct.referenceLabel} · laad bij selectie`
? onDemandProduct.availabilityLabel
: 'Bron nog niet ingeladen'}
</small>
</span>
@@ -1628,7 +1635,7 @@ export function MapWorkspace({
? 'VHA-dwarsprofielen Vlaanderen'
: activeThemeDataset
? getDatasetDisplayName(activeThemeDataset)
: activeOnDemandRasterProduct?.displayName ?? 'Geen databron beschikbaar'}
: activeOnDemandMapProduct?.displayName ?? 'Geen databron beschikbaar'}
</strong>
<small>
{analysisOverlayActive
@@ -1641,19 +1648,19 @@ export function MapWorkspace({
? `${activeThemePartitions.length} gecontroleerde gemeentepartities · selectie wordt ruimtelijk samengevoegd`
: activeThemeDataset
? `${getDatasetSourceDisplayName(activeThemeDataset)} · ${formatDatasetObservation(activeThemeDataset)}`
: activeOnDemandRasterProduct
? `${activeOnDemandRasterProduct.attribution} · wordt alleen voor de gekozen selectie ingeladen`
: activeOnDemandMapProduct
? `${activeOnDemandMapProduct.attribution} · wordt alleen voor de gekozen selectie ingeladen`
: activeTheme.description}
</small>
</div>
{officialRasterProductsLoading && flandersScopeSelected ? (
<p className="geo-data-notice">Beschikbare Vlaamse rasterbronnen worden gecontroleerd</p>
{officialMapProductsLoading && flandersScopeSelected ? (
<p className="geo-data-notice">Beschikbare Vlaamse kaartbronnen worden gecontroleerd</p>
) : null}
{officialRasterProductsError && flandersScopeSelected ? (
<p className="error">{officialRasterProductsError}</p>
{officialMapProductsError && flandersScopeSelected ? (
<p className="error">{officialMapProductsError}</p>
) : null}
{analysisMode === 'current' && activeTheme.id === 'elevation' && officialRasterProducts.dhmv.length > 0 ? (
{analysisMode === 'current' && activeTheme.id === 'elevation' && officialMapProducts.dhmv.length > 0 ? (
<label className="geo-scope-select">
Hoogtemodel
<select
@@ -1674,7 +1681,7 @@ export function MapWorkspace({
}
}}
>
{officialRasterProducts.dhmv.map((product) => (
{officialMapProducts.dhmv.map((product) => (
<option key={product.key} value={product.key}>{product.display_name}</option>
))}
</select>
@@ -1682,7 +1689,7 @@ export function MapWorkspace({
</label>
) : null}
{analysisMode === 'current' && activeTheme.id === 'flood_hazard' && officialRasterProducts.floodHazard.length > 0 ? (
{analysisMode === 'current' && activeTheme.id === 'flood_hazard' && officialMapProducts.floodHazard.length > 0 ? (
<label className="geo-scope-select">
Overstromingsscenario
<select
@@ -1699,7 +1706,7 @@ export function MapWorkspace({
}
}}
>
{officialRasterProducts.floodHazard.map((product) => (
{officialMapProducts.floodHazard.map((product) => (
<option key={product.key} value={product.key}>{product.display_name}</option>
))}
</select>
@@ -1801,8 +1808,8 @@ export function MapWorkspace({
? 'Sleep nu een rechthoek op de kaart.'
: regionalRasterThemeActive
? 'Teken een rechthoek; de juiste gemeentelijke rasters worden automatisch gecombineerd.'
: onDemandRasterThemeActive
? 'Teken een rechthoek; officiële Vlaamse rasters worden begrensd opgehaald, bewaard en hergebruikt.'
: onDemandThemeActive
? 'Teken een rechthoek; officiële Vlaamse kaartbronnen worden begrensd opgehaald, bewaard en hergebruikt.'
: regionalBathymetryThemeActive
? 'Teken een rechthoek of analyseer Vlaanderen; alleen overlappende VHA-partities worden samengevoegd.'
: 'Sleep een rechthoek of analyseer het volledige werkgebied.'}
@@ -1820,16 +1827,16 @@ export function MapWorkspace({
</button>
<button
className="secondary-action"
disabled={!activeThemeAvailable || regionalRasterThemeActive || regionalOnDemandRasterThemeActive || (analysisMode === 'evolution' && activeTemporalSeries.length < 2) || !selectedAreaBbox || mapSelectionLoading || themeResultsLoading}
disabled={!activeThemeAvailable || regionalRasterThemeActive || regionalOnDemandThemeActive || (analysisMode === 'evolution' && activeTemporalSeries.length < 2) || !selectedAreaBbox || mapSelectionLoading || themeResultsLoading}
type="button"
title={
regionalRasterThemeActive || regionalOnDemandRasterThemeActive
? 'Teken een begrensde rechthoek voor een regionale rasteranalyse.'
regionalRasterThemeActive || regionalOnDemandThemeActive
? 'Teken een begrensde rechthoek voor deze regionale analyse.'
: undefined
}
onClick={() => selectedAreaBbox && void analyzeSelection(selectedAreaBbox, selectedMapArea?.id)}
>
{regionalRasterThemeActive || regionalOnDemandRasterThemeActive ? 'Selecteer een deelgebied' : 'Volledig werkgebied'}
{regionalRasterThemeActive || regionalOnDemandThemeActive ? 'Selecteer een deelgebied' : 'Volledig werkgebied'}
</button>
<button className="secondary-action" disabled={!mapSelectionBbox} type="button" onClick={clearAreaSelection}>
Wis selectie
@@ -2198,7 +2205,7 @@ export function MapWorkspace({
? `VHA-dwarsprofielen Vlaanderen · ${activeThemePartitions.length} gemeentepartities`
: activeThemeDataset
? getDatasetDisplayName(activeThemeDataset)
: activeOnDemandRasterProduct?.displayName
: activeOnDemandMapProduct?.displayName
?? 'niet beschikbaar'}
</span>
{usesDefaultOsmBasemap ? <span><strong>Ondergrond:</strong> OpenStreetMap</span> : null}
@@ -80,7 +80,8 @@ export function ProviderPanel({
<div className="provider-detail-stack">
<strong>Officiële referentiebronnen</strong>
<div>
Reeds ingeladen GRB-lagen blijven lokaal beschikbaar. De live GRB- en OSM-koppelingen staan uit en halen dus niet ongemerkt externe gegevens op.
GRB is beschikbaar voor expliciet begrensde kaartselecties en wordt daarna lokaal bewaard.
OSM blijft uitgeschakeld; geen enkele bron haalt ongemerkt gegevens op.
</div>
</div>
<ul className="system-provider-list">
@@ -6,7 +6,7 @@ import { terrainSelectionToMapSelection } from '../lib/terrainSelection'
import { floodHazardSelectionToMapSelection } from '../lib/floodHazardSelection'
import { thematicRasterSelectionToMapSelection } from '../lib/thematicRaster'
export type MapThemeAcquisitionKind = 'thematic_raster' | 'dhmv' | 'flood_hazard'
export type MapThemeAcquisitionKind = 'thematic_raster' | 'dhmv' | 'flood_hazard' | 'grb'
export interface MapThemeAcquisition {
kind: MapThemeAcquisitionKind
@@ -72,30 +72,34 @@ export function useMapThemeSelectionInsights<TThemeId extends string>(
queries.map(async ({ themeId, dataset: existingDataset, partitioned, acquisition }) => {
let dataset = existingDataset
if (acquisition) {
const commonPayload = {
bbox,
area_id: areaId,
force_refresh: false,
}
const acquisitionJob = acquisition.kind === 'thematic_raster'
? await datasetsApi.acquireThematicRaster(selectedProjectId, {
bbox,
area_id: areaId,
...commonPayload,
product_key: acquisition.productKey,
force_refresh: false,
})
: acquisition.kind === 'dhmv'
? await datasetsApi.acquireDhmv(selectedProjectId, {
bbox,
area_id: areaId,
...commonPayload,
product_key: acquisition.productKey as 'dtm_1m' | 'dsm_1m',
force_refresh: false,
})
: await datasetsApi.acquireFloodHazard(selectedProjectId, {
bbox,
area_id: areaId,
product_key: acquisition.productKey,
force_refresh: false,
})
: acquisition.kind === 'flood_hazard'
? await datasetsApi.acquireFloodHazard(selectedProjectId, {
...commonPayload,
product_key: acquisition.productKey,
})
: await datasetsApi.acquireGrb(selectedProjectId, {
...commonPayload,
product_key: acquisition.productKey as 'buildings' | 'roads' | 'water' | 'parcels',
})
if (acquisitionJob.status !== 'success' || !acquisitionJob.output_dataset_id) {
throw new Error(
acquisitionJob.error_message
|| `De officiële rasterbron ${acquisition.displayName} kon niet worden ingeladen.`,
|| `De officiële kaartbron ${acquisition.displayName} kon niet worden ingeladen.`,
)
}
dataset = await datasetsApi.get(selectedProjectId, acquisitionJob.output_dataset_id)
@@ -4,23 +4,26 @@ import { formatError } from '../lib/formatError'
import type {
DhmvProductRead,
FloodHazardProductRead,
GrbProductRead,
ThematicRasterProductRead,
} from '../types'
interface OfficialRasterProducts {
interface OfficialMapProducts {
thematic: ThematicRasterProductRead[]
dhmv: DhmvProductRead[]
floodHazard: FloodHazardProductRead[]
grb: GrbProductRead[]
}
const EMPTY_PRODUCTS: OfficialRasterProducts = {
const EMPTY_PRODUCTS: OfficialMapProducts = {
thematic: [],
dhmv: [],
floodHazard: [],
grb: [],
}
export function useOfficialRasterProducts(selectedProjectId: string | null) {
const [products, setProducts] = useState<OfficialRasterProducts>(EMPTY_PRODUCTS)
export function useOfficialMapProducts(selectedProjectId: string | null) {
const [products, setProducts] = useState<OfficialMapProducts>(EMPTY_PRODUCTS)
const [loading, setLoading] = useState(false)
const [error, setError] = useState<string | null>(null)
@@ -41,20 +44,22 @@ export function useOfficialRasterProducts(selectedProjectId: string | null) {
datasetsApi.listThematicRasterProducts(selectedProjectId),
datasetsApi.listDhmvProducts(selectedProjectId),
datasetsApi.listFloodHazardProducts(selectedProjectId),
datasetsApi.listGrbProducts(selectedProjectId),
])
.then(([thematic, dhmv, floodHazard]) => {
.then(([thematic, dhmv, floodHazard, grb]) => {
if (!cancelled) {
setProducts({
thematic: thematic.items,
dhmv: dhmv.items,
floodHazard: floodHazard.items,
grb: grb.items,
})
}
})
.catch((requestError) => {
if (!cancelled) {
setProducts(EMPTY_PRODUCTS)
setError(formatError(requestError, 'De officiële Vlaamse rastercatalogi konden niet worden geladen.'))
setError(formatError(requestError, 'De officiële Vlaamse kaartcatalogi konden niet worden geladen.'))
}
})
.finally(() => {
+6
View File
@@ -26,6 +26,8 @@ import type {
BathymetrySourceProbeRead,
BathymetrySourceRead,
DhmvProductRead,
GrbAcquireRequest,
GrbProductRead,
TerrainSelectionResponse,
ThematicRasterAcquireRequest,
ThematicRasterProductRead,
@@ -146,6 +148,10 @@ export const datasetsApi = {
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/dhmv/acquire`, payload),
listDhmvProducts: (projectId: string): Promise<{ items: DhmvProductRead[]; total: number }> =>
apiGet<{ items: DhmvProductRead[]; total: number }>(`/api/v1/projects/${projectId}/datasets/dhmv/products`),
acquireGrb: (projectId: string, payload: GrbAcquireRequest): Promise<JobRead> =>
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/grb/acquire`, payload),
listGrbProducts: (projectId: string): Promise<{ items: GrbProductRead[]; total: number }> =>
apiGet<{ items: GrbProductRead[]; total: number }>(`/api/v1/projects/${projectId}/datasets/grb/products`),
selectTerrain: (
projectId: string,
datasetId: string,
+21
View File
@@ -362,6 +362,27 @@ export interface DhmvProductRead {
limitation_message: string
}
export interface GrbAcquireRequest {
bbox: VectorSelectionBBox
area_id?: string | null
product_key: 'buildings' | 'roads' | 'water' | 'parcels'
force_refresh?: boolean
}
export interface GrbProductRead {
key: 'buildings' | 'roads' | 'water' | 'parcels'
display_name: string
reference_layer_name: string
collections: string[]
geometry_types: string[]
source_crs: 'EPSG:4326'
authority_level: 'authoritative'
catalog_url: string
attribution: string
license_note: string
limitation_message: string
}
export interface TerrainSelectionResponse {
dataset_id: string
dataset_ids: string[]