Add bounded GRB map acquisition
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-17 21:29:26 +02:00
parent 47cde21e17
commit 488da4cc83
28 changed files with 1644 additions and 134 deletions
+32
View File
@@ -1360,6 +1360,38 @@ idempotent for the exact plan/raster checksum, creates a new immutable raster
Dataset and DatasetVersion with the official edition, and retains every older
snapshot. No command is scheduled or invoked by startup or browser actions.
## Governed bounded GRB acquisition
`GET /api/v1/projects/{project_id}/datasets/grb/products` exposes four fixed
official vector products: building footprints, road segments, water
surfaces/lines and administrative parcels. `POST .../datasets/grb/acquire`
accepts an EPSG:4326 rectangle, optional project Area, one product key and an
explicit refresh flag.
The service queries only the allowlisted GRB OGC API collection paths, follows
complete same-host pagination and clips every geometry to `bbox ∩ Area`.
It explicitly requests OGC CRS84 GeoJSON for bbox and output; the native
EPSG:31370 storage CRS remains provenance rather than being guessed from raw
coordinates.
Requests fail closed above 20 km per side, 200 pages, 150,000 retained
features, 20 MiB per page or 256 MiB total. No partial Dataset is persisted
when a limit is exceeded. Official ids, request URLs, page checksums and the
final artifact checksum are retained as provenance.
Persistence uses the existing synchronous Job plus
`DatasetService.import_vector_bytes`, so Dataset, DatasetVersion and
VectorFeature rows remain one canonical flow. The browser never contacts the
provider directly. Exact request identities are reused for 24 hours. Buildings
return footprint area in hectares, roads return line length in kilometres,
water returns surface area plus supporting water-line length, and parcels
return mapped area. GRB cannot provide water volume, legal parcel boundaries
or traffic information.
Settings: `GRB_ENABLED`, `GRB_OGC_API_URL`, `GRB_MIN_SIDE_M`,
`GRB_MAX_SIDE_M`, `GRB_PAGE_SIZE`, `GRB_MAX_PAGES`, `GRB_MAX_FEATURES`,
`GRB_TIMEOUT_SECONDS`, `GRB_MAX_RESPONSE_MB`,
`GRB_MAX_TOTAL_RESPONSE_MB` and `GRB_CACHE_TTL_HOURS`.
## Governed DHMV terrain acquisition
`GET /api/v1/projects/{project_id}/datasets/dhmv/products` exposes the fixed
+26
View File
@@ -31,6 +31,7 @@ from app.schemas import (
BathymetryProfileAcquireRequest,
ThematicRasterAcquireRequest,
ThematicRasterSelectionRequest,
GrbAcquireRequest,
VectorBBoxResponse,
VectorBufferRequest,
VectorClipRequest,
@@ -49,6 +50,7 @@ from app.services.dataset_service import DatasetService
from app.services.source_freshness_service import SourceFreshnessService
from app.services.source_catalog_probe_service import SourceCatalogProbeService
from app.services.grb_refresh_plan_service import GrbRefreshPlanService
from app.services.grb_acquisition_service import GrbAcquisitionService
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.terrain_analysis_service import TerrainAnalysisService
@@ -193,6 +195,30 @@ def list_dhmv_products(project_id: UUID, db: Session = Depends(get_db)):
return envelope({"items": items, "total": len(items)})
@router.post("/datasets/grb/acquire", response_model=dict)
def acquire_bounded_grb(
project_id: UUID,
payload: GrbAcquireRequest,
db: Session = Depends(get_db),
):
job = JobService.run_sync_job(
db=db,
project_id=project_id,
job_type="vector.grb.acquire",
parameters=payload.model_dump(mode="json"),
operation=lambda: GrbAcquisitionService.acquire(db, project_id, payload),
)
return envelope(job)
@router.get("/datasets/grb/products", response_model=dict)
def list_grb_products(project_id: UUID, db: Session = Depends(get_db)):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
items = GrbAcquisitionService.list_products()
return envelope({"items": items, "total": len(items)})
@router.post("/datasets/flood-hazard/acquire", response_model=dict)
def acquire_bounded_flood_hazard(
project_id: UUID,
+1 -1
View File
@@ -39,7 +39,7 @@ def capabilities() -> dict:
geopandas=_dependency_enabled("geopandas"),
yolo=False,
sam=False,
grb="planned",
grb="bounded",
sentinel="planned",
providers=providers,
).model_dump()}
+19
View File
@@ -68,6 +68,25 @@ class Settings(BaseSettings):
le=86_400,
validation_alias="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS",
)
grb_enabled: bool = Field(default=True, validation_alias="GRB_ENABLED")
grb_ogc_api_url: str = Field(
default="https://geo.api.vlaanderen.be/GRB/ogc/features/v1",
validation_alias="GRB_OGC_API_URL",
)
grb_min_side_m: float = Field(default=10.0, gt=0, validation_alias="GRB_MIN_SIDE_M")
grb_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="GRB_MAX_SIDE_M")
grb_page_size: int = Field(default=1000, ge=1, le=1000, validation_alias="GRB_PAGE_SIZE")
grb_max_pages: int = Field(default=200, ge=1, le=1000, validation_alias="GRB_MAX_PAGES")
grb_max_features: int = Field(default=150_000, ge=1, validation_alias="GRB_MAX_FEATURES")
grb_timeout_seconds: int = Field(default=180, ge=1, le=600, validation_alias="GRB_TIMEOUT_SECONDS")
grb_max_response_mb: int = Field(default=20, ge=1, le=100, validation_alias="GRB_MAX_RESPONSE_MB")
grb_max_total_response_mb: int = Field(
default=256,
ge=1,
le=2048,
validation_alias="GRB_MAX_TOTAL_RESPONSE_MB",
)
grb_cache_ttl_hours: int = Field(default=24, ge=0, le=8760, validation_alias="GRB_CACHE_TTL_HOURS")
dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED")
dhmv_wcs_url: str = Field(
default="https://geo.api.vlaanderen.be/DHMV/wcs",
+21 -7
View File
@@ -9,12 +9,26 @@ class GRBProvider(BaseReferenceProvider):
provider_name="grb",
display_name="GRB",
authority_level="authoritative",
supported_layers=["buildings", "roads", "parcels"],
supported_layers=["buildings", "roads", "water", "parcels"],
supported_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"],
supported_query_modes=["area"],
fetch_signature="POST /api/v1/external/grb/fetch",
limitation_message="GRB live WFS/download integration is not configured in Sprint 7B.",
attribution="Digitaal Vlaanderen - Basiskaart Vlaanderen (GRB)",
license_note="Use must follow Digitaal Vlaanderen open data and attribution terms.",
configured=False,
supported_query_modes=["bbox", "persisted_area"],
fetch_signature="POST /api/v1/projects/{project_id}/datasets/grb/acquire",
limitation_message=(
"Alleen expliciet begrensde selecties tot 20 km per zijde worden opgehaald. "
"Volledige providerdownloads en onbeperkte queries zijn niet toegestaan."
),
attribution="Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen",
license_note="Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen.",
configured=True,
)
def fetch(self, project_id: str, area_id: str | None, layers: list[str]) -> dict:
del project_id, area_id, layers
return {
"provider": self.provider_name,
"status": "bounded_request_required",
"message": (
"Use POST /api/v1/projects/{project_id}/datasets/grb/acquire with an EPSG:4326 "
"bounding box and one governed product key."
),
}
+14 -2
View File
@@ -78,11 +78,23 @@ class ExternalProviderRegistry:
del project_id, area_id
provider = self.get(provider_name)
mapping = self.dataset_mapping(provider.provider_name, requested_dataset_role=requested_dataset_role)
if provider.provider_name in {"grb", "osm"}:
if provider.provider_name == "grb":
return ProviderImportResult(
provider_name="grb",
status="bounded_request_required",
message=(
"Use the governed project GRB acquisition endpoint with an EPSG:4326 bounding box "
"and one supported layer."
),
requested_layers=layers,
dataset_role=mapping.dataset_role,
source_name=mapping.source_name,
)
if provider.provider_name == "osm":
return ProviderImportResult(
provider_name=provider.provider_name,
status="not_configured",
message=f"No live {provider.display_name} import is configured in Sprint 7B.",
message=f"No live {provider.display_name} import is configured.",
requested_layers=layers,
dataset_role=mapping.dataset_role,
source_name=mapping.source_name,
+4
View File
@@ -17,6 +17,7 @@ from .source_catalog import (
SourceCatalogProbeSummary,
)
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
from .detection import (
DetectionListResponse,
DetectionModelCapability,
@@ -161,6 +162,9 @@ __all__ = [
"GrbRefreshLayerPlan",
"GrbRefreshPlan",
"GrbRefreshPlanSummary",
"GrbAcquireRequest",
"GrbAcquisitionResult",
"GrbProductRead",
"DetectionListResponse",
"DetectionModelCapability",
"DetectionModelsResponse",
+45
View File
@@ -0,0 +1,45 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel
from .operations import VectorSelectionBBox
class GrbAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str = "buildings"
force_refresh: bool = False
class GrbProductRead(BaseModel):
key: str
display_name: str
reference_layer_name: str
collections: list[str]
geometry_types: list[str]
source_crs: str
authority_level: str
catalog_url: str
attribution: str
license_note: str
limitation_message: str
class GrbAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
reference_layer_name: str
collections: list[str]
feature_count: int
candidate_feature_count: int
page_count: int
bbox_epsg4326: list[float]
source_version: str
attribution: str
limitation_message: str
@@ -0,0 +1,779 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
import hashlib
import json
import math
from pathlib import Path
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
from urllib.request import HTTPRedirectHandler, Request, build_opener
from uuid import UUID
from geoalchemy2.shape import to_shape
from pyproj import Transformer
from shapely.geometry import LineString, MultiLineString, MultiPolygon, Polygon, box, mapping, shape
from shapely.ops import unary_union
from shapely.validation import make_valid
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import Area, Dataset, Project
from app.schemas.grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
from app.services.dataset_service import DatasetService
class _RejectRedirects(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
del req, fp, code, msg, headers, newurl
return None
_NO_REDIRECT_OPENER = build_opener(_RejectRedirects())
@dataclass(frozen=True)
class GrbCollection:
name: str
geometry_dimension: int
@dataclass(frozen=True)
class GrbProduct:
key: str
display_name: str
reference_layer_name: str
layer_type: str
collections: tuple[GrbCollection, ...]
geometry_types: tuple[str, ...]
metric_key: str
metric_method: str
metric_label: str
metric_unit: str
metric_dimension: int
metric_warning: str
limitation_message: str
class GrbAcquisitionService:
PROVIDER = "grb"
SOURCE_CRS = "EPSG:4326"
OGC_CRS84_URI = "http://www.opengis.net/def/crs/OGC/1.3/CRS84"
AUTHORITY_LEVEL = "authoritative"
ATTRIBUTION = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen"
LICENSE_NOTE = "Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen."
CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/basiskaart-vlaanderen-grb"
@staticmethod
def _products() -> dict[str, GrbProduct]:
products = (
GrbProduct(
key="buildings",
display_name="GRB gebouwcontouren",
reference_layer_name="buildings",
layer_type="building",
collections=(GrbCollection("GBG", 2),),
geometry_types=("Polygon", "MultiPolygon"),
metric_key="footprint_area",
metric_method="intersection_area",
metric_label="Bebouwde grondoppervlakte",
metric_unit="ha",
metric_dimension=2,
metric_warning=(
"Dit is de grondoppervlakte van gebouwcontouren, niet de totale vloeroppervlakte "
"of het gebouwvolume."
),
limitation_message=(
"GRB GBG bevat gebouwcontouren uit de basiskaart. Registratie en fysieke verandering "
"kunnen in tijd verschillen."
),
),
GrbProduct(
key="roads",
display_name="GRB wegsegmenten",
reference_layer_name="roads",
layer_type="road",
collections=(GrbCollection("Wegsegment", 1),),
geometry_types=("LineString", "MultiLineString"),
metric_key="road_length",
metric_method="intersection_length",
metric_label="Totale weglengte",
metric_unit="km",
metric_dimension=1,
metric_warning=(
"De lengte volgt GRB-wegsegmenten en zegt niets over rijstroken, verkeersvolume "
"of verhardingsoppervlakte."
),
limitation_message=(
"GRB Wegsegment beschrijft netwerkgeometrie en is geen verkeersmodel of routeadvies."
),
),
GrbProduct(
key="water",
display_name="GRB wateroppervlakken en waterlijnen",
reference_layer_name="water",
layer_type="water",
collections=(
GrbCollection("WTZ", 2),
GrbCollection("WLAS", 1),
GrbCollection("WGR", 1),
),
geometry_types=("LineString", "MultiLineString", "Polygon", "MultiPolygon"),
metric_key="water_area",
metric_method="intersection_area",
metric_label="Wateroppervlakte",
metric_unit="ha",
metric_dimension=2,
metric_warning=(
"Watervolume is niet berekenbaar zonder betrouwbare diepte- of bathymetrische gegevens. "
"De GRB-bron levert alleen oppervlakte- en lijngeometrie."
),
limitation_message=(
"GRB-water combineert wateroppervlakken en watergerelateerde lijnen. Objectaantallen en "
"oppervlakte zijn geen actueel waterpeil of watervolume."
),
),
GrbProduct(
key="parcels",
display_name="GRB administratieve percelen",
reference_layer_name="parcels",
layer_type="parcel",
collections=(GrbCollection("ADP", 2),),
geometry_types=("Polygon", "MultiPolygon"),
metric_key="parcel_area",
metric_method="intersection_area",
metric_label="Perceeloppervlakte",
metric_unit="ha",
metric_dimension=2,
metric_warning=(
"GRB-percelen zijn een grafische referentie en vormen geen juridische grensopmeting."
),
limitation_message=(
"GRB ADP toont de vermoedelijke ligging van kadastrale percelen en is geen juridische grens."
),
),
)
return {product.key: product for product in products}
@staticmethod
def list_products() -> list[dict[str, Any]]:
return [
GrbProductRead(
key=product.key,
display_name=product.display_name,
reference_layer_name=product.reference_layer_name,
collections=[collection.name for collection in product.collections],
geometry_types=list(product.geometry_types),
source_crs=GrbAcquisitionService.SOURCE_CRS,
authority_level=GrbAcquisitionService.AUTHORITY_LEVEL,
catalog_url=GrbAcquisitionService.CATALOG_URL,
attribution=GrbAcquisitionService.ATTRIBUTION,
license_note=GrbAcquisitionService.LICENSE_NOTE,
limitation_message=product.limitation_message,
).model_dump()
for product in GrbAcquisitionService._products().values()
]
@staticmethod
def _product(product_key: str) -> GrbProduct:
product = GrbAcquisitionService._products().get(product_key.strip().lower())
if product is None:
raise AppError(
code="GRB_PRODUCT_NOT_SUPPORTED",
message="Select buildings, roads, water or parcels from the governed GRB product registry",
details={"product_key": product_key},
status_code=422,
)
return product
@staticmethod
def _validate_scope(
db,
project_id: UUID,
payload: GrbAcquireRequest,
settings: Settings,
) -> tuple[Any, list[float], list[float]]:
if not settings.grb_enabled:
raise AppError(code="GRB_NOT_CONFIGURED", message="Bounded GRB acquisition is disabled", status_code=503)
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
if payload.bbox.crs.upper() != "EPSG:4326":
raise AppError(code="GRB_INVALID_CRS", message="GRB acquisition requires EPSG:4326", status_code=400)
values = (payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
if not all(math.isfinite(value) for value in values):
raise AppError(code="GRB_INVALID_BBOX", message="Bounding box values must be finite", status_code=400)
if values[0] >= values[2] or values[1] >= values[3]:
raise AppError(code="GRB_INVALID_BBOX", message="Bounding box has no area", status_code=400)
if values[0] < -180 or values[2] > 180 or values[1] < -90 or values[3] > 90:
raise AppError(code="GRB_INVALID_BBOX", message="Bounding box is outside EPSG:4326", status_code=400)
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
metric_bounds = transformer.transform_bounds(*values, densify_pts=21)
width_m = float(metric_bounds[2] - metric_bounds[0])
height_m = float(metric_bounds[3] - metric_bounds[1])
if width_m < settings.grb_min_side_m or height_m < settings.grb_min_side_m:
raise AppError(
code="GRB_SELECTION_TOO_SMALL",
message=f"Select an area of at least {settings.grb_min_side_m:g} by {settings.grb_min_side_m:g} metres",
status_code=422,
)
if width_m > settings.grb_max_side_m or height_m > settings.grb_max_side_m:
raise AppError(
code="GRB_SELECTION_TOO_LARGE",
message=f"Select an area no larger than {settings.grb_max_side_m:g} by {settings.grb_max_side_m:g} metres",
details={"width_m": width_m, "height_m": height_m},
status_code=422,
)
selection = box(*values)
if payload.area_id is None:
scope_geometry = selection
else:
area = db.get(Area, payload.area_id)
if area is None:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
scope_geometry = to_shape(area.geometry).intersection(selection)
if scope_geometry.is_empty:
raise AppError(
code="GRB_SCOPE_EMPTY",
message="The requested bounding box does not intersect the selected area",
status_code=400,
)
return scope_geometry, [float(value) for value in values], [float(value) for value in metric_bounds]
@staticmethod
def _geometry_dimension(geometry: Any) -> int:
if geometry is None or geometry.is_empty:
return -1
if "Polygon" in geometry.geom_type:
return 2
if "LineString" in geometry.geom_type or geometry.geom_type == "LinearRing":
return 1
if "Point" in geometry.geom_type:
return 0
if hasattr(geometry, "geoms"):
return max((GrbAcquisitionService._geometry_dimension(item) for item in geometry.geoms), default=-1)
return -1
@staticmethod
def _extract_dimension(geometry: Any, expected_dimension: int) -> Any | None:
if geometry is None or geometry.is_empty:
return None
if not geometry.is_valid:
geometry = make_valid(geometry)
parts: list[Any] = []
def collect(candidate: Any) -> None:
if candidate is None or candidate.is_empty:
return
if expected_dimension == 2:
if isinstance(candidate, Polygon):
parts.append(candidate)
return
if isinstance(candidate, MultiPolygon):
parts.extend(item for item in candidate.geoms if not item.is_empty)
return
if expected_dimension == 1:
if isinstance(candidate, LineString):
parts.append(candidate)
return
if isinstance(candidate, MultiLineString):
parts.extend(item for item in candidate.geoms if not item.is_empty)
return
if hasattr(candidate, "geoms"):
for item in candidate.geoms:
collect(item)
collect(geometry)
if not parts:
return None
normalized = unary_union(parts)
if normalized.is_empty:
return None
if not normalized.is_valid:
normalized = make_valid(normalized)
if (
normalized.is_empty
or not normalized.is_valid
or GrbAcquisitionService._geometry_dimension(normalized) != expected_dimension
):
return None
return normalized
@staticmethod
def _collection_url(settings: Settings, collection: GrbCollection, bbox_values: tuple[float, ...]) -> str:
base = settings.grb_ogc_api_url.rstrip("/")
query = urlencode(
{
"f": "application/geo+json",
"limit": str(settings.grb_page_size),
"bbox": ",".join(f"{value:.8f}" for value in bbox_values),
"bbox-crs": GrbAcquisitionService.OGC_CRS84_URI,
"crs": GrbAcquisitionService.OGC_CRS84_URI,
}
)
return f"{base}/collections/{collection.name}/items?{query}"
@staticmethod
def _validated_page_url(
url: str,
settings: Settings,
product: GrbProduct,
bbox_values: tuple[float, ...],
) -> str:
parsed = urlparse(url)
base = urlparse(settings.grb_ogc_api_url)
allowed_paths = {
f"{base.path.rstrip('/')}/collections/{collection.name}/items"
for collection in product.collections
}
if (
parsed.scheme != "https"
or base.scheme != "https"
or parsed.netloc.casefold() != base.netloc.casefold()
or parsed.path not in allowed_paths
):
raise AppError(
code="GRB_PROVIDER_INVALID_PAGINATION",
message="GRB returned a pagination URL outside the governed OGC API allowlist",
status_code=502,
)
query = dict(parse_qsl(parsed.query, keep_blank_values=True))
query.update(
{
"f": "application/geo+json",
"limit": str(settings.grb_page_size),
"bbox": ",".join(f"{value:.8f}" for value in bbox_values),
"bbox-crs": GrbAcquisitionService.OGC_CRS84_URI,
"crs": GrbAcquisitionService.OGC_CRS84_URI,
}
)
return urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", urlencode(query), ""))
@staticmethod
def _read_page(
url: str,
settings: Settings,
opener: Callable[..., Any] | None,
) -> tuple[dict[str, Any], str, int]:
request = Request(
url,
headers={
"Accept": "application/geo+json, application/json",
"User-Agent": "GeoIntel/1.0 bounded-grb-acquisition",
},
)
try:
with (opener or _NO_REDIRECT_OPENER.open)(
request,
timeout=settings.grb_timeout_seconds,
) as response:
limit = settings.grb_max_response_mb * 1024 * 1024
content = response.read(limit + 1)
except HTTPError as exc:
raise AppError(
code="GRB_PROVIDER_HTTP_ERROR",
message="The GRB OGC API returned an HTTP error",
details={"status_code": exc.code},
status_code=502,
) from exc
except (TimeoutError, URLError, OSError) as exc:
raise AppError(
code="GRB_PROVIDER_UNAVAILABLE",
message="The GRB OGC API is unavailable",
status_code=502,
) from exc
if len(content) > limit:
raise AppError(
code="GRB_PROVIDER_RESPONSE_TOO_LARGE",
message="A GRB response page exceeded the configured size limit",
status_code=502,
)
try:
payload = json.loads(content.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AppError(
code="GRB_PROVIDER_INVALID_RESPONSE",
message="The GRB OGC API returned invalid GeoJSON",
status_code=502,
) from exc
if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection":
raise AppError(
code="GRB_PROVIDER_INVALID_RESPONSE",
message="The GRB OGC API returned a non-FeatureCollection response",
status_code=502,
)
return payload, hashlib.sha256(content).hexdigest(), len(content)
@staticmethod
def _next_url(payload: dict[str, Any], current_url: str) -> str | None:
links = payload.get("links")
if not isinstance(links, list):
return None
for link in links:
if isinstance(link, dict) and link.get("rel") == "next" and link.get("href"):
return urljoin(current_url, str(link["href"]))
return None
@staticmethod
def _fetch_features(
product: GrbProduct,
scope_geometry: Any,
bbox_values: tuple[float, ...],
coverage_scope: str,
settings: Settings,
opener: Callable[..., Any] | None,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
retained: list[dict[str, Any]] = []
seen_ids: set[str] = set()
request_urls: list[str] = []
response_sha256: list[str] = []
candidate_feature_count = 0
total_response_bytes = 0
geometry_types: dict[str, int] = {}
collection_counts: dict[str, int] = {item.name: 0 for item in product.collections}
for collection in product.collections:
url: str | None = GrbAcquisitionService._collection_url(settings, collection, bbox_values)
seen_pages: set[str] = set()
while url:
url = GrbAcquisitionService._validated_page_url(url, settings, product, bbox_values)
if url in seen_pages:
raise AppError(
code="GRB_PROVIDER_PAGINATION_LOOP",
message="The GRB OGC API repeated a pagination URL",
status_code=502,
)
if len(request_urls) >= settings.grb_max_pages:
raise AppError(
code="GRB_SELECTION_TOO_LARGE",
message="GRB acquisition exceeded the configured page limit",
details={"max_pages": settings.grb_max_pages},
status_code=422,
)
seen_pages.add(url)
payload, response_hash, response_size = GrbAcquisitionService._read_page(url, settings, opener)
request_urls.append(url)
response_sha256.append(response_hash)
total_response_bytes += response_size
if total_response_bytes > settings.grb_max_total_response_mb * 1024 * 1024:
raise AppError(
code="GRB_PROVIDER_RESPONSE_TOO_LARGE",
message="The complete GRB response exceeded the configured transfer limit",
status_code=502,
)
source_features = payload.get("features")
if not isinstance(source_features, list):
raise AppError(
code="GRB_PROVIDER_INVALID_RESPONSE",
message="The GRB FeatureCollection has no valid feature list",
status_code=502,
)
for source_feature in source_features:
candidate_feature_count += 1
if not isinstance(source_feature, dict):
continue
raw_id = str(source_feature.get("id") or "").strip()
if not raw_id:
raise AppError(
code="GRB_PROVIDER_INVALID_RESPONSE",
message=f"GRB {collection.name} returned a feature without an official identity",
status_code=502,
)
feature_id = f"{collection.name}:{raw_id}"
if feature_id in seen_ids:
continue
seen_ids.add(feature_id)
try:
source_geometry = GrbAcquisitionService._extract_dimension(
shape(source_feature.get("geometry")),
collection.geometry_dimension,
)
except Exception as exc:
raise AppError(
code="GRB_PROVIDER_INVALID_GEOMETRY",
message=f"GRB {collection.name} returned invalid geometry",
status_code=502,
) from exc
if source_geometry is None or not source_geometry.intersects(scope_geometry):
continue
retained_geometry = GrbAcquisitionService._extract_dimension(
source_geometry.intersection(scope_geometry),
collection.geometry_dimension,
)
if retained_geometry is None:
continue
if len(retained) >= settings.grb_max_features:
raise AppError(
code="GRB_SELECTION_TOO_LARGE",
message="GRB selection exceeds the configured feature limit; draw a smaller rectangle",
details={"max_features": settings.grb_max_features},
status_code=422,
)
properties = dict(source_feature.get("properties") or {})
properties.update(
{
"source_name": GrbAcquisitionService.PROVIDER,
"source_collection": collection.name,
"source_feature_id": feature_id,
"reference_layer_name": product.reference_layer_name,
"layer_type": product.layer_type,
"theme": product.key,
"authority_level": GrbAcquisitionService.AUTHORITY_LEVEL,
"coverage_scope": coverage_scope,
"geometry_clipped_to_selection": not scope_geometry.covers(source_geometry),
"attribution": GrbAcquisitionService.ATTRIBUTION,
}
)
retained.append(
{
"type": "Feature",
"id": feature_id,
"geometry": mapping(retained_geometry),
"properties": properties,
}
)
collection_counts[collection.name] += 1
geometry_types[retained_geometry.geom_type] = geometry_types.get(retained_geometry.geom_type, 0) + 1
url = GrbAcquisitionService._next_url(payload, url)
return retained, {
"candidate_feature_count": candidate_feature_count,
"feature_count": len(retained),
"page_count": len(request_urls),
"request_urls": request_urls,
"response_sha256": response_sha256,
"response_size_bytes": total_response_bytes,
"collection_feature_counts": collection_counts,
"geometry_types": geometry_types,
"output_crs": GrbAcquisitionService.OGC_CRS84_URI,
"reference_truncated": False,
}
@staticmethod
def _cached_dataset(
db,
project_id: UUID,
product: GrbProduct,
request_hash: str,
settings: Settings,
) -> Dataset | None:
if settings.grb_cache_ttl_hours <= 0:
return None
candidates = (
db.query(Dataset)
.filter(
Dataset.project_id == project_id,
Dataset.source_name == GrbAcquisitionService.PROVIDER,
Dataset.reference_layer_name == product.reference_layer_name,
Dataset.status == "ready",
)
.order_by(Dataset.imported_at.desc())
.all()
)
cutoff = datetime.now(UTC) - timedelta(hours=settings.grb_cache_ttl_hours)
for candidate in candidates:
provenance = candidate.provenance_metadata if isinstance(candidate.provenance_metadata, dict) else {}
imported_at = candidate.imported_at
if (
provenance.get("request_hash") == request_hash
and candidate.storage_path
and Path(candidate.storage_path).is_file()
and imported_at is not None
and imported_at >= cutoff
):
return candidate
return None
@staticmethod
def _result(
dataset: Dataset,
product: GrbProduct,
*,
reused: bool,
bbox_values: list[float],
) -> dict[str, Any]:
source_metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
provenance = dataset.provenance_metadata if isinstance(dataset.provenance_metadata, dict) else {}
metadata = dataset.metadata_json if isinstance(dataset.metadata_json, dict) else {}
return GrbAcquisitionResult(
output_dataset_id=dataset.id,
reused=reused,
provider=GrbAcquisitionService.PROVIDER,
product_key=product.key,
display_name=product.display_name,
reference_layer_name=product.reference_layer_name,
collections=[collection.name for collection in product.collections],
feature_count=int(metadata.get("feature_count", source_metadata.get("feature_count", 0))),
candidate_feature_count=int(provenance.get("candidate_feature_count", 0)),
page_count=int(provenance.get("page_count", 0)),
bbox_epsg4326=bbox_values,
source_version=str(dataset.source_version or ""),
attribution=GrbAcquisitionService.ATTRIBUTION,
limitation_message=product.limitation_message,
).model_dump(mode="json")
@staticmethod
def acquire(
db,
project_id: UUID,
payload: GrbAcquireRequest,
*,
settings: Settings | None = None,
opener: Callable[..., Any] | None = None,
) -> dict[str, Any]:
resolved_settings = settings or get_settings()
product = GrbAcquisitionService._product(payload.product_key)
scope_geometry, bbox_values, metric_bounds = GrbAcquisitionService._validate_scope(
db,
project_id,
payload,
resolved_settings,
)
request_identity = {
"provider": GrbAcquisitionService.PROVIDER,
"product_key": product.key,
"bbox_epsg4326": [round(value, 8) for value in bbox_values],
"area_id": str(payload.area_id) if payload.area_id else None,
}
request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode()).hexdigest()
if not payload.force_refresh:
cached = GrbAcquisitionService._cached_dataset(
db,
project_id,
product,
request_hash,
resolved_settings,
)
if cached is not None:
return GrbAcquisitionService._result(cached, product, reused=True, bbox_values=bbox_values)
area = db.get(Area, payload.area_id) if payload.area_id else None
coverage_scope = (
"municipality"
if area is not None and area.name.strip().lower().startswith("gemeente ")
else "bounded_selection"
)
features, transfer = GrbAcquisitionService._fetch_features(
product,
scope_geometry,
tuple(scope_geometry.bounds),
coverage_scope,
resolved_settings,
opener,
)
acquired_at = datetime.now(UTC)
source_version = acquired_at.date().isoformat()
feature_collection = {
"type": "FeatureCollection",
"name": product.display_name,
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
"features": features,
}
artifact = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
filename = f"grb_{product.key}_{source_version}_{request_hash[:12]}.geojson"
source_metadata: dict[str, Any] = {
"provider": "Digitaal Vlaanderen",
"service": "OGC API Features",
"product_key": product.key,
"product_display_name": product.display_name,
"collections": [collection.name for collection in product.collections],
"authority_level": GrbAcquisitionService.AUTHORITY_LEVEL,
"theme": product.key,
"layer_type": product.layer_type,
"coverage_scope": coverage_scope,
"geometry_clipped_to_area": payload.area_id is not None,
"geometry_clipped_to_selection": True,
"bbox_epsg4326": bbox_values,
"bbox_epsg31370": metric_bounds,
"feature_count": len(features),
"collection_feature_counts": transfer["collection_feature_counts"],
"identity_stable": True,
"identity_scheme": "grb_ogc_feature_id",
"source_storage_crs": "EPSG:31370",
"requested_output_crs": GrbAcquisitionService.OGC_CRS84_URI,
"selection_aggregation": {
"metric_key": product.metric_key,
"method": product.metric_method,
"label": product.metric_label,
"unit": product.metric_unit,
"geometry_dimension": product.metric_dimension,
"is_estimate": False,
"warning": product.metric_warning,
},
"attribution": GrbAcquisitionService.ATTRIBUTION,
"license_note": GrbAcquisitionService.LICENSE_NOTE,
"catalog_url": GrbAcquisitionService.CATALOG_URL,
"limitation_message": product.limitation_message,
}
if product.key == "water":
source_metadata["selection_metrics"] = [
{
"metric_key": "water_length",
"method": "intersection_length",
"label": "Lengte watergerelateerde lijnen",
"unit": "km",
"geometry_dimension": 1,
"is_estimate": False,
}
]
try:
dataset_response = DatasetService.import_vector_bytes(
db,
project_id=project_id,
area_id=payload.area_id,
filename=filename,
content=artifact,
source="Digitaal Vlaanderen GRB OGC API Features",
source_name=GrbAcquisitionService.PROVIDER,
dataset_role="reference",
reference_layer_name=product.reference_layer_name,
temporal_series_key=f"grb:{product.key}:{request_hash[:24]}",
observed_at=datetime(
acquired_at.year,
acquired_at.month,
acquired_at.day,
tzinfo=UTC,
),
temporal_granularity="snapshot",
source_version=source_version,
source_metadata=source_metadata,
provenance_metadata={
"acquisition": "explicit_bounded_ogc_api_features",
"acquired_at": acquired_at.isoformat(),
"request_hash": request_hash,
"request_urls": transfer["request_urls"],
"response_sha256": transfer["response_sha256"],
"response_size_bytes": transfer["response_size_bytes"],
"page_count": transfer["page_count"],
"candidate_feature_count": transfer["candidate_feature_count"],
"exact_feature_count": transfer["feature_count"],
"reference_truncated": False,
"artifact_sha256": hashlib.sha256(artifact).hexdigest(),
"clipped_to_area_id": str(payload.area_id) if payload.area_id else None,
"scope_geometry_type": scope_geometry.geom_type,
"limitation_message": product.limitation_message,
},
)
except AppError:
raise
except Exception as exc:
raise AppError(
code="GRB_PERSISTENCE_FAILED",
message="The validated GRB selection could not be persisted",
details={"reason": str(exc)},
status_code=500,
) from exc
persisted = db.get(Dataset, dataset_response.id)
if persisted is None:
raise AppError(
code="GRB_PERSISTENCE_FAILED",
message="The persisted GRB dataset could not be reloaded",
status_code=500,
)
return GrbAcquisitionService._result(persisted, product, reused=False, bbox_values=bbox_values)
@@ -54,4 +54,5 @@ def test_detection_lab_surfaces_local_model_asset_selection() -> None:
assert "onSelectModelAsset" in lab
assert "modelAssets={modelAssets}" in app
assert "Officiële referentiebronnen" in provider_panel
assert "live GRB- en OSM-koppelingen staan uit" in provider_panel
assert "GRB is beschikbaar voor expliciet begrensde kaartselecties" in provider_panel
assert "OSM blijft uitgeschakeld" in provider_panel
@@ -10,12 +10,12 @@ def read(path: str) -> str:
def test_flanders_workspace_exposes_governed_thematic_products_on_demand() -> None:
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
product_hook = read("frontend/src/hooks/useOfficialRasterProducts.ts")
product_hook = read("frontend/src/hooks/useOfficialMapProducts.ts")
selection_hook = read("frontend/src/hooks/useMapThemeSelectionInsights.ts")
api = read("frontend/src/services/api/datasets.ts")
assert "activeScopeProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME" in workspace
assert "new Map<DataThemeId, OnDemandRasterProduct>" in workspace
assert "new Map<DataThemeId, OnDemandMapProduct>" in workspace
assert "'Op aanvraag'" in workspace
assert "theme.id === 'space_occupation'" in workspace
assert "setActiveThemeId(fallbackTheme.id)" in workspace
@@ -36,23 +36,23 @@ def test_selection_runs_all_available_themes_and_refreshes_persisted_datasets()
assert "for (const theme of DATA_THEMES)" in workspace
assert "await loadThemeInsights(bbox, availableThemes, areaId)" in workspace
assert "!regionalPartitionedThemeActive && !onDemandRasterThemeActive" in workspace
assert "!regionalPartitionedThemeActive && !onDemandThemeActive" in workspace
assert "onRefreshProjectData" in workspace
assert "selectedProjectId ? loadProjectData(selectedProjectId)" in app
assert "successful.some((item) => item.acquisition)" in selection_hook
assert "await onDatasetsChanged()" in selection_hook
def test_regional_on_demand_rasters_require_a_bounded_drawn_selection() -> None:
def test_regional_on_demand_sources_require_a_bounded_drawn_selection() -> None:
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
assert "regionalOnDemandRasterThemeActive" in workspace
assert "regionalRasterThemeActive || regionalOnDemandRasterThemeActive" in workspace
assert "Teken een begrensde rechthoek voor een regionale rasteranalyse." in workspace
assert "officiële Vlaamse rasters worden begrensd opgehaald, bewaard en hergebruikt" in workspace
assert "regionalOnDemandThemeActive" in workspace
assert "regionalRasterThemeActive || regionalOnDemandThemeActive" in workspace
assert "Teken een begrensde rechthoek voor deze regionale analyse." in workspace
assert "Vlaamse kaartbronnen worden begrensd opgehaald, bewaard en hergebruikt" in workspace
def test_frontend_does_not_contact_the_external_wcs_directly() -> None:
def test_frontend_does_not_contact_external_map_services_directly() -> None:
frontend_sources = "\n".join(
path.read_text(encoding="utf-8")
for path in (ROOT / "frontend/src").rglob("*")
@@ -60,4 +60,5 @@ def test_frontend_does_not_contact_the_external_wcs_directly() -> None:
)
assert "mercatornet.be" not in frontend_sources.casefold()
assert "geo.api.vlaanderen.be" not in frontend_sources.casefold()
assert "GetCoverage" not in frontend_sources
@@ -11,12 +11,13 @@ def read(path: str) -> str:
return (ROOT / path).read_text(encoding="utf-8")
def test_official_raster_catalog_hook_loads_all_governed_registries() -> None:
hook = read("frontend/src/hooks/useOfficialRasterProducts.ts")
def test_official_map_catalog_hook_loads_all_governed_registries() -> None:
hook = read("frontend/src/hooks/useOfficialMapProducts.ts")
assert "datasetsApi.listThematicRasterProducts" in hook
assert "datasetsApi.listDhmvProducts" in hook
assert "datasetsApi.listFloodHazardProducts" in hook
assert "datasetsApi.listGrbProducts" in hook
assert "Promise.all([" in hook
@@ -24,7 +25,7 @@ def test_map_selection_can_acquire_dhmv_and_flood_hazard_products() -> None:
selection_hook = read("frontend/src/hooks/useMapThemeSelectionInsights.ts")
workspace = read("frontend/src/components/map/MapWorkspace.tsx")
assert "'thematic_raster' | 'dhmv' | 'flood_hazard'" in selection_hook
assert "'thematic_raster' | 'dhmv' | 'flood_hazard' | 'grb'" in selection_hook
assert "datasetsApi.acquireDhmv" in selection_hook
assert "datasetsApi.acquireFloodHazard" in selection_hook
assert "datasetsApi.acquireThematicRaster" in selection_hook
@@ -0,0 +1,392 @@
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import parse_qs, urlparse
from uuid import uuid4
import pytest
from fastapi.testclient import TestClient
from geoalchemy2.shape import from_shape
from shapely.geometry import MultiPolygon, Polygon
from app.core.config import Settings
from app.core.errors import AppError
from app.db.session import get_db
from app.main import app
from app.models import Area, Dataset, Job, Project
from app.schemas.grb import GrbAcquireRequest
from app.services.dataset_service import DatasetService
from app.services.grb_acquisition_service import GrbAcquisitionService
ROOT = Path(__file__).resolve().parents[2]
class FakeQuery:
def __init__(self, result=None):
self.result = result
def filter(self, *_args):
return self
def order_by(self, *_args):
return self
def all(self):
return self.result if isinstance(self.result, list) else []
class FakeSession:
def __init__(self, rows=None, query_result=None):
self.rows = rows or {}
self.query_result = query_result
self.added = []
def get(self, model, row_id):
row = self.rows.get((model, row_id))
if row is not None:
return row
return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
def add(self, row):
self.added.append(row)
def commit(self):
return None
def rollback(self):
return None
def refresh(self, row):
return row
def query(self, _model):
return FakeQuery(self.query_result)
class JsonResponse:
def __init__(self, payload):
self.content = json.dumps(payload).encode("utf-8")
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, size=-1):
return self.content if size < 0 else self.content[:size]
def request(*, product_key="buildings", area_id=None, force_refresh=True) -> GrbAcquireRequest:
return GrbAcquireRequest(
bbox={
"min_x": 5.15,
"min_y": 51.18,
"max_x": 5.17,
"max_y": 51.20,
"crs": "EPSG:4326",
},
area_id=area_id,
product_key=product_key,
force_refresh=force_refresh,
)
def polygon_feature(feature_id: str, coordinates) -> dict:
return {
"type": "Feature",
"id": feature_id,
"geometry": {"type": "Polygon", "coordinates": [coordinates]},
"properties": {"source_field": feature_id},
}
def test_grb_registry_exposes_four_governed_products() -> None:
products = {item["key"]: item for item in GrbAcquisitionService.list_products()}
assert set(products) == {"buildings", "roads", "water", "parcels"}
assert products["buildings"]["collections"] == ["GBG"]
assert products["roads"]["collections"] == ["Wegsegment"]
assert products["water"]["collections"] == ["WTZ", "WLAS", "WGR"]
assert products["parcels"]["collections"] == ["ADP"]
assert all(item["authority_level"] == "authoritative" for item in products.values())
def test_grb_fetch_follows_pagination_clips_geometry_and_preserves_official_identity() -> None:
product = GrbAcquisitionService._product("buildings")
settings = Settings(_env_file=None)
pages = []
first = polygon_feature(
"GBG.1",
[(5.155, 51.185), (5.175, 51.185), (5.175, 51.195), (5.155, 51.195), (5.155, 51.185)],
)
second = polygon_feature(
"GBG.2",
[(5.151, 51.181), (5.152, 51.181), (5.152, 51.182), (5.151, 51.182), (5.151, 51.181)],
)
outside = polygon_feature(
"GBG.3",
[(5.3, 51.3), (5.31, 51.3), (5.31, 51.31), (5.3, 51.31), (5.3, 51.3)],
)
def opener(raw_request, timeout):
assert timeout == settings.grb_timeout_seconds
parsed = urlparse(raw_request.full_url)
query = parse_qs(parsed.query)
pages.append(raw_request.full_url)
assert query["bbox-crs"] == [GrbAcquisitionService.OGC_CRS84_URI]
assert query["crs"] == [GrbAcquisitionService.OGC_CRS84_URI]
if query.get("cursor") == ["next"]:
return JsonResponse({"type": "FeatureCollection", "features": [second, outside], "links": []})
return JsonResponse(
{
"type": "FeatureCollection",
"features": [first],
"links": [
{
"rel": "next",
"href": (
"https://geo.api.vlaanderen.be/GRB/ogc/features/v1/"
"collections/GBG/items?cursor=next"
),
}
],
}
)
scope = Polygon(
[(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)]
)
features, transfer = GrbAcquisitionService._fetch_features(
product,
scope,
scope.bounds,
"bounded_selection",
settings,
opener,
)
assert len(pages) == 2
assert transfer["page_count"] == 2
assert transfer["candidate_feature_count"] == 3
assert transfer["feature_count"] == 2
assert transfer["reference_truncated"] is False
assert {feature["id"] for feature in features} == {"GBG:GBG.1", "GBG:GBG.2"}
clipped = next(feature for feature in features if feature["id"] == "GBG:GBG.1")
assert clipped["properties"]["source_feature_id"] == "GBG:GBG.1"
assert clipped["properties"]["geometry_clipped_to_selection"] is True
assert clipped["properties"]["coverage_scope"] == "bounded_selection"
def test_grb_fetch_rejects_untrusted_pagination_and_unbounded_feature_volume() -> None:
product = GrbAcquisitionService._product("buildings")
settings = Settings(_env_file=None)
feature = polygon_feature(
"GBG.1",
[(5.151, 51.181), (5.152, 51.181), (5.152, 51.182), (5.151, 51.182), (5.151, 51.181)],
)
scope = Polygon(
[(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)]
)
def hostile_opener(_request, timeout):
del timeout
return JsonResponse(
{
"type": "FeatureCollection",
"features": [feature],
"links": [{"rel": "next", "href": "https://example.test/private"}],
}
)
with pytest.raises(AppError) as invalid_next:
GrbAcquisitionService._fetch_features(
product,
scope,
scope.bounds,
"bounded_selection",
settings,
hostile_opener,
)
assert invalid_next.value.code == "GRB_PROVIDER_INVALID_PAGINATION"
def oversized_opener(_request, timeout):
del timeout
return JsonResponse(
{
"type": "FeatureCollection",
"features": [
feature,
{**feature, "id": "GBG.2"},
],
"links": [],
}
)
with pytest.raises(AppError) as oversized:
GrbAcquisitionService._fetch_features(
product,
scope,
scope.bounds,
"bounded_selection",
Settings(_env_file=None, GRB_MAX_FEATURES=1),
oversized_opener,
)
assert oversized.value.code == "GRB_SELECTION_TOO_LARGE"
def test_grb_acquisition_rejects_large_scope_before_network_access() -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Vlaanderen")})
payload = GrbAcquireRequest(
bbox={"min_x": 4.0, "min_y": 50.7, "max_x": 5.0, "max_y": 51.7, "crs": "EPSG:4326"},
product_key="buildings",
)
with pytest.raises(AppError) as exc_info:
GrbAcquisitionService.acquire(db, project_id, payload, settings=Settings(_env_file=None))
assert exc_info.value.code == "GRB_SELECTION_TOO_LARGE"
def test_grb_acquisition_persists_via_dataset_service_with_selection_metrics(monkeypatch) -> None:
project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4()
municipality = MultiPolygon(
[
Polygon(
[(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)]
)
]
)
project = Project(id=project_id, name="Vlaanderen")
area = Area(
id=area_id,
project_id=project_id,
name="Gemeente Mol - officieel",
geometry=from_shape(municipality, srid=4326),
)
db = FakeSession({(Project, project_id): project, (Area, area_id): area})
captured = {}
def opener(_request, timeout):
del timeout
parsed = urlparse(_request.full_url)
collection = parsed.path.split("/")[-2]
if collection == "WTZ":
features = [
polygon_feature(
"WTZ.1",
[(5.151, 51.181), (5.16, 51.181), (5.16, 51.19), (5.151, 51.19), (5.151, 51.181)],
)
]
else:
features = []
return JsonResponse({"type": "FeatureCollection", "features": features, "links": []})
def persist(_db, **kwargs):
captured.update(kwargs)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
area_id=area_id,
name=kwargs["filename"],
dataset_type="vector",
source=kwargs["source"],
dataset_role=kwargs["dataset_role"],
source_name=kwargs["source_name"],
reference_layer_name=kwargs["reference_layer_name"],
temporal_series_key=kwargs["temporal_series_key"],
observed_at=kwargs["observed_at"],
source_version=kwargs["source_version"],
source_metadata=kwargs["source_metadata"],
provenance_metadata=kwargs["provenance_metadata"],
metadata_json={"feature_count": 1},
status="ready",
)
db.rows[(Dataset, dataset_id)] = dataset
return SimpleNamespace(id=dataset_id)
monkeypatch.setattr(DatasetService, "import_vector_bytes", persist)
result = GrbAcquisitionService.acquire(
db,
project_id,
request(product_key="water", area_id=area_id),
settings=Settings(_env_file=None),
opener=opener,
)
assert result["output_dataset_id"] == str(dataset_id)
assert result["feature_count"] == 1
assert captured["dataset_role"] == "reference"
assert captured["source_name"] == "grb"
assert captured["reference_layer_name"] == "water"
assert captured["source_metadata"]["coverage_scope"] == "municipality"
assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == "water_area"
assert captured["source_metadata"]["selection_metrics"][0]["metric_key"] == "water_length"
assert captured["provenance_metadata"]["reference_truncated"] is False
collection = json.loads(captured["content"])
assert collection["features"][0]["properties"]["coverage_scope"] == "municipality"
def test_grb_routes_use_canonical_envelopes_and_existing_job_contract(monkeypatch) -> None:
project_id, dataset_id = uuid4(), uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
monkeypatch.setattr(
GrbAcquisitionService,
"acquire",
lambda *_args, **_kwargs: {
"output_dataset_id": str(dataset_id),
"provider": "grb",
"product_key": "buildings",
"feature_count": 2,
},
)
app.dependency_overrides[get_db] = lambda: db
try:
client = TestClient(app)
products_response = client.get(f"/api/v1/projects/{project_id}/datasets/grb/products")
acquire_response = client.post(
f"/api/v1/projects/{project_id}/datasets/grb/acquire",
json=request().model_dump(mode="json"),
)
finally:
app.dependency_overrides.clear()
assert products_response.status_code == 200
assert set(products_response.json()) == {"data"}
assert products_response.json()["data"]["total"] == 4
assert acquire_response.status_code == 200
assert set(acquire_response.json()) == {"data"}
assert acquire_response.json()["data"]["job_type"] == "vector.grb.acquire"
assert acquire_response.json()["data"]["output_dataset_id"] == str(dataset_id)
assert any(isinstance(item, Job) for item in db.added)
def test_system_capabilities_reports_bounded_grb_integration() -> None:
response = TestClient(app).get("/api/v1/system/capabilities")
assert response.status_code == 200
assert response.json()["data"]["grb"] == "bounded"
grb = next(
item for item in response.json()["data"]["providers"]
if item["provider_name"] == "grb"
)
assert grb["status"] == "configured"
assert grb["fetch_signature"].endswith("/datasets/grb/acquire")
def test_grb_frontend_and_contracts_use_only_the_governed_backend_path() -> None:
selection_hook = (ROOT / "frontend/src/hooks/useMapThemeSelectionInsights.ts").read_text(encoding="utf-8")
catalog_hook = (ROOT / "frontend/src/hooks/useOfficialMapProducts.ts").read_text(encoding="utf-8")
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
contracts = (ROOT / "docs/API_CONTRACTS.md").read_text(encoding="utf-8")
assert "datasetsApi.acquireGrb" in selection_hook
assert "datasetsApi.listGrbProducts" in catalog_hook
assert "officialMapProducts.grb" in workspace
assert "/datasets/grb/acquire" in contracts
assert "geo.api.vlaanderen.be" not in workspace
@@ -286,10 +286,10 @@ def test_provider_capabilities_expose_sprint7a_contract() -> None:
assert capabilities["osm"]["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"]
assert capabilities["osm"]["supported_query_modes"] == ["area"]
assert capabilities["osm"]["status"] == "not_configured"
assert capabilities["grb"]["supported_layers"] == ["buildings", "roads", "parcels"]
assert capabilities["grb"]["supported_layers"] == ["buildings", "roads", "water", "parcels"]
assert capabilities["grb"]["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"]
assert capabilities["grb"]["supported_query_modes"] == ["area"]
assert capabilities["grb"]["status"] == "not_configured"
assert capabilities["grb"]["supported_query_modes"] == ["bbox", "persisted_area"]
assert capabilities["grb"]["status"] == "configured"
def test_sprint7a_migration_declares_foundation_tables_and_indexes() -> None:
@@ -16,8 +16,8 @@ def test_provider_registry_lists_sprint7b_providers() -> None:
assert set(providers) == {"grb", "osm", "manual", "fixture"}
assert providers["grb"].authority_level == "authoritative"
assert providers["grb"].configured is False
assert providers["grb"].status == "not_configured"
assert providers["grb"].configured is True
assert providers["grb"].status == "configured"
assert providers["osm"].authority_level == "contextual"
assert providers["osm"].configured is False
assert providers["osm"].status == "not_configured"
@@ -34,9 +34,9 @@ def test_provider_capabilities_include_required_metadata() -> None:
assert grb["provider_name"] == "grb"
assert grb["display_name"] == "GRB"
assert grb["supported_layers"] == ["buildings", "roads", "parcels"]
assert grb["supported_layers"] == ["buildings", "roads", "water", "parcels"]
assert grb["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"]
assert grb["supported_query_modes"] == ["area"]
assert grb["supported_query_modes"] == ["bbox", "persisted_area"]
assert grb["limitation_message"]
assert grb["attribution"]
assert grb["license_note"]
@@ -56,13 +56,13 @@ def test_provider_to_dataset_mapping_is_enforced() -> None:
assert get_provider_dataset_mapping("osm", requested_dataset_role="reference").dataset_role == "reference"
def test_grb_osm_import_contract_returns_not_configured_without_fetching() -> None:
def test_grb_import_contract_requires_bounded_request_and_osm_remains_not_configured() -> None:
grb = import_provider_dataset("grb", project_id="project", area_id="area", layers=["buildings"])
osm = import_provider_dataset("osm", project_id="project", area_id="area", layers=["buildings"])
assert grb.status == "not_configured"
assert grb.status == "bounded_request_required"
assert grb.dataset_id is None
assert "No live" in grb.message
assert "bounding box" in grb.message
assert osm.status == "not_configured"
assert osm.dataset_id is None
@@ -107,7 +107,7 @@ def test_provider_api_envelopes_and_invalid_provider() -> None:
assert invalid_response.json()["message"] == "Provider not found"
def test_provider_import_api_returns_clear_not_configured_response() -> None:
def test_provider_import_api_points_grb_to_governed_bounded_endpoint() -> None:
client = TestClient(app)
response = client.post(
@@ -121,7 +121,7 @@ def test_provider_import_api_returns_clear_not_configured_response() -> None:
assert response.status_code == 200
assert response.json()["data"]["provider_name"] == "grb"
assert response.json()["data"]["status"] == "not_configured"
assert response.json()["data"]["status"] == "bounded_request_required"
assert response.json()["data"]["dataset_id"] is None