From 488da4cc83265b6fa50e16ed6786692c9d07f674 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 17 Jul 2026 21:29:26 +0200 Subject: [PATCH] Add bounded GRB map acquisition --- .env.example | 11 + CHANGELOG.md | 17 + backend/README.md | 32 + backend/app/api/routes/datasets.py | 26 + backend/app/api/routes/health.py | 2 +- backend/app/core/config.py | 19 + backend/app/providers/grb.py | 28 +- backend/app/providers/registry.py | 16 +- backend/app/schemas/__init__.py | 4 + backend/app/schemas/grb.py | 45 + .../app/services/grb_acquisition_service.py | 779 ++++++++++++++++++ .../tests/test_sprint118_yolo_preflight_ui.py | 3 +- ...t_sprint237_flanders_thematic_on_demand.py | 19 +- ...test_sprint238_flanders_raster_catalogs.py | 7 +- .../test_sprint239_bounded_grb_acquisition.py | 392 +++++++++ .../test_sprint7a_persistence_foundation.py | 6 +- .../tests/test_sprint7b_provider_registry.py | 18 +- docker-compose.unraid.yml | 11 + docker-compose.yml | 11 + docs/API_CONTRACTS.md | 84 +- docs/CODEX_EXECUTION_LOG.md | 32 + frontend/README.md | 10 + frontend/src/components/map/MapWorkspace.tsx | 127 +-- .../components/providers/ProviderPanel.tsx | 3 +- .../src/hooks/useMapThemeSelectionInsights.ts | 32 +- ...rProducts.ts => useOfficialMapProducts.ts} | 17 +- frontend/src/services/api/datasets.ts | 6 + frontend/src/types.ts | 21 + 28 files changed, 1644 insertions(+), 134 deletions(-) create mode 100644 backend/app/schemas/grb.py create mode 100644 backend/app/services/grb_acquisition_service.py create mode 100644 backend/tests/test_sprint239_bounded_grb_acquisition.py rename frontend/src/hooks/{useOfficialRasterProducts.ts => useOfficialMapProducts.ts} (76%) diff --git a/.env.example b/.env.example index 320baaf8..d6baff7f 100644 --- a/.env.example +++ b/.env.example @@ -14,6 +14,17 @@ ORTHOPHOTO_MAX_SIDE_M=1024 ORTHOPHOTO_CACHE_TTL_HOURS=24 SOURCE_CATALOG_PROBE_ENABLED=true SOURCE_CATALOG_GRB_WFS_URL=https://geo.api.vlaanderen.be/GRB/wfs +GRB_ENABLED=true +GRB_OGC_API_URL=https://geo.api.vlaanderen.be/GRB/ogc/features/v1 +GRB_MIN_SIDE_M=10 +GRB_MAX_SIDE_M=20000 +GRB_PAGE_SIZE=1000 +GRB_MAX_PAGES=200 +GRB_MAX_FEATURES=150000 +GRB_TIMEOUT_SECONDS=180 +GRB_MAX_RESPONSE_MB=20 +GRB_MAX_TOTAL_RESPONSE_MB=256 +GRB_CACHE_TTL_HOURS=24 SOURCE_CATALOG_STATBEL_DCAT_URL=https://doc.statbel.be/publications/DCAT/DCAT_opendata_datasets.ttl SOURCE_CATALOG_STATBEL_MAX_RESPONSE_MB=5 SOURCE_CATALOG_ALZ_RELEASE_URL=https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen diff --git a/CHANGELOG.md b/CHANGELOG.md index c964c5f1..8e29bfc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ # Changelog +## Sprint 239 Governed bounded GRB map acquisition (2026-07-17) + +- Added a fixed four-product GRB registry for buildings, roads, water and + administrative parcels through the official OGC API Features service. +- Added bounded acquisition with exact `bbox ∩ Area` clipping, complete + allowlisted pagination, response/feature limits, request and artifact + checksums, 24-hour identity cache and canonical Job/Dataset/VectorFeature + persistence. +- Added semantic selection metrics for building footprint, road length, water + area/supporting line length and parcel area without fabricating traffic, + legal-boundary or water-volume data. +- Extended the Flanders map catalog and all-theme analysis so the four GRB + products are usable on demand without browser-side external calls. +- Updated the provider registry to report the bounded GRB integration as + configured while keeping the old generic import/fetch contract non-fetching + and directing callers to the governed project endpoint. + ## Sprint 238 Governed Flanders terrain and flood selection (2026-07-17) - Unified the Flanders map selection flow behind one official-raster diff --git a/backend/README.md b/backend/README.md index 73421d64..b19dd7a5 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1360,6 +1360,38 @@ idempotent for the exact plan/raster checksum, creates a new immutable raster Dataset and DatasetVersion with the official edition, and retains every older snapshot. No command is scheduled or invoked by startup or browser actions. +## Governed bounded GRB acquisition + +`GET /api/v1/projects/{project_id}/datasets/grb/products` exposes four fixed +official vector products: building footprints, road segments, water +surfaces/lines and administrative parcels. `POST .../datasets/grb/acquire` +accepts an EPSG:4326 rectangle, optional project Area, one product key and an +explicit refresh flag. + +The service queries only the allowlisted GRB OGC API collection paths, follows +complete same-host pagination and clips every geometry to `bbox ∩ Area`. +It explicitly requests OGC CRS84 GeoJSON for bbox and output; the native +EPSG:31370 storage CRS remains provenance rather than being guessed from raw +coordinates. +Requests fail closed above 20 km per side, 200 pages, 150,000 retained +features, 20 MiB per page or 256 MiB total. No partial Dataset is persisted +when a limit is exceeded. Official ids, request URLs, page checksums and the +final artifact checksum are retained as provenance. + +Persistence uses the existing synchronous Job plus +`DatasetService.import_vector_bytes`, so Dataset, DatasetVersion and +VectorFeature rows remain one canonical flow. The browser never contacts the +provider directly. Exact request identities are reused for 24 hours. Buildings +return footprint area in hectares, roads return line length in kilometres, +water returns surface area plus supporting water-line length, and parcels +return mapped area. GRB cannot provide water volume, legal parcel boundaries +or traffic information. + +Settings: `GRB_ENABLED`, `GRB_OGC_API_URL`, `GRB_MIN_SIDE_M`, +`GRB_MAX_SIDE_M`, `GRB_PAGE_SIZE`, `GRB_MAX_PAGES`, `GRB_MAX_FEATURES`, +`GRB_TIMEOUT_SECONDS`, `GRB_MAX_RESPONSE_MB`, +`GRB_MAX_TOTAL_RESPONSE_MB` and `GRB_CACHE_TTL_HOURS`. + ## Governed DHMV terrain acquisition `GET /api/v1/projects/{project_id}/datasets/dhmv/products` exposes the fixed diff --git a/backend/app/api/routes/datasets.py b/backend/app/api/routes/datasets.py index 049f73a7..d46a1a0f 100644 --- a/backend/app/api/routes/datasets.py +++ b/backend/app/api/routes/datasets.py @@ -31,6 +31,7 @@ from app.schemas import ( BathymetryProfileAcquireRequest, ThematicRasterAcquireRequest, ThematicRasterSelectionRequest, + GrbAcquireRequest, VectorBBoxResponse, VectorBufferRequest, VectorClipRequest, @@ -49,6 +50,7 @@ from app.services.dataset_service import DatasetService from app.services.source_freshness_service import SourceFreshnessService from app.services.source_catalog_probe_service import SourceCatalogProbeService from app.services.grb_refresh_plan_service import GrbRefreshPlanService +from app.services.grb_acquisition_service import GrbAcquisitionService from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService from app.services.dhmv_acquisition_service import DhmvAcquisitionService from app.services.terrain_analysis_service import TerrainAnalysisService @@ -193,6 +195,30 @@ def list_dhmv_products(project_id: UUID, db: Session = Depends(get_db)): return envelope({"items": items, "total": len(items)}) +@router.post("/datasets/grb/acquire", response_model=dict) +def acquire_bounded_grb( + project_id: UUID, + payload: GrbAcquireRequest, + db: Session = Depends(get_db), +): + job = JobService.run_sync_job( + db=db, + project_id=project_id, + job_type="vector.grb.acquire", + parameters=payload.model_dump(mode="json"), + operation=lambda: GrbAcquisitionService.acquire(db, project_id, payload), + ) + return envelope(job) + + +@router.get("/datasets/grb/products", response_model=dict) +def list_grb_products(project_id: UUID, db: Session = Depends(get_db)): + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + items = GrbAcquisitionService.list_products() + return envelope({"items": items, "total": len(items)}) + + @router.post("/datasets/flood-hazard/acquire", response_model=dict) def acquire_bounded_flood_hazard( project_id: UUID, diff --git a/backend/app/api/routes/health.py b/backend/app/api/routes/health.py index f9bb6ef7..5d4914ae 100644 --- a/backend/app/api/routes/health.py +++ b/backend/app/api/routes/health.py @@ -39,7 +39,7 @@ def capabilities() -> dict: geopandas=_dependency_enabled("geopandas"), yolo=False, sam=False, - grb="planned", + grb="bounded", sentinel="planned", providers=providers, ).model_dump()} diff --git a/backend/app/core/config.py b/backend/app/core/config.py index a52b09aa..20878e09 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -68,6 +68,25 @@ class Settings(BaseSettings): le=86_400, validation_alias="SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS", ) + grb_enabled: bool = Field(default=True, validation_alias="GRB_ENABLED") + grb_ogc_api_url: str = Field( + default="https://geo.api.vlaanderen.be/GRB/ogc/features/v1", + validation_alias="GRB_OGC_API_URL", + ) + grb_min_side_m: float = Field(default=10.0, gt=0, validation_alias="GRB_MIN_SIDE_M") + grb_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="GRB_MAX_SIDE_M") + grb_page_size: int = Field(default=1000, ge=1, le=1000, validation_alias="GRB_PAGE_SIZE") + grb_max_pages: int = Field(default=200, ge=1, le=1000, validation_alias="GRB_MAX_PAGES") + grb_max_features: int = Field(default=150_000, ge=1, validation_alias="GRB_MAX_FEATURES") + grb_timeout_seconds: int = Field(default=180, ge=1, le=600, validation_alias="GRB_TIMEOUT_SECONDS") + grb_max_response_mb: int = Field(default=20, ge=1, le=100, validation_alias="GRB_MAX_RESPONSE_MB") + grb_max_total_response_mb: int = Field( + default=256, + ge=1, + le=2048, + validation_alias="GRB_MAX_TOTAL_RESPONSE_MB", + ) + grb_cache_ttl_hours: int = Field(default=24, ge=0, le=8760, validation_alias="GRB_CACHE_TTL_HOURS") dhmv_enabled: bool = Field(default=True, validation_alias="DHMV_ENABLED") dhmv_wcs_url: str = Field( default="https://geo.api.vlaanderen.be/DHMV/wcs", diff --git a/backend/app/providers/grb.py b/backend/app/providers/grb.py index 5773c0ef..16e08915 100644 --- a/backend/app/providers/grb.py +++ b/backend/app/providers/grb.py @@ -9,12 +9,26 @@ class GRBProvider(BaseReferenceProvider): provider_name="grb", display_name="GRB", authority_level="authoritative", - supported_layers=["buildings", "roads", "parcels"], + supported_layers=["buildings", "roads", "water", "parcels"], supported_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"], - supported_query_modes=["area"], - fetch_signature="POST /api/v1/external/grb/fetch", - limitation_message="GRB live WFS/download integration is not configured in Sprint 7B.", - attribution="Digitaal Vlaanderen - Basiskaart Vlaanderen (GRB)", - license_note="Use must follow Digitaal Vlaanderen open data and attribution terms.", - configured=False, + supported_query_modes=["bbox", "persisted_area"], + fetch_signature="POST /api/v1/projects/{project_id}/datasets/grb/acquire", + limitation_message=( + "Alleen expliciet begrensde selecties tot 20 km per zijde worden opgehaald. " + "Volledige providerdownloads en onbeperkte queries zijn niet toegestaan." + ), + attribution="Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen", + license_note="Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen.", + configured=True, ) + + def fetch(self, project_id: str, area_id: str | None, layers: list[str]) -> dict: + del project_id, area_id, layers + return { + "provider": self.provider_name, + "status": "bounded_request_required", + "message": ( + "Use POST /api/v1/projects/{project_id}/datasets/grb/acquire with an EPSG:4326 " + "bounding box and one governed product key." + ), + } diff --git a/backend/app/providers/registry.py b/backend/app/providers/registry.py index 44d7b0cc..cce248f9 100644 --- a/backend/app/providers/registry.py +++ b/backend/app/providers/registry.py @@ -78,11 +78,23 @@ class ExternalProviderRegistry: del project_id, area_id provider = self.get(provider_name) mapping = self.dataset_mapping(provider.provider_name, requested_dataset_role=requested_dataset_role) - if provider.provider_name in {"grb", "osm"}: + if provider.provider_name == "grb": + return ProviderImportResult( + provider_name="grb", + status="bounded_request_required", + message=( + "Use the governed project GRB acquisition endpoint with an EPSG:4326 bounding box " + "and one supported layer." + ), + requested_layers=layers, + dataset_role=mapping.dataset_role, + source_name=mapping.source_name, + ) + if provider.provider_name == "osm": return ProviderImportResult( provider_name=provider.provider_name, status="not_configured", - message=f"No live {provider.display_name} import is configured in Sprint 7B.", + message=f"No live {provider.display_name} import is configured.", requested_layers=layers, dataset_role=mapping.dataset_role, source_name=mapping.source_name, diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index 91b0f989..f3b51e01 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -17,6 +17,7 @@ from .source_catalog import ( SourceCatalogProbeSummary, ) from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary +from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead from .detection import ( DetectionListResponse, DetectionModelCapability, @@ -161,6 +162,9 @@ __all__ = [ "GrbRefreshLayerPlan", "GrbRefreshPlan", "GrbRefreshPlanSummary", + "GrbAcquireRequest", + "GrbAcquisitionResult", + "GrbProductRead", "DetectionListResponse", "DetectionModelCapability", "DetectionModelsResponse", diff --git a/backend/app/schemas/grb.py b/backend/app/schemas/grb.py new file mode 100644 index 00000000..86396a99 --- /dev/null +++ b/backend/app/schemas/grb.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from uuid import UUID + +from pydantic import BaseModel + +from .operations import VectorSelectionBBox + + +class GrbAcquireRequest(BaseModel): + bbox: VectorSelectionBBox + area_id: UUID | None = None + product_key: str = "buildings" + force_refresh: bool = False + + +class GrbProductRead(BaseModel): + key: str + display_name: str + reference_layer_name: str + collections: list[str] + geometry_types: list[str] + source_crs: str + authority_level: str + catalog_url: str + attribution: str + license_note: str + limitation_message: str + + +class GrbAcquisitionResult(BaseModel): + output_dataset_id: UUID + reused: bool + provider: str + product_key: str + display_name: str + reference_layer_name: str + collections: list[str] + feature_count: int + candidate_feature_count: int + page_count: int + bbox_epsg4326: list[float] + source_version: str + attribution: str + limitation_message: str diff --git a/backend/app/services/grb_acquisition_service.py b/backend/app/services/grb_acquisition_service.py new file mode 100644 index 00000000..dd94e136 --- /dev/null +++ b/backend/app/services/grb_acquisition_service.py @@ -0,0 +1,779 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +import hashlib +import json +import math +from pathlib import Path +from typing import Any, Callable +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse +from urllib.request import HTTPRedirectHandler, Request, build_opener +from uuid import UUID + +from geoalchemy2.shape import to_shape +from pyproj import Transformer +from shapely.geometry import LineString, MultiLineString, MultiPolygon, Polygon, box, mapping, shape +from shapely.ops import unary_union +from shapely.validation import make_valid + +from app.core.config import Settings, get_settings +from app.core.errors import AppError +from app.models import Area, Dataset, Project +from app.schemas.grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead +from app.services.dataset_service import DatasetService + + +class _RejectRedirects(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + del req, fp, code, msg, headers, newurl + return None + + +_NO_REDIRECT_OPENER = build_opener(_RejectRedirects()) + + +@dataclass(frozen=True) +class GrbCollection: + name: str + geometry_dimension: int + + +@dataclass(frozen=True) +class GrbProduct: + key: str + display_name: str + reference_layer_name: str + layer_type: str + collections: tuple[GrbCollection, ...] + geometry_types: tuple[str, ...] + metric_key: str + metric_method: str + metric_label: str + metric_unit: str + metric_dimension: int + metric_warning: str + limitation_message: str + + +class GrbAcquisitionService: + PROVIDER = "grb" + SOURCE_CRS = "EPSG:4326" + OGC_CRS84_URI = "http://www.opengis.net/def/crs/OGC/1.3/CRS84" + AUTHORITY_LEVEL = "authoritative" + ATTRIBUTION = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen" + LICENSE_NOTE = "Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen." + CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/basiskaart-vlaanderen-grb" + + @staticmethod + def _products() -> dict[str, GrbProduct]: + products = ( + GrbProduct( + key="buildings", + display_name="GRB gebouwcontouren", + reference_layer_name="buildings", + layer_type="building", + collections=(GrbCollection("GBG", 2),), + geometry_types=("Polygon", "MultiPolygon"), + metric_key="footprint_area", + metric_method="intersection_area", + metric_label="Bebouwde grondoppervlakte", + metric_unit="ha", + metric_dimension=2, + metric_warning=( + "Dit is de grondoppervlakte van gebouwcontouren, niet de totale vloeroppervlakte " + "of het gebouwvolume." + ), + limitation_message=( + "GRB GBG bevat gebouwcontouren uit de basiskaart. Registratie en fysieke verandering " + "kunnen in tijd verschillen." + ), + ), + GrbProduct( + key="roads", + display_name="GRB wegsegmenten", + reference_layer_name="roads", + layer_type="road", + collections=(GrbCollection("Wegsegment", 1),), + geometry_types=("LineString", "MultiLineString"), + metric_key="road_length", + metric_method="intersection_length", + metric_label="Totale weglengte", + metric_unit="km", + metric_dimension=1, + metric_warning=( + "De lengte volgt GRB-wegsegmenten en zegt niets over rijstroken, verkeersvolume " + "of verhardingsoppervlakte." + ), + limitation_message=( + "GRB Wegsegment beschrijft netwerkgeometrie en is geen verkeersmodel of routeadvies." + ), + ), + GrbProduct( + key="water", + display_name="GRB wateroppervlakken en waterlijnen", + reference_layer_name="water", + layer_type="water", + collections=( + GrbCollection("WTZ", 2), + GrbCollection("WLAS", 1), + GrbCollection("WGR", 1), + ), + geometry_types=("LineString", "MultiLineString", "Polygon", "MultiPolygon"), + metric_key="water_area", + metric_method="intersection_area", + metric_label="Wateroppervlakte", + metric_unit="ha", + metric_dimension=2, + metric_warning=( + "Watervolume is niet berekenbaar zonder betrouwbare diepte- of bathymetrische gegevens. " + "De GRB-bron levert alleen oppervlakte- en lijngeometrie." + ), + limitation_message=( + "GRB-water combineert wateroppervlakken en watergerelateerde lijnen. Objectaantallen en " + "oppervlakte zijn geen actueel waterpeil of watervolume." + ), + ), + GrbProduct( + key="parcels", + display_name="GRB administratieve percelen", + reference_layer_name="parcels", + layer_type="parcel", + collections=(GrbCollection("ADP", 2),), + geometry_types=("Polygon", "MultiPolygon"), + metric_key="parcel_area", + metric_method="intersection_area", + metric_label="Perceeloppervlakte", + metric_unit="ha", + metric_dimension=2, + metric_warning=( + "GRB-percelen zijn een grafische referentie en vormen geen juridische grensopmeting." + ), + limitation_message=( + "GRB ADP toont de vermoedelijke ligging van kadastrale percelen en is geen juridische grens." + ), + ), + ) + return {product.key: product for product in products} + + @staticmethod + def list_products() -> list[dict[str, Any]]: + return [ + GrbProductRead( + key=product.key, + display_name=product.display_name, + reference_layer_name=product.reference_layer_name, + collections=[collection.name for collection in product.collections], + geometry_types=list(product.geometry_types), + source_crs=GrbAcquisitionService.SOURCE_CRS, + authority_level=GrbAcquisitionService.AUTHORITY_LEVEL, + catalog_url=GrbAcquisitionService.CATALOG_URL, + attribution=GrbAcquisitionService.ATTRIBUTION, + license_note=GrbAcquisitionService.LICENSE_NOTE, + limitation_message=product.limitation_message, + ).model_dump() + for product in GrbAcquisitionService._products().values() + ] + + @staticmethod + def _product(product_key: str) -> GrbProduct: + product = GrbAcquisitionService._products().get(product_key.strip().lower()) + if product is None: + raise AppError( + code="GRB_PRODUCT_NOT_SUPPORTED", + message="Select buildings, roads, water or parcels from the governed GRB product registry", + details={"product_key": product_key}, + status_code=422, + ) + return product + + @staticmethod + def _validate_scope( + db, + project_id: UUID, + payload: GrbAcquireRequest, + settings: Settings, + ) -> tuple[Any, list[float], list[float]]: + if not settings.grb_enabled: + raise AppError(code="GRB_NOT_CONFIGURED", message="Bounded GRB acquisition is disabled", status_code=503) + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + if payload.bbox.crs.upper() != "EPSG:4326": + raise AppError(code="GRB_INVALID_CRS", message="GRB acquisition requires EPSG:4326", status_code=400) + values = (payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y) + if not all(math.isfinite(value) for value in values): + raise AppError(code="GRB_INVALID_BBOX", message="Bounding box values must be finite", status_code=400) + if values[0] >= values[2] or values[1] >= values[3]: + raise AppError(code="GRB_INVALID_BBOX", message="Bounding box has no area", status_code=400) + if values[0] < -180 or values[2] > 180 or values[1] < -90 or values[3] > 90: + raise AppError(code="GRB_INVALID_BBOX", message="Bounding box is outside EPSG:4326", status_code=400) + + transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) + metric_bounds = transformer.transform_bounds(*values, densify_pts=21) + width_m = float(metric_bounds[2] - metric_bounds[0]) + height_m = float(metric_bounds[3] - metric_bounds[1]) + if width_m < settings.grb_min_side_m or height_m < settings.grb_min_side_m: + raise AppError( + code="GRB_SELECTION_TOO_SMALL", + message=f"Select an area of at least {settings.grb_min_side_m:g} by {settings.grb_min_side_m:g} metres", + status_code=422, + ) + if width_m > settings.grb_max_side_m or height_m > settings.grb_max_side_m: + raise AppError( + code="GRB_SELECTION_TOO_LARGE", + message=f"Select an area no larger than {settings.grb_max_side_m:g} by {settings.grb_max_side_m:g} metres", + details={"width_m": width_m, "height_m": height_m}, + status_code=422, + ) + + selection = box(*values) + if payload.area_id is None: + scope_geometry = selection + else: + area = db.get(Area, payload.area_id) + if area is None: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + if area.project_id != project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400) + scope_geometry = to_shape(area.geometry).intersection(selection) + if scope_geometry.is_empty: + raise AppError( + code="GRB_SCOPE_EMPTY", + message="The requested bounding box does not intersect the selected area", + status_code=400, + ) + return scope_geometry, [float(value) for value in values], [float(value) for value in metric_bounds] + + @staticmethod + def _geometry_dimension(geometry: Any) -> int: + if geometry is None or geometry.is_empty: + return -1 + if "Polygon" in geometry.geom_type: + return 2 + if "LineString" in geometry.geom_type or geometry.geom_type == "LinearRing": + return 1 + if "Point" in geometry.geom_type: + return 0 + if hasattr(geometry, "geoms"): + return max((GrbAcquisitionService._geometry_dimension(item) for item in geometry.geoms), default=-1) + return -1 + + @staticmethod + def _extract_dimension(geometry: Any, expected_dimension: int) -> Any | None: + if geometry is None or geometry.is_empty: + return None + if not geometry.is_valid: + geometry = make_valid(geometry) + parts: list[Any] = [] + + def collect(candidate: Any) -> None: + if candidate is None or candidate.is_empty: + return + if expected_dimension == 2: + if isinstance(candidate, Polygon): + parts.append(candidate) + return + if isinstance(candidate, MultiPolygon): + parts.extend(item for item in candidate.geoms if not item.is_empty) + return + if expected_dimension == 1: + if isinstance(candidate, LineString): + parts.append(candidate) + return + if isinstance(candidate, MultiLineString): + parts.extend(item for item in candidate.geoms if not item.is_empty) + return + if hasattr(candidate, "geoms"): + for item in candidate.geoms: + collect(item) + + collect(geometry) + if not parts: + return None + normalized = unary_union(parts) + if normalized.is_empty: + return None + if not normalized.is_valid: + normalized = make_valid(normalized) + if ( + normalized.is_empty + or not normalized.is_valid + or GrbAcquisitionService._geometry_dimension(normalized) != expected_dimension + ): + return None + return normalized + + @staticmethod + def _collection_url(settings: Settings, collection: GrbCollection, bbox_values: tuple[float, ...]) -> str: + base = settings.grb_ogc_api_url.rstrip("/") + query = urlencode( + { + "f": "application/geo+json", + "limit": str(settings.grb_page_size), + "bbox": ",".join(f"{value:.8f}" for value in bbox_values), + "bbox-crs": GrbAcquisitionService.OGC_CRS84_URI, + "crs": GrbAcquisitionService.OGC_CRS84_URI, + } + ) + return f"{base}/collections/{collection.name}/items?{query}" + + @staticmethod + def _validated_page_url( + url: str, + settings: Settings, + product: GrbProduct, + bbox_values: tuple[float, ...], + ) -> str: + parsed = urlparse(url) + base = urlparse(settings.grb_ogc_api_url) + allowed_paths = { + f"{base.path.rstrip('/')}/collections/{collection.name}/items" + for collection in product.collections + } + if ( + parsed.scheme != "https" + or base.scheme != "https" + or parsed.netloc.casefold() != base.netloc.casefold() + or parsed.path not in allowed_paths + ): + raise AppError( + code="GRB_PROVIDER_INVALID_PAGINATION", + message="GRB returned a pagination URL outside the governed OGC API allowlist", + status_code=502, + ) + query = dict(parse_qsl(parsed.query, keep_blank_values=True)) + query.update( + { + "f": "application/geo+json", + "limit": str(settings.grb_page_size), + "bbox": ",".join(f"{value:.8f}" for value in bbox_values), + "bbox-crs": GrbAcquisitionService.OGC_CRS84_URI, + "crs": GrbAcquisitionService.OGC_CRS84_URI, + } + ) + return urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", urlencode(query), "")) + + @staticmethod + def _read_page( + url: str, + settings: Settings, + opener: Callable[..., Any] | None, + ) -> tuple[dict[str, Any], str, int]: + request = Request( + url, + headers={ + "Accept": "application/geo+json, application/json", + "User-Agent": "GeoIntel/1.0 bounded-grb-acquisition", + }, + ) + try: + with (opener or _NO_REDIRECT_OPENER.open)( + request, + timeout=settings.grb_timeout_seconds, + ) as response: + limit = settings.grb_max_response_mb * 1024 * 1024 + content = response.read(limit + 1) + except HTTPError as exc: + raise AppError( + code="GRB_PROVIDER_HTTP_ERROR", + message="The GRB OGC API returned an HTTP error", + details={"status_code": exc.code}, + status_code=502, + ) from exc + except (TimeoutError, URLError, OSError) as exc: + raise AppError( + code="GRB_PROVIDER_UNAVAILABLE", + message="The GRB OGC API is unavailable", + status_code=502, + ) from exc + if len(content) > limit: + raise AppError( + code="GRB_PROVIDER_RESPONSE_TOO_LARGE", + message="A GRB response page exceeded the configured size limit", + status_code=502, + ) + try: + payload = json.loads(content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise AppError( + code="GRB_PROVIDER_INVALID_RESPONSE", + message="The GRB OGC API returned invalid GeoJSON", + status_code=502, + ) from exc + if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection": + raise AppError( + code="GRB_PROVIDER_INVALID_RESPONSE", + message="The GRB OGC API returned a non-FeatureCollection response", + status_code=502, + ) + return payload, hashlib.sha256(content).hexdigest(), len(content) + + @staticmethod + def _next_url(payload: dict[str, Any], current_url: str) -> str | None: + links = payload.get("links") + if not isinstance(links, list): + return None + for link in links: + if isinstance(link, dict) and link.get("rel") == "next" and link.get("href"): + return urljoin(current_url, str(link["href"])) + return None + + @staticmethod + def _fetch_features( + product: GrbProduct, + scope_geometry: Any, + bbox_values: tuple[float, ...], + coverage_scope: str, + settings: Settings, + opener: Callable[..., Any] | None, + ) -> tuple[list[dict[str, Any]], dict[str, Any]]: + retained: list[dict[str, Any]] = [] + seen_ids: set[str] = set() + request_urls: list[str] = [] + response_sha256: list[str] = [] + candidate_feature_count = 0 + total_response_bytes = 0 + geometry_types: dict[str, int] = {} + collection_counts: dict[str, int] = {item.name: 0 for item in product.collections} + + for collection in product.collections: + url: str | None = GrbAcquisitionService._collection_url(settings, collection, bbox_values) + seen_pages: set[str] = set() + while url: + url = GrbAcquisitionService._validated_page_url(url, settings, product, bbox_values) + if url in seen_pages: + raise AppError( + code="GRB_PROVIDER_PAGINATION_LOOP", + message="The GRB OGC API repeated a pagination URL", + status_code=502, + ) + if len(request_urls) >= settings.grb_max_pages: + raise AppError( + code="GRB_SELECTION_TOO_LARGE", + message="GRB acquisition exceeded the configured page limit", + details={"max_pages": settings.grb_max_pages}, + status_code=422, + ) + seen_pages.add(url) + payload, response_hash, response_size = GrbAcquisitionService._read_page(url, settings, opener) + request_urls.append(url) + response_sha256.append(response_hash) + total_response_bytes += response_size + if total_response_bytes > settings.grb_max_total_response_mb * 1024 * 1024: + raise AppError( + code="GRB_PROVIDER_RESPONSE_TOO_LARGE", + message="The complete GRB response exceeded the configured transfer limit", + status_code=502, + ) + + source_features = payload.get("features") + if not isinstance(source_features, list): + raise AppError( + code="GRB_PROVIDER_INVALID_RESPONSE", + message="The GRB FeatureCollection has no valid feature list", + status_code=502, + ) + for source_feature in source_features: + candidate_feature_count += 1 + if not isinstance(source_feature, dict): + continue + raw_id = str(source_feature.get("id") or "").strip() + if not raw_id: + raise AppError( + code="GRB_PROVIDER_INVALID_RESPONSE", + message=f"GRB {collection.name} returned a feature without an official identity", + status_code=502, + ) + feature_id = f"{collection.name}:{raw_id}" + if feature_id in seen_ids: + continue + seen_ids.add(feature_id) + try: + source_geometry = GrbAcquisitionService._extract_dimension( + shape(source_feature.get("geometry")), + collection.geometry_dimension, + ) + except Exception as exc: + raise AppError( + code="GRB_PROVIDER_INVALID_GEOMETRY", + message=f"GRB {collection.name} returned invalid geometry", + status_code=502, + ) from exc + if source_geometry is None or not source_geometry.intersects(scope_geometry): + continue + retained_geometry = GrbAcquisitionService._extract_dimension( + source_geometry.intersection(scope_geometry), + collection.geometry_dimension, + ) + if retained_geometry is None: + continue + if len(retained) >= settings.grb_max_features: + raise AppError( + code="GRB_SELECTION_TOO_LARGE", + message="GRB selection exceeds the configured feature limit; draw a smaller rectangle", + details={"max_features": settings.grb_max_features}, + status_code=422, + ) + properties = dict(source_feature.get("properties") or {}) + properties.update( + { + "source_name": GrbAcquisitionService.PROVIDER, + "source_collection": collection.name, + "source_feature_id": feature_id, + "reference_layer_name": product.reference_layer_name, + "layer_type": product.layer_type, + "theme": product.key, + "authority_level": GrbAcquisitionService.AUTHORITY_LEVEL, + "coverage_scope": coverage_scope, + "geometry_clipped_to_selection": not scope_geometry.covers(source_geometry), + "attribution": GrbAcquisitionService.ATTRIBUTION, + } + ) + retained.append( + { + "type": "Feature", + "id": feature_id, + "geometry": mapping(retained_geometry), + "properties": properties, + } + ) + collection_counts[collection.name] += 1 + geometry_types[retained_geometry.geom_type] = geometry_types.get(retained_geometry.geom_type, 0) + 1 + url = GrbAcquisitionService._next_url(payload, url) + + return retained, { + "candidate_feature_count": candidate_feature_count, + "feature_count": len(retained), + "page_count": len(request_urls), + "request_urls": request_urls, + "response_sha256": response_sha256, + "response_size_bytes": total_response_bytes, + "collection_feature_counts": collection_counts, + "geometry_types": geometry_types, + "output_crs": GrbAcquisitionService.OGC_CRS84_URI, + "reference_truncated": False, + } + + @staticmethod + def _cached_dataset( + db, + project_id: UUID, + product: GrbProduct, + request_hash: str, + settings: Settings, + ) -> Dataset | None: + if settings.grb_cache_ttl_hours <= 0: + return None + candidates = ( + db.query(Dataset) + .filter( + Dataset.project_id == project_id, + Dataset.source_name == GrbAcquisitionService.PROVIDER, + Dataset.reference_layer_name == product.reference_layer_name, + Dataset.status == "ready", + ) + .order_by(Dataset.imported_at.desc()) + .all() + ) + cutoff = datetime.now(UTC) - timedelta(hours=settings.grb_cache_ttl_hours) + for candidate in candidates: + provenance = candidate.provenance_metadata if isinstance(candidate.provenance_metadata, dict) else {} + imported_at = candidate.imported_at + if ( + provenance.get("request_hash") == request_hash + and candidate.storage_path + and Path(candidate.storage_path).is_file() + and imported_at is not None + and imported_at >= cutoff + ): + return candidate + return None + + @staticmethod + def _result( + dataset: Dataset, + product: GrbProduct, + *, + reused: bool, + bbox_values: list[float], + ) -> dict[str, Any]: + source_metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {} + provenance = dataset.provenance_metadata if isinstance(dataset.provenance_metadata, dict) else {} + metadata = dataset.metadata_json if isinstance(dataset.metadata_json, dict) else {} + return GrbAcquisitionResult( + output_dataset_id=dataset.id, + reused=reused, + provider=GrbAcquisitionService.PROVIDER, + product_key=product.key, + display_name=product.display_name, + reference_layer_name=product.reference_layer_name, + collections=[collection.name for collection in product.collections], + feature_count=int(metadata.get("feature_count", source_metadata.get("feature_count", 0))), + candidate_feature_count=int(provenance.get("candidate_feature_count", 0)), + page_count=int(provenance.get("page_count", 0)), + bbox_epsg4326=bbox_values, + source_version=str(dataset.source_version or ""), + attribution=GrbAcquisitionService.ATTRIBUTION, + limitation_message=product.limitation_message, + ).model_dump(mode="json") + + @staticmethod + def acquire( + db, + project_id: UUID, + payload: GrbAcquireRequest, + *, + settings: Settings | None = None, + opener: Callable[..., Any] | None = None, + ) -> dict[str, Any]: + resolved_settings = settings or get_settings() + product = GrbAcquisitionService._product(payload.product_key) + scope_geometry, bbox_values, metric_bounds = GrbAcquisitionService._validate_scope( + db, + project_id, + payload, + resolved_settings, + ) + request_identity = { + "provider": GrbAcquisitionService.PROVIDER, + "product_key": product.key, + "bbox_epsg4326": [round(value, 8) for value in bbox_values], + "area_id": str(payload.area_id) if payload.area_id else None, + } + request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode()).hexdigest() + if not payload.force_refresh: + cached = GrbAcquisitionService._cached_dataset( + db, + project_id, + product, + request_hash, + resolved_settings, + ) + if cached is not None: + return GrbAcquisitionService._result(cached, product, reused=True, bbox_values=bbox_values) + + area = db.get(Area, payload.area_id) if payload.area_id else None + coverage_scope = ( + "municipality" + if area is not None and area.name.strip().lower().startswith("gemeente ") + else "bounded_selection" + ) + features, transfer = GrbAcquisitionService._fetch_features( + product, + scope_geometry, + tuple(scope_geometry.bounds), + coverage_scope, + resolved_settings, + opener, + ) + acquired_at = datetime.now(UTC) + source_version = acquired_at.date().isoformat() + feature_collection = { + "type": "FeatureCollection", + "name": product.display_name, + "crs": {"type": "name", "properties": {"name": "EPSG:4326"}}, + "features": features, + } + artifact = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + filename = f"grb_{product.key}_{source_version}_{request_hash[:12]}.geojson" + source_metadata: dict[str, Any] = { + "provider": "Digitaal Vlaanderen", + "service": "OGC API Features", + "product_key": product.key, + "product_display_name": product.display_name, + "collections": [collection.name for collection in product.collections], + "authority_level": GrbAcquisitionService.AUTHORITY_LEVEL, + "theme": product.key, + "layer_type": product.layer_type, + "coverage_scope": coverage_scope, + "geometry_clipped_to_area": payload.area_id is not None, + "geometry_clipped_to_selection": True, + "bbox_epsg4326": bbox_values, + "bbox_epsg31370": metric_bounds, + "feature_count": len(features), + "collection_feature_counts": transfer["collection_feature_counts"], + "identity_stable": True, + "identity_scheme": "grb_ogc_feature_id", + "source_storage_crs": "EPSG:31370", + "requested_output_crs": GrbAcquisitionService.OGC_CRS84_URI, + "selection_aggregation": { + "metric_key": product.metric_key, + "method": product.metric_method, + "label": product.metric_label, + "unit": product.metric_unit, + "geometry_dimension": product.metric_dimension, + "is_estimate": False, + "warning": product.metric_warning, + }, + "attribution": GrbAcquisitionService.ATTRIBUTION, + "license_note": GrbAcquisitionService.LICENSE_NOTE, + "catalog_url": GrbAcquisitionService.CATALOG_URL, + "limitation_message": product.limitation_message, + } + if product.key == "water": + source_metadata["selection_metrics"] = [ + { + "metric_key": "water_length", + "method": "intersection_length", + "label": "Lengte watergerelateerde lijnen", + "unit": "km", + "geometry_dimension": 1, + "is_estimate": False, + } + ] + try: + dataset_response = DatasetService.import_vector_bytes( + db, + project_id=project_id, + area_id=payload.area_id, + filename=filename, + content=artifact, + source="Digitaal Vlaanderen GRB OGC API Features", + source_name=GrbAcquisitionService.PROVIDER, + dataset_role="reference", + reference_layer_name=product.reference_layer_name, + temporal_series_key=f"grb:{product.key}:{request_hash[:24]}", + observed_at=datetime( + acquired_at.year, + acquired_at.month, + acquired_at.day, + tzinfo=UTC, + ), + temporal_granularity="snapshot", + source_version=source_version, + source_metadata=source_metadata, + provenance_metadata={ + "acquisition": "explicit_bounded_ogc_api_features", + "acquired_at": acquired_at.isoformat(), + "request_hash": request_hash, + "request_urls": transfer["request_urls"], + "response_sha256": transfer["response_sha256"], + "response_size_bytes": transfer["response_size_bytes"], + "page_count": transfer["page_count"], + "candidate_feature_count": transfer["candidate_feature_count"], + "exact_feature_count": transfer["feature_count"], + "reference_truncated": False, + "artifact_sha256": hashlib.sha256(artifact).hexdigest(), + "clipped_to_area_id": str(payload.area_id) if payload.area_id else None, + "scope_geometry_type": scope_geometry.geom_type, + "limitation_message": product.limitation_message, + }, + ) + except AppError: + raise + except Exception as exc: + raise AppError( + code="GRB_PERSISTENCE_FAILED", + message="The validated GRB selection could not be persisted", + details={"reason": str(exc)}, + status_code=500, + ) from exc + persisted = db.get(Dataset, dataset_response.id) + if persisted is None: + raise AppError( + code="GRB_PERSISTENCE_FAILED", + message="The persisted GRB dataset could not be reloaded", + status_code=500, + ) + return GrbAcquisitionService._result(persisted, product, reused=False, bbox_values=bbox_values) diff --git a/backend/tests/test_sprint118_yolo_preflight_ui.py b/backend/tests/test_sprint118_yolo_preflight_ui.py index bb0163b8..d1994cfc 100644 --- a/backend/tests/test_sprint118_yolo_preflight_ui.py +++ b/backend/tests/test_sprint118_yolo_preflight_ui.py @@ -54,4 +54,5 @@ def test_detection_lab_surfaces_local_model_asset_selection() -> None: assert "onSelectModelAsset" in lab assert "modelAssets={modelAssets}" in app assert "Officiële referentiebronnen" in provider_panel - assert "live GRB- en OSM-koppelingen staan uit" in provider_panel + assert "GRB is beschikbaar voor expliciet begrensde kaartselecties" in provider_panel + assert "OSM blijft uitgeschakeld" in provider_panel diff --git a/backend/tests/test_sprint237_flanders_thematic_on_demand.py b/backend/tests/test_sprint237_flanders_thematic_on_demand.py index 84466bff..4fb343af 100644 --- a/backend/tests/test_sprint237_flanders_thematic_on_demand.py +++ b/backend/tests/test_sprint237_flanders_thematic_on_demand.py @@ -10,12 +10,12 @@ def read(path: str) -> str: def test_flanders_workspace_exposes_governed_thematic_products_on_demand() -> None: workspace = read("frontend/src/components/map/MapWorkspace.tsx") - product_hook = read("frontend/src/hooks/useOfficialRasterProducts.ts") + product_hook = read("frontend/src/hooks/useOfficialMapProducts.ts") selection_hook = read("frontend/src/hooks/useMapThemeSelectionInsights.ts") api = read("frontend/src/services/api/datasets.ts") assert "activeScopeProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME" in workspace - assert "new Map" in workspace + assert "new Map" in workspace assert "'Op aanvraag'" in workspace assert "theme.id === 'space_occupation'" in workspace assert "setActiveThemeId(fallbackTheme.id)" in workspace @@ -36,23 +36,23 @@ def test_selection_runs_all_available_themes_and_refreshes_persisted_datasets() assert "for (const theme of DATA_THEMES)" in workspace assert "await loadThemeInsights(bbox, availableThemes, areaId)" in workspace - assert "!regionalPartitionedThemeActive && !onDemandRasterThemeActive" in workspace + assert "!regionalPartitionedThemeActive && !onDemandThemeActive" in workspace assert "onRefreshProjectData" in workspace assert "selectedProjectId ? loadProjectData(selectedProjectId)" in app assert "successful.some((item) => item.acquisition)" in selection_hook assert "await onDatasetsChanged()" in selection_hook -def test_regional_on_demand_rasters_require_a_bounded_drawn_selection() -> None: +def test_regional_on_demand_sources_require_a_bounded_drawn_selection() -> None: workspace = read("frontend/src/components/map/MapWorkspace.tsx") - assert "regionalOnDemandRasterThemeActive" in workspace - assert "regionalRasterThemeActive || regionalOnDemandRasterThemeActive" in workspace - assert "Teken een begrensde rechthoek voor een regionale rasteranalyse." in workspace - assert "officiële Vlaamse rasters worden begrensd opgehaald, bewaard en hergebruikt" in workspace + assert "regionalOnDemandThemeActive" in workspace + assert "regionalRasterThemeActive || regionalOnDemandThemeActive" in workspace + assert "Teken een begrensde rechthoek voor deze regionale analyse." in workspace + assert "Vlaamse kaartbronnen worden begrensd opgehaald, bewaard en hergebruikt" in workspace -def test_frontend_does_not_contact_the_external_wcs_directly() -> None: +def test_frontend_does_not_contact_external_map_services_directly() -> None: frontend_sources = "\n".join( path.read_text(encoding="utf-8") for path in (ROOT / "frontend/src").rglob("*") @@ -60,4 +60,5 @@ def test_frontend_does_not_contact_the_external_wcs_directly() -> None: ) assert "mercatornet.be" not in frontend_sources.casefold() + assert "geo.api.vlaanderen.be" not in frontend_sources.casefold() assert "GetCoverage" not in frontend_sources diff --git a/backend/tests/test_sprint238_flanders_raster_catalogs.py b/backend/tests/test_sprint238_flanders_raster_catalogs.py index 79a9be51..5edade02 100644 --- a/backend/tests/test_sprint238_flanders_raster_catalogs.py +++ b/backend/tests/test_sprint238_flanders_raster_catalogs.py @@ -11,12 +11,13 @@ def read(path: str) -> str: return (ROOT / path).read_text(encoding="utf-8") -def test_official_raster_catalog_hook_loads_all_governed_registries() -> None: - hook = read("frontend/src/hooks/useOfficialRasterProducts.ts") +def test_official_map_catalog_hook_loads_all_governed_registries() -> None: + hook = read("frontend/src/hooks/useOfficialMapProducts.ts") assert "datasetsApi.listThematicRasterProducts" in hook assert "datasetsApi.listDhmvProducts" in hook assert "datasetsApi.listFloodHazardProducts" in hook + assert "datasetsApi.listGrbProducts" in hook assert "Promise.all([" in hook @@ -24,7 +25,7 @@ def test_map_selection_can_acquire_dhmv_and_flood_hazard_products() -> None: selection_hook = read("frontend/src/hooks/useMapThemeSelectionInsights.ts") workspace = read("frontend/src/components/map/MapWorkspace.tsx") - assert "'thematic_raster' | 'dhmv' | 'flood_hazard'" in selection_hook + assert "'thematic_raster' | 'dhmv' | 'flood_hazard' | 'grb'" in selection_hook assert "datasetsApi.acquireDhmv" in selection_hook assert "datasetsApi.acquireFloodHazard" in selection_hook assert "datasetsApi.acquireThematicRaster" in selection_hook diff --git a/backend/tests/test_sprint239_bounded_grb_acquisition.py b/backend/tests/test_sprint239_bounded_grb_acquisition.py new file mode 100644 index 00000000..915a8224 --- /dev/null +++ b/backend/tests/test_sprint239_bounded_grb_acquisition.py @@ -0,0 +1,392 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from urllib.parse import parse_qs, urlparse +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from geoalchemy2.shape import from_shape +from shapely.geometry import MultiPolygon, Polygon + +from app.core.config import Settings +from app.core.errors import AppError +from app.db.session import get_db +from app.main import app +from app.models import Area, Dataset, Job, Project +from app.schemas.grb import GrbAcquireRequest +from app.services.dataset_service import DatasetService +from app.services.grb_acquisition_service import GrbAcquisitionService + + +ROOT = Path(__file__).resolve().parents[2] + + +class FakeQuery: + def __init__(self, result=None): + self.result = result + + def filter(self, *_args): + return self + + def order_by(self, *_args): + return self + + def all(self): + return self.result if isinstance(self.result, list) else [] + + +class FakeSession: + def __init__(self, rows=None, query_result=None): + self.rows = rows or {} + self.query_result = query_result + self.added = [] + + def get(self, model, row_id): + row = self.rows.get((model, row_id)) + if row is not None: + return row + return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None) + + def add(self, row): + self.added.append(row) + + def commit(self): + return None + + def rollback(self): + return None + + def refresh(self, row): + return row + + def query(self, _model): + return FakeQuery(self.query_result) + + +class JsonResponse: + def __init__(self, payload): + self.content = json.dumps(payload).encode("utf-8") + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, size=-1): + return self.content if size < 0 else self.content[:size] + + +def request(*, product_key="buildings", area_id=None, force_refresh=True) -> GrbAcquireRequest: + return GrbAcquireRequest( + bbox={ + "min_x": 5.15, + "min_y": 51.18, + "max_x": 5.17, + "max_y": 51.20, + "crs": "EPSG:4326", + }, + area_id=area_id, + product_key=product_key, + force_refresh=force_refresh, + ) + + +def polygon_feature(feature_id: str, coordinates) -> dict: + return { + "type": "Feature", + "id": feature_id, + "geometry": {"type": "Polygon", "coordinates": [coordinates]}, + "properties": {"source_field": feature_id}, + } + + +def test_grb_registry_exposes_four_governed_products() -> None: + products = {item["key"]: item for item in GrbAcquisitionService.list_products()} + + assert set(products) == {"buildings", "roads", "water", "parcels"} + assert products["buildings"]["collections"] == ["GBG"] + assert products["roads"]["collections"] == ["Wegsegment"] + assert products["water"]["collections"] == ["WTZ", "WLAS", "WGR"] + assert products["parcels"]["collections"] == ["ADP"] + assert all(item["authority_level"] == "authoritative" for item in products.values()) + + +def test_grb_fetch_follows_pagination_clips_geometry_and_preserves_official_identity() -> None: + product = GrbAcquisitionService._product("buildings") + settings = Settings(_env_file=None) + pages = [] + + first = polygon_feature( + "GBG.1", + [(5.155, 51.185), (5.175, 51.185), (5.175, 51.195), (5.155, 51.195), (5.155, 51.185)], + ) + second = polygon_feature( + "GBG.2", + [(5.151, 51.181), (5.152, 51.181), (5.152, 51.182), (5.151, 51.182), (5.151, 51.181)], + ) + outside = polygon_feature( + "GBG.3", + [(5.3, 51.3), (5.31, 51.3), (5.31, 51.31), (5.3, 51.31), (5.3, 51.3)], + ) + + def opener(raw_request, timeout): + assert timeout == settings.grb_timeout_seconds + parsed = urlparse(raw_request.full_url) + query = parse_qs(parsed.query) + pages.append(raw_request.full_url) + assert query["bbox-crs"] == [GrbAcquisitionService.OGC_CRS84_URI] + assert query["crs"] == [GrbAcquisitionService.OGC_CRS84_URI] + if query.get("cursor") == ["next"]: + return JsonResponse({"type": "FeatureCollection", "features": [second, outside], "links": []}) + return JsonResponse( + { + "type": "FeatureCollection", + "features": [first], + "links": [ + { + "rel": "next", + "href": ( + "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/" + "collections/GBG/items?cursor=next" + ), + } + ], + } + ) + + scope = Polygon( + [(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)] + ) + features, transfer = GrbAcquisitionService._fetch_features( + product, + scope, + scope.bounds, + "bounded_selection", + settings, + opener, + ) + + assert len(pages) == 2 + assert transfer["page_count"] == 2 + assert transfer["candidate_feature_count"] == 3 + assert transfer["feature_count"] == 2 + assert transfer["reference_truncated"] is False + assert {feature["id"] for feature in features} == {"GBG:GBG.1", "GBG:GBG.2"} + clipped = next(feature for feature in features if feature["id"] == "GBG:GBG.1") + assert clipped["properties"]["source_feature_id"] == "GBG:GBG.1" + assert clipped["properties"]["geometry_clipped_to_selection"] is True + assert clipped["properties"]["coverage_scope"] == "bounded_selection" + + +def test_grb_fetch_rejects_untrusted_pagination_and_unbounded_feature_volume() -> None: + product = GrbAcquisitionService._product("buildings") + settings = Settings(_env_file=None) + feature = polygon_feature( + "GBG.1", + [(5.151, 51.181), (5.152, 51.181), (5.152, 51.182), (5.151, 51.182), (5.151, 51.181)], + ) + scope = Polygon( + [(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)] + ) + + def hostile_opener(_request, timeout): + del timeout + return JsonResponse( + { + "type": "FeatureCollection", + "features": [feature], + "links": [{"rel": "next", "href": "https://example.test/private"}], + } + ) + + with pytest.raises(AppError) as invalid_next: + GrbAcquisitionService._fetch_features( + product, + scope, + scope.bounds, + "bounded_selection", + settings, + hostile_opener, + ) + assert invalid_next.value.code == "GRB_PROVIDER_INVALID_PAGINATION" + + def oversized_opener(_request, timeout): + del timeout + return JsonResponse( + { + "type": "FeatureCollection", + "features": [ + feature, + {**feature, "id": "GBG.2"}, + ], + "links": [], + } + ) + + with pytest.raises(AppError) as oversized: + GrbAcquisitionService._fetch_features( + product, + scope, + scope.bounds, + "bounded_selection", + Settings(_env_file=None, GRB_MAX_FEATURES=1), + oversized_opener, + ) + assert oversized.value.code == "GRB_SELECTION_TOO_LARGE" + + +def test_grb_acquisition_rejects_large_scope_before_network_access() -> None: + project_id = uuid4() + db = FakeSession({(Project, project_id): Project(id=project_id, name="Vlaanderen")}) + payload = GrbAcquireRequest( + bbox={"min_x": 4.0, "min_y": 50.7, "max_x": 5.0, "max_y": 51.7, "crs": "EPSG:4326"}, + product_key="buildings", + ) + + with pytest.raises(AppError) as exc_info: + GrbAcquisitionService.acquire(db, project_id, payload, settings=Settings(_env_file=None)) + + assert exc_info.value.code == "GRB_SELECTION_TOO_LARGE" + + +def test_grb_acquisition_persists_via_dataset_service_with_selection_metrics(monkeypatch) -> None: + project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4() + municipality = MultiPolygon( + [ + Polygon( + [(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)] + ) + ] + ) + project = Project(id=project_id, name="Vlaanderen") + area = Area( + id=area_id, + project_id=project_id, + name="Gemeente Mol - officieel", + geometry=from_shape(municipality, srid=4326), + ) + db = FakeSession({(Project, project_id): project, (Area, area_id): area}) + captured = {} + + def opener(_request, timeout): + del timeout + parsed = urlparse(_request.full_url) + collection = parsed.path.split("/")[-2] + if collection == "WTZ": + features = [ + polygon_feature( + "WTZ.1", + [(5.151, 51.181), (5.16, 51.181), (5.16, 51.19), (5.151, 51.19), (5.151, 51.181)], + ) + ] + else: + features = [] + return JsonResponse({"type": "FeatureCollection", "features": features, "links": []}) + + def persist(_db, **kwargs): + captured.update(kwargs) + dataset = Dataset( + id=dataset_id, + project_id=project_id, + area_id=area_id, + name=kwargs["filename"], + dataset_type="vector", + source=kwargs["source"], + dataset_role=kwargs["dataset_role"], + source_name=kwargs["source_name"], + reference_layer_name=kwargs["reference_layer_name"], + temporal_series_key=kwargs["temporal_series_key"], + observed_at=kwargs["observed_at"], + source_version=kwargs["source_version"], + source_metadata=kwargs["source_metadata"], + provenance_metadata=kwargs["provenance_metadata"], + metadata_json={"feature_count": 1}, + status="ready", + ) + db.rows[(Dataset, dataset_id)] = dataset + return SimpleNamespace(id=dataset_id) + + monkeypatch.setattr(DatasetService, "import_vector_bytes", persist) + result = GrbAcquisitionService.acquire( + db, + project_id, + request(product_key="water", area_id=area_id), + settings=Settings(_env_file=None), + opener=opener, + ) + + assert result["output_dataset_id"] == str(dataset_id) + assert result["feature_count"] == 1 + assert captured["dataset_role"] == "reference" + assert captured["source_name"] == "grb" + assert captured["reference_layer_name"] == "water" + assert captured["source_metadata"]["coverage_scope"] == "municipality" + assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == "water_area" + assert captured["source_metadata"]["selection_metrics"][0]["metric_key"] == "water_length" + assert captured["provenance_metadata"]["reference_truncated"] is False + collection = json.loads(captured["content"]) + assert collection["features"][0]["properties"]["coverage_scope"] == "municipality" + + +def test_grb_routes_use_canonical_envelopes_and_existing_job_contract(monkeypatch) -> None: + project_id, dataset_id = uuid4(), uuid4() + db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}) + monkeypatch.setattr( + GrbAcquisitionService, + "acquire", + lambda *_args, **_kwargs: { + "output_dataset_id": str(dataset_id), + "provider": "grb", + "product_key": "buildings", + "feature_count": 2, + }, + ) + app.dependency_overrides[get_db] = lambda: db + try: + client = TestClient(app) + products_response = client.get(f"/api/v1/projects/{project_id}/datasets/grb/products") + acquire_response = client.post( + f"/api/v1/projects/{project_id}/datasets/grb/acquire", + json=request().model_dump(mode="json"), + ) + finally: + app.dependency_overrides.clear() + + assert products_response.status_code == 200 + assert set(products_response.json()) == {"data"} + assert products_response.json()["data"]["total"] == 4 + assert acquire_response.status_code == 200 + assert set(acquire_response.json()) == {"data"} + assert acquire_response.json()["data"]["job_type"] == "vector.grb.acquire" + assert acquire_response.json()["data"]["output_dataset_id"] == str(dataset_id) + assert any(isinstance(item, Job) for item in db.added) + + +def test_system_capabilities_reports_bounded_grb_integration() -> None: + response = TestClient(app).get("/api/v1/system/capabilities") + + assert response.status_code == 200 + assert response.json()["data"]["grb"] == "bounded" + grb = next( + item for item in response.json()["data"]["providers"] + if item["provider_name"] == "grb" + ) + assert grb["status"] == "configured" + assert grb["fetch_signature"].endswith("/datasets/grb/acquire") + + +def test_grb_frontend_and_contracts_use_only_the_governed_backend_path() -> None: + selection_hook = (ROOT / "frontend/src/hooks/useMapThemeSelectionInsights.ts").read_text(encoding="utf-8") + catalog_hook = (ROOT / "frontend/src/hooks/useOfficialMapProducts.ts").read_text(encoding="utf-8") + workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8") + contracts = (ROOT / "docs/API_CONTRACTS.md").read_text(encoding="utf-8") + + assert "datasetsApi.acquireGrb" in selection_hook + assert "datasetsApi.listGrbProducts" in catalog_hook + assert "officialMapProducts.grb" in workspace + assert "/datasets/grb/acquire" in contracts + assert "geo.api.vlaanderen.be" not in workspace diff --git a/backend/tests/test_sprint7a_persistence_foundation.py b/backend/tests/test_sprint7a_persistence_foundation.py index 49abae0d..be11a73e 100644 --- a/backend/tests/test_sprint7a_persistence_foundation.py +++ b/backend/tests/test_sprint7a_persistence_foundation.py @@ -286,10 +286,10 @@ def test_provider_capabilities_expose_sprint7a_contract() -> None: assert capabilities["osm"]["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"] assert capabilities["osm"]["supported_query_modes"] == ["area"] assert capabilities["osm"]["status"] == "not_configured" - assert capabilities["grb"]["supported_layers"] == ["buildings", "roads", "parcels"] + assert capabilities["grb"]["supported_layers"] == ["buildings", "roads", "water", "parcels"] assert capabilities["grb"]["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"] - assert capabilities["grb"]["supported_query_modes"] == ["area"] - assert capabilities["grb"]["status"] == "not_configured" + assert capabilities["grb"]["supported_query_modes"] == ["bbox", "persisted_area"] + assert capabilities["grb"]["status"] == "configured" def test_sprint7a_migration_declares_foundation_tables_and_indexes() -> None: diff --git a/backend/tests/test_sprint7b_provider_registry.py b/backend/tests/test_sprint7b_provider_registry.py index ed9869df..a788cac9 100644 --- a/backend/tests/test_sprint7b_provider_registry.py +++ b/backend/tests/test_sprint7b_provider_registry.py @@ -16,8 +16,8 @@ def test_provider_registry_lists_sprint7b_providers() -> None: assert set(providers) == {"grb", "osm", "manual", "fixture"} assert providers["grb"].authority_level == "authoritative" - assert providers["grb"].configured is False - assert providers["grb"].status == "not_configured" + assert providers["grb"].configured is True + assert providers["grb"].status == "configured" assert providers["osm"].authority_level == "contextual" assert providers["osm"].configured is False assert providers["osm"].status == "not_configured" @@ -34,9 +34,9 @@ def test_provider_capabilities_include_required_metadata() -> None: assert grb["provider_name"] == "grb" assert grb["display_name"] == "GRB" - assert grb["supported_layers"] == ["buildings", "roads", "parcels"] + assert grb["supported_layers"] == ["buildings", "roads", "water", "parcels"] assert grb["supported_geometry_types"] == ["Polygon", "MultiPolygon", "LineString", "MultiLineString"] - assert grb["supported_query_modes"] == ["area"] + assert grb["supported_query_modes"] == ["bbox", "persisted_area"] assert grb["limitation_message"] assert grb["attribution"] assert grb["license_note"] @@ -56,13 +56,13 @@ def test_provider_to_dataset_mapping_is_enforced() -> None: assert get_provider_dataset_mapping("osm", requested_dataset_role="reference").dataset_role == "reference" -def test_grb_osm_import_contract_returns_not_configured_without_fetching() -> None: +def test_grb_import_contract_requires_bounded_request_and_osm_remains_not_configured() -> None: grb = import_provider_dataset("grb", project_id="project", area_id="area", layers=["buildings"]) osm = import_provider_dataset("osm", project_id="project", area_id="area", layers=["buildings"]) - assert grb.status == "not_configured" + assert grb.status == "bounded_request_required" assert grb.dataset_id is None - assert "No live" in grb.message + assert "bounding box" in grb.message assert osm.status == "not_configured" assert osm.dataset_id is None @@ -107,7 +107,7 @@ def test_provider_api_envelopes_and_invalid_provider() -> None: assert invalid_response.json()["message"] == "Provider not found" -def test_provider_import_api_returns_clear_not_configured_response() -> None: +def test_provider_import_api_points_grb_to_governed_bounded_endpoint() -> None: client = TestClient(app) response = client.post( @@ -121,7 +121,7 @@ def test_provider_import_api_returns_clear_not_configured_response() -> None: assert response.status_code == 200 assert response.json()["data"]["provider_name"] == "grb" - assert response.json()["data"]["status"] == "not_configured" + assert response.json()["data"]["status"] == "bounded_request_required" assert response.json()["data"]["dataset_id"] is None diff --git a/docker-compose.unraid.yml b/docker-compose.unraid.yml index b68dc52a..0e03a5d2 100644 --- a/docker-compose.unraid.yml +++ b/docker-compose.unraid.yml @@ -27,6 +27,17 @@ services: ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24} SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true} SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs} + GRB_ENABLED: ${GRB_ENABLED:-true} + GRB_OGC_API_URL: ${GRB_OGC_API_URL:-https://geo.api.vlaanderen.be/GRB/ogc/features/v1} + GRB_MIN_SIDE_M: ${GRB_MIN_SIDE_M:-10} + GRB_MAX_SIDE_M: ${GRB_MAX_SIDE_M:-20000} + GRB_PAGE_SIZE: ${GRB_PAGE_SIZE:-1000} + GRB_MAX_PAGES: ${GRB_MAX_PAGES:-200} + GRB_MAX_FEATURES: ${GRB_MAX_FEATURES:-150000} + GRB_TIMEOUT_SECONDS: ${GRB_TIMEOUT_SECONDS:-180} + GRB_MAX_RESPONSE_MB: ${GRB_MAX_RESPONSE_MB:-20} + GRB_MAX_TOTAL_RESPONSE_MB: ${GRB_MAX_TOTAL_RESPONSE_MB:-256} + GRB_CACHE_TTL_HOURS: ${GRB_CACHE_TTL_HOURS:-24} SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS: ${SOURCE_CATALOG_PROBE_TIMEOUT_SECONDS:-10} SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB: ${SOURCE_CATALOG_PROBE_MAX_RESPONSE_MB:-2} SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS: ${SOURCE_CATALOG_PROBE_CACHE_TTL_SECONDS:-900} diff --git a/docker-compose.yml b/docker-compose.yml index b26720d6..5cc9e3c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,17 @@ services: ORTHOPHOTO_CACHE_TTL_HOURS: ${ORTHOPHOTO_CACHE_TTL_HOURS:-24} SOURCE_CATALOG_PROBE_ENABLED: ${SOURCE_CATALOG_PROBE_ENABLED:-true} SOURCE_CATALOG_GRB_WFS_URL: ${SOURCE_CATALOG_GRB_WFS_URL:-https://geo.api.vlaanderen.be/GRB/wfs} + GRB_ENABLED: ${GRB_ENABLED:-true} + GRB_OGC_API_URL: ${GRB_OGC_API_URL:-https://geo.api.vlaanderen.be/GRB/ogc/features/v1} + GRB_MIN_SIDE_M: ${GRB_MIN_SIDE_M:-10} + GRB_MAX_SIDE_M: ${GRB_MAX_SIDE_M:-20000} + GRB_PAGE_SIZE: ${GRB_PAGE_SIZE:-1000} + GRB_MAX_PAGES: ${GRB_MAX_PAGES:-200} + GRB_MAX_FEATURES: ${GRB_MAX_FEATURES:-150000} + GRB_TIMEOUT_SECONDS: ${GRB_TIMEOUT_SECONDS:-180} + GRB_MAX_RESPONSE_MB: ${GRB_MAX_RESPONSE_MB:-20} + GRB_MAX_TOTAL_RESPONSE_MB: ${GRB_MAX_TOTAL_RESPONSE_MB:-256} + GRB_CACHE_TTL_HOURS: ${GRB_CACHE_TTL_HOURS:-24} SOURCE_CATALOG_STATBEL_DCAT_URL: ${SOURCE_CATALOG_STATBEL_DCAT_URL:-https://doc.statbel.be/publications/DCAT/DCAT_opendata_datasets.ttl} SOURCE_CATALOG_STATBEL_MAX_RESPONSE_MB: ${SOURCE_CATALOG_STATBEL_MAX_RESPONSE_MB:-5} SOURCE_CATALOG_ALZ_RELEASE_URL: ${SOURCE_CATALOG_ALZ_RELEASE_URL:-https://landbouwcijfers.vlaanderen.be/open-geodata-landbouwgebruikspercelen} diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 2740249e..ac6ebe22 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -64,23 +64,23 @@ Returns enabled feature flags and tool availability. "geopandas": true, "yolo": false, "sam": false, - "grb": "planned", + "grb": "bounded", "sentinel": "planned", "providers": [ { "provider_name": "grb", "display_name": "GRB", "authority_level": "authoritative", - "supported_layers": ["buildings", "roads", "parcels"], + "supported_layers": ["buildings", "roads", "water", "parcels"], "supported_geometry_types": ["Polygon", "MultiPolygon", "LineString", "MultiLineString"], - "supported_query_modes": ["area"], - "fetch_signature": "POST /api/v1/external/grb/fetch", - "configured": false, - "status": "not_configured", - "limitation_message": "GRB live WFS/download integration is not configured in Sprint 7B.", - "attribution": "Digitaal Vlaanderen - Basiskaart Vlaanderen (GRB)", - "license_note": "Use must follow Digitaal Vlaanderen open data and attribution terms.", - "not_configured_reason": "Provider integration is not configured yet" + "supported_query_modes": ["bbox", "persisted_area"], + "fetch_signature": "POST /api/v1/projects/{project_id}/datasets/grb/acquire", + "configured": true, + "status": "configured", + "limitation_message": "Alleen expliciet begrensde selecties tot 20 km per zijde worden opgehaald.", + "attribution": "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen", + "license_note": "Hergebruik volgens de open-datavoorwaarden en bronvermelding van Digitaal Vlaanderen.", + "not_configured_reason": null } ] } @@ -233,6 +233,45 @@ dimensions, EPSG:4326/EPSG:31370 bounds, sampling resolution, attribution, cache reuse and limitation text. Historical products also persist their observation/validity period and a spatially scoped temporal-series key. +### GET `/api/v1/projects/{project_id}/datasets/grb/products` + +Returns the fixed official GRB vector registry in the canonical envelope. +Only `buildings` (`GBG`), `roads` (`Wegsegment`), `water` +(`WTZ`/`WLAS`/`WGR`) and `parcels` (`ADP`) are exposed. Each product reports +its exact source collections, supported geometry types, attribution, licence +and semantic limitation. + +### POST `/api/v1/projects/{project_id}/datasets/grb/acquire` + +Acquires one explicitly bounded GRB product from the allowlisted Digitaal +Vlaanderen OGC API Features service behind the synchronous Job abstraction: + +```json +{ + "bbox": {"min_x": 5.12, "min_y": 51.18, "max_x": 5.17, "max_y": 51.22, "crs": "EPSG:4326"}, + "area_id": "optional-project-area-uuid", + "product_key": "buildings", + "force_refresh": false +} +``` + +The backend requires EPSG:4326, intersects the rectangle with the optional +persisted Area, rejects sides below 10 m or above 20 km, follows only +same-host allowlisted collection pagination and fails instead of truncating +when page, transfer or feature limits are reached. Every official feature id, +request URL, response checksum and artifact checksum remains provenance. +Although GRB is stored natively in EPSG:31370, the service explicitly +negotiates OGC CRS84 longitude/latitude GeoJSON for both bbox and output before +performing the exact geometry intersection. + +Output is an ordinary reference Dataset persisted through `DatasetService`, +`DatasetVersion` and `VectorFeatureService`; providers never write directly +to `vector_features`. Exact requests are reused for 24 hours unless +`force_refresh=true`. Selection metrics are footprint hectares for buildings, +line kilometres for roads, hectares plus supporting line kilometres for +water, and hectares for parcels. Water volume is unsupported because GRB +contains no depth or bathymetry. + ### GET `/api/v1/projects/{project_id}/datasets/dhmv/products` Returns the fixed official DHMV II product registry in the canonical envelope. @@ -898,7 +937,8 @@ Returns configured/status/limitation fields. ### POST `/api/v1/external/providers/{provider_name}/import` -Defines the future provider import contract. Sprint 7B does not perform live imports or write datasets. +Compatibility contract for provider-specific import flows. It never writes +datasets itself. Request: @@ -911,13 +951,13 @@ Request: } ``` -GRB/OSM response: +GRB response: ```json { "provider_name": "grb", - "status": "not_configured", - "message": "No live GRB import is configured in Sprint 7B.", + "status": "bounded_request_required", + "message": "Use the governed project GRB acquisition endpoint with an EPSG:4326 bounding box and one supported layer.", "requested_layers": ["buildings"], "dataset_id": null, "dataset_role": "reference", @@ -925,7 +965,10 @@ GRB/OSM response: } ``` -Manual and fixture providers point callers to existing upload/fixture flows. No provider writes directly to `vector_features`; all future provider output must flow through `DatasetService` and `VectorFeatureService`. +OSM still returns `not_configured`. Manual and fixture providers point callers +to existing upload/fixture flows. No provider writes directly to +`vector_features`; the bounded GRB integration and all future provider output +flow through `DatasetService` and `VectorFeatureService`. ## External data fetchers @@ -953,7 +996,10 @@ Request: } ``` -V1 may initially implement this as a service interface with a clear `not_configured` response until the exact WFS endpoint is wired. +This legacy compatibility endpoint never fetches. It returns +`bounded_request_required` and directs callers to +`POST /api/v1/projects/{project_id}/datasets/grb/acquire`, where bbox, Area, +product allowlist, limits and persistence are enforceable. Sprint 7B provider contract responses expose capabilities only. Providers must report: @@ -973,7 +1019,8 @@ Sprint 7B provider contract responses expose capabilities only. Providers must r } ``` -No GRB WFS, OSM Overpass or provider downloads are implemented in Sprint 7B. +No unbounded GRB WFS download or OSM Overpass integration is implemented. +Bounded GRB OGC API Features acquisition is documented under Datasets. ## Demo workflow @@ -1518,7 +1565,8 @@ GeoJSON feature properties include: Limitations: -- No live GRB/OSM/Sentinel fetching. +- Change detection itself performs no provider fetch. Bounded GRB acquisition + is a separate explicit Dataset operation; OSM and Sentinel remain disabled. - No fake object lifecycle classification. - No `changed` classification without durable object ids/versioning. - No first-class change table yet; the current output is stored in job diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 7ce465ea..9ae53cb9 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -10222,3 +10222,35 @@ Validation: (49 tests). - Frontend TypeScript typecheck and production build passed before the full release gate. + +## Sprint 239 - Governed bounded GRB map acquisition (2026-07-17) + +Implemented: +- Added a server-allowlisted GRB OGC API Features registry for `GBG`, + `Wegsegment`, `WTZ`/`WLAS`/`WGR` and `ADP`, exposed as buildings, roads, + water and parcels. +- Added canonical product and acquisition endpoints. Acquisition validates + EPSG:4326 and metric side lengths, intersects the request with the persisted + Area, follows only trusted pagination, fails instead of truncating and + retains response/artifact checksums. +- Routed all output through the existing synchronous Job and + `DatasetService.import_vector_bytes` flow. The resulting reference Datasets, + DatasetVersions and PostGIS VectorFeatures use the same persistence boundary + as uploaded data. +- Added explicit semantic selection metadata for hectares and kilometres. + Water volume, legal parcel certainty and traffic semantics remain + unsupported. +- Extended the Flanders map product catalog with the four GRB products and + kept all provider traffic behind the backend. + +Validation: +- Focused provider, acquisition, route-envelope and frontend contract tests + cover complete pagination, exact clipping, official identity, hostile next + links, feature limits, persistent metadata and the canonical Job response. +- A read-only live provider probe over one small Mol rectangle returned 8 + `GBG` buildings, 23 `Wegsegment` roads, 20 water features across + `WTZ`/`WLAS`/`WGR` and 96 `ADP` parcels. Every collection completed without + truncation using explicit OGC CRS84 bbox and output negotiation. +- The complete readiness gate passed 943 backend tests, backend compilation, + the 118-route API contract audit, Alembic head `202607160001`, frontend + TypeScript typecheck and the production Vite build. diff --git a/frontend/README.md b/frontend/README.md index 7b9f193d..20d1a1bd 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -607,6 +607,16 @@ browser never contacts WCS directly. An exact repeat reuses the persisted request. A complete-Flanders raster request remains disabled by the regional size guard, while a complete municipality remains a valid bounded analysis. +The same explicit-selection workflow exposes GRB buildings, roads, water and +parcels. These are vector products rather than rasters: selecting a +municipality or drawing a rectangle starts a bounded backend OGC API request, +persists the complete clipped result and analyzes the persisted +`vector_features`. Cards show `Op aanvraag` before first use and measured +hectares or kilometres afterwards. Full-Flanders acquisition remains disabled +by the 20 km side limit; users select a municipality or smaller rectangle. +The map never calls Digitaal Vlaanderen directly and never displays a +silently truncated reference layer. + Selecting `Hoogte & reliëf` shows the source distinction before analysis: DTM represents terrain after removal of buildings and other objects; DSM represents the visible surface including buildings and vegetation. Selecting diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index 2e32368e..2a7ce602 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -2,7 +2,7 @@ import { useEffect, useMemo, useState } from 'react' import GeoMap from '../GeoMap' import type { AreaRead, DatasetCreateResponse, DetectionQaResult, MapResultExportRequest, MapViewportState, OrthophotoAcquisitionResult, OrthophotoProductRead, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionMetric, VectorSelectionResponse } from '../../types' import { useMapThemeSelectionInsights, type MapThemeAcquisition, type MapThemeQuery } from '../../hooks/useMapThemeSelectionInsights' -import { useOfficialRasterProducts } from '../../hooks/useOfficialRasterProducts' +import { useOfficialMapProducts } from '../../hooks/useOfficialMapProducts' import { useTemporalComparison } from '../../hooks/useTemporalComparison' import { getDatasetDisplayName, getDatasetSourceDisplayName } from '../../lib/datasetDisplay' import { TemporalTrendChart } from './TemporalTrendChart' @@ -54,10 +54,9 @@ interface TemporalSeriesGroup { items: DatasetCreateResponse[] } -interface OnDemandRasterProduct extends MapThemeAcquisition { +interface OnDemandMapProduct extends MapThemeAcquisition { theme: DataThemeId - nativeResolutionM: number - referenceLabel: string + availabilityLabel: string attribution: string limitationMessage: string } @@ -669,10 +668,10 @@ export function MapWorkspace({ clearThemeInsights, } = useMapThemeSelectionInsights(selectedProjectId, onRefreshProjectData) const { - products: officialRasterProducts, - loading: officialRasterProductsLoading, - error: officialRasterProductsError, - } = useOfficialRasterProducts(flandersScopeSelected ? selectedProjectId : null) + products: officialMapProducts, + loading: officialMapProductsLoading, + error: officialMapProductsError, + } = useOfficialMapProducts(flandersScopeSelected ? selectedProjectId : null) const { temporalComparison, temporalComparisonLoading, @@ -760,10 +759,10 @@ export function MapWorkspace({ ) if (selectedFloodHazard) { result.flood_hazard = selectedFloodHazard - } else if (flandersScopeSelected && officialRasterProducts.floodHazard.length > 0) { + } else if (flandersScopeSelected && officialMapProducts.floodHazard.length > 0) { result.flood_hazard = null } - if (flandersScopeSelected && officialRasterProducts.dhmv.length > 0) { + if (flandersScopeSelected && officialMapProducts.dhmv.length > 0) { result.elevation = availableMapDatasets.find( (dataset) => dataset.source_name === 'digitaal_vlaanderen_dhmv' @@ -776,8 +775,8 @@ export function MapWorkspace({ availableMapDatasets, flandersScopeSelected, floodHazardDatasets, - officialRasterProducts.dhmv.length, - officialRasterProducts.floodHazard.length, + officialMapProducts.dhmv.length, + officialMapProducts.floodHazard.length, regionalScopeSelected, selectedDhmvProductKey, selectedFloodHazardDatasetId, @@ -800,38 +799,47 @@ export function MapWorkspace({ [availableMapDatasets, regionalScopeSelected, selectedMapAreaId, themeDatasetMap], ) const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0] - const onDemandRasterProductMap = useMemo( + const onDemandProductMap = useMemo( () => { - const result = new Map() + const result = new Map() if (!flandersScopeSelected) { return result } - for (const product of officialRasterProducts.thematic) { + for (const product of officialMapProducts.thematic) { result.set(product.theme, { kind: 'thematic_raster', productKey: product.key, displayName: product.display_name, theme: product.theme, - nativeResolutionM: product.native_resolution_m, - referenceLabel: String(product.observation_year), + availabilityLabel: `${product.native_resolution_m} m · ${product.observation_year} · laad bij selectie`, attribution: product.attribution, limitationMessage: product.limitation_message, }) } - const dhmvProduct = officialRasterProducts.dhmv.find((product) => product.key === selectedDhmvProductKey) + for (const product of officialMapProducts.grb) { + result.set(product.key, { + kind: 'grb', + productKey: product.key, + displayName: product.display_name, + theme: product.key, + availabilityLabel: 'officiële vectorbron · laad bij selectie', + attribution: product.attribution, + limitationMessage: product.limitation_message, + }) + } + const dhmvProduct = officialMapProducts.dhmv.find((product) => product.key === selectedDhmvProductKey) if (dhmvProduct) { result.set('elevation', { kind: 'dhmv', productKey: dhmvProduct.key, displayName: dhmvProduct.display_name, theme: 'elevation', - nativeResolutionM: dhmvProduct.native_resolution_m, - referenceLabel: dhmvProduct.acquisition_period, + availabilityLabel: `${dhmvProduct.native_resolution_m} m · ${dhmvProduct.acquisition_period} · laad bij selectie`, attribution: dhmvProduct.attribution, limitationMessage: dhmvProduct.limitation_message, }) } - const floodProduct = officialRasterProducts.floodHazard.find( + const floodProduct = officialMapProducts.floodHazard.find( (product) => product.key === selectedFloodHazardProductKey, ) if (floodProduct) { @@ -840,8 +848,7 @@ export function MapWorkspace({ productKey: floodProduct.key, displayName: floodProduct.display_name, theme: 'flood_hazard', - nativeResolutionM: floodProduct.native_resolution_m, - referenceLabel: `${floodProduct.climate_context} · T${floodProduct.return_period_years}`, + availabilityLabel: `${floodProduct.native_resolution_m} m · ${floodProduct.climate_context} · T${floodProduct.return_period_years} · laad bij selectie`, attribution: floodProduct.attribution, limitationMessage: floodProduct.limitation_message, }) @@ -850,12 +857,12 @@ export function MapWorkspace({ }, [ flandersScopeSelected, - officialRasterProducts, + officialMapProducts, selectedDhmvProductKey, selectedFloodHazardProductKey, ], ) - const activeOnDemandRasterProduct = onDemandRasterProductMap.get(activeTheme.id) ?? null + const activeOnDemandMapProduct = onDemandProductMap.get(activeTheme.id) ?? null const activeThemeMapStyle = DATA_THEME_MAP_STYLES[activeTheme.id] const analysisOverlayActive = mapContentMode === 'analysis' && analysisLayerAvailable && Boolean(mapFeatureCollection) const selectedOrthophotoProduct = orthophotoProducts.find((item) => item.key === selectedOrthophotoProductKey) ?? null @@ -875,24 +882,24 @@ export function MapWorkspace({ const regionalRasterThemeActive = regionalScopeSelected && isPartitionedRaster(activeThemeDataset) const regionalBathymetryThemeActive = regionalScopeSelected && isPartitionedBathymetry(activeThemeDataset) const regionalPartitionedThemeActive = regionalRasterThemeActive || regionalBathymetryThemeActive - const onDemandRasterThemeActive = analysisMode === 'current' && Boolean(activeOnDemandRasterProduct) - const regionalOnDemandRasterThemeActive = regionalScopeSelected && onDemandRasterThemeActive - const activeThemeAvailable = Boolean(activeThemeDataset) || onDemandRasterThemeActive + const onDemandThemeActive = analysisMode === 'current' && Boolean(activeOnDemandMapProduct) + const regionalOnDemandThemeActive = regionalScopeSelected && onDemandThemeActive + const activeThemeAvailable = Boolean(activeThemeDataset) || onDemandThemeActive useEffect(() => { if ( !flandersScopeSelected || analysisMode !== 'current' || activeThemeAvailable - || (onDemandRasterProductMap.size === 0 && !officialRasterProductsError) + || (onDemandProductMap.size === 0 && !officialMapProductsError) ) { return } const fallbackTheme = DATA_THEMES.find((theme) => theme.id === 'space_occupation' - && Boolean(themeDatasetMap[theme.id] || onDemandRasterProductMap.get(theme.id)), + && Boolean(themeDatasetMap[theme.id] || onDemandProductMap.get(theme.id)), ) ?? DATA_THEMES.find((theme) => - Boolean(themeDatasetMap[theme.id] || onDemandRasterProductMap.get(theme.id)), + Boolean(themeDatasetMap[theme.id] || onDemandProductMap.get(theme.id)), ) if (!fallbackTheme) { return @@ -906,8 +913,8 @@ export function MapWorkspace({ activeThemeAvailable, analysisMode, flandersScopeSelected, - officialRasterProductsError, - onDemandRasterProductMap, + officialMapProductsError, + onDemandProductMap, onOpenDatasetInMap, themeDatasetMap, ]) @@ -999,12 +1006,12 @@ export function MapWorkspace({ useEffect(() => { const contextSourceLabel = analysisMode === 'evolution' ? activeTemporalSeriesGroup?.label ?? null : regionalBathymetryThemeActive ? 'VHA-dwarsprofielen Vlaanderen' - : activeOnDemandRasterProduct?.displayName ?? null + : activeOnDemandMapProduct?.displayName ?? null onSetContextSourceLabel(contextSourceLabel) return () => onSetContextSourceLabel(null) }, [ activeTemporalSeriesGroup?.label, - activeOnDemandRasterProduct?.displayName, + activeOnDemandMapProduct?.displayName, analysisMode, onSetContextSourceLabel, regionalBathymetryThemeActive, @@ -1112,7 +1119,7 @@ export function MapWorkspace({ } return } - if (flandersScopeSelected && officialRasterProducts.floodHazard.length > 0) { + if (flandersScopeSelected && officialMapProducts.floodHazard.length > 0) { setSelectedFloodHazardDatasetId('') return } @@ -1126,7 +1133,7 @@ export function MapWorkspace({ }, [ flandersScopeSelected, floodHazardDatasets, - officialRasterProducts.floodHazard.length, + officialMapProducts.floodHazard.length, selectedFloodHazardDatasetId, selectedFloodHazardProductKey, ]) @@ -1350,7 +1357,7 @@ export function MapWorkspace({ ? temporalGroup?.items[temporalGroup.items.length - 1] ?? null : themeDatasetMap[theme.id] const onDemandProduct = analysisMode === 'current' - ? onDemandRasterProductMap.get(theme.id) + ? onDemandProductMap.get(theme.id) : null if (!dataset && !onDemandProduct) { return @@ -1381,7 +1388,7 @@ export function MapWorkspace({ const availableThemes: Array> = [] for (const theme of DATA_THEMES) { const onDemandProduct = analysisMode === 'current' - ? onDemandRasterProductMap.get(theme.id) + ? onDemandProductMap.get(theme.id) : null if (onDemandProduct) { availableThemes.push({ @@ -1410,7 +1417,7 @@ export function MapWorkspace({ const analyzeSelection = async (bbox: VectorSelectionBBox, areaId?: string) => { setSelectionBbox(bbox) const tasks: Array> = [loadAllThemeResults(bbox, areaId)] - if (!regionalPartitionedThemeActive && !onDemandRasterThemeActive) { + if (!regionalPartitionedThemeActive && !onDemandThemeActive) { tasks.push(onRunMapSelectionExtract(bbox, areaId)) } if (analysisMode === 'evolution' && earlierDatasetId && laterDatasetId) { @@ -1571,7 +1578,7 @@ export function MapWorkspace({
{DATA_THEMES.map((theme) => { const dataset = themeDatasetMap[theme.id] - const onDemandProduct = onDemandRasterProductMap.get(theme.id) + const onDemandProduct = onDemandProductMap.get(theme.id) const partitions = themePartitionMap[theme.id] const temporalGroups = themeTemporalSeriesMap[theme.id] const temporalGroup = temporalGroups[0] @@ -1603,7 +1610,7 @@ export function MapWorkspace({ : dataset ? datasetAvailabilityLabel(dataset, partitions) : onDemandProduct - ? `${onDemandProduct.nativeResolutionM} m · ${onDemandProduct.referenceLabel} · laad bij selectie` + ? onDemandProduct.availabilityLabel : 'Bron nog niet ingeladen'} @@ -1628,7 +1635,7 @@ export function MapWorkspace({ ? 'VHA-dwarsprofielen Vlaanderen' : activeThemeDataset ? getDatasetDisplayName(activeThemeDataset) - : activeOnDemandRasterProduct?.displayName ?? 'Geen databron beschikbaar'} + : activeOnDemandMapProduct?.displayName ?? 'Geen databron beschikbaar'} {analysisOverlayActive @@ -1641,19 +1648,19 @@ export function MapWorkspace({ ? `${activeThemePartitions.length} gecontroleerde gemeentepartities · selectie wordt ruimtelijk samengevoegd` : activeThemeDataset ? `${getDatasetSourceDisplayName(activeThemeDataset)} · ${formatDatasetObservation(activeThemeDataset)}` - : activeOnDemandRasterProduct - ? `${activeOnDemandRasterProduct.attribution} · wordt alleen voor de gekozen selectie ingeladen` + : activeOnDemandMapProduct + ? `${activeOnDemandMapProduct.attribution} · wordt alleen voor de gekozen selectie ingeladen` : activeTheme.description}
- {officialRasterProductsLoading && flandersScopeSelected ? ( -

Beschikbare Vlaamse rasterbronnen worden gecontroleerd…

+ {officialMapProductsLoading && flandersScopeSelected ? ( +

Beschikbare Vlaamse kaartbronnen worden gecontroleerd…

) : null} - {officialRasterProductsError && flandersScopeSelected ? ( -

{officialRasterProductsError}

+ {officialMapProductsError && flandersScopeSelected ? ( +

{officialMapProductsError}

) : null} - {analysisMode === 'current' && activeTheme.id === 'elevation' && officialRasterProducts.dhmv.length > 0 ? ( + {analysisMode === 'current' && activeTheme.id === 'elevation' && officialMapProducts.dhmv.length > 0 ? ( ) : null} - {analysisMode === 'current' && activeTheme.id === 'flood_hazard' && officialRasterProducts.floodHazard.length > 0 ? ( + {analysisMode === 'current' && activeTheme.id === 'flood_hazard' && officialMapProducts.floodHazard.length > 0 ? (