Add governed bathymetry profile workflow
This commit is contained in:
@@ -36,6 +36,13 @@ FLOOD_HAZARD_MAX_SIDE_M=20000
|
||||
FLOOD_HAZARD_MAX_PIXELS=12000000
|
||||
FLOOD_HAZARD_TIMEOUT_SECONDS=300
|
||||
FLOOD_HAZARD_MAX_RESPONSE_MB=160
|
||||
BATHYMETRY_PROFILES_ENABLED=true
|
||||
BATHYMETRY_PROFILES_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0
|
||||
BATHYMETRY_WATERCOURSE_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1
|
||||
BATHYMETRY_PROFILES_PAGE_SIZE=1000
|
||||
BATHYMETRY_PROFILES_MAX_FEATURES=50000
|
||||
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
|
||||
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
|
||||
THEMATIC_RASTER_ENABLED=true
|
||||
THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs
|
||||
THEMATIC_RASTER_MIN_SIDE_M=100
|
||||
|
||||
@@ -7,6 +7,21 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 235 Governed bathymetry profiles and Belgian scale architecture (2026-07-17)
|
||||
|
||||
- Added bounded official VHA cross-section acquisition with exact Area
|
||||
clipping, complete paging, checksums and ordinary Dataset/VectorFeature
|
||||
persistence.
|
||||
- Added nullable structured depth/width metrics, official profile-document
|
||||
evidence and explicit unsupported volume semantics.
|
||||
- Added a bathymetry source registry covering operational VHA profiles and
|
||||
audited MDK North Sea, SPW Walloon and port candidates without pretending
|
||||
those future connectors are configured.
|
||||
- Added the `Waterbodem` Map theme and a concise profile inspector.
|
||||
- Added the Mol operator command and a staged expansion roadmap for Flanders,
|
||||
Belgium, the territorial sea, EEZ and continental shelf with strict
|
||||
TAW/LAT/mDNG separation.
|
||||
|
||||
## Sprint 234 Full audit closure and workspace lifecycle cleanup (2026-07-17)
|
||||
|
||||
- Added reversible active/archived project lifecycle handling and made active
|
||||
|
||||
@@ -1708,3 +1708,23 @@ The command never deletes projects or related datasets, jobs, analyses,
|
||||
quality checks and exports. It always preserves `Kempen Regional Workbench`
|
||||
and `Mol Municipality Workbench`, defaults to dry-run and can print every
|
||||
matched name with `--show-names`.
|
||||
|
||||
## Governed VHA bathymetry profiles
|
||||
|
||||
`POST /api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire`
|
||||
performs a bounded official VHA ArcGIS query, exact persisted-Area clipping,
|
||||
watercourse-name normalization and ordinary Dataset/VectorFeature persistence.
|
||||
`GET /api/v1/projects/{project_id}/datasets/bathymetry/sources` reports VHA as
|
||||
operational and the audited MDK/SPW candidates as unavailable for acquisition.
|
||||
|
||||
Runtime controls are `BATHYMETRY_PROFILES_ENABLED`,
|
||||
`BATHYMETRY_PROFILES_LAYER_URL`, `BATHYMETRY_WATERCOURSE_LAYER_URL`,
|
||||
`BATHYMETRY_PROFILES_PAGE_SIZE`, `BATHYMETRY_PROFILES_MAX_FEATURES`,
|
||||
`BATHYMETRY_PROFILES_TIMEOUT_SECONDS` and
|
||||
`BATHYMETRY_PROFILES_MAX_RESPONSE_MB`. The feature limit intentionally forces
|
||||
large Flemish scopes into exact Area partitions.
|
||||
|
||||
The Dataset exposes profile count and nullable structured depth/width metrics.
|
||||
It does not claim a continuous bed model, current depth or volume. Use
|
||||
`scripts/provision_mol_bathymetry_profiles.py` for the canonical Mol operator
|
||||
flow.
|
||||
|
||||
@@ -27,6 +27,7 @@ from app.schemas import (
|
||||
FloodHazardAcquireRequest,
|
||||
FloodHazardPartitionSelectionRequest,
|
||||
FloodHazardSelectionRequest,
|
||||
BathymetryProfileAcquireRequest,
|
||||
ThematicRasterAcquireRequest,
|
||||
ThematicRasterSelectionRequest,
|
||||
VectorBBoxResponse,
|
||||
@@ -52,6 +53,7 @@ from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||
from app.services.terrain_analysis_service import TerrainAnalysisService
|
||||
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
||||
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
||||
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
|
||||
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
||||
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
||||
from app.utils.response import envelope
|
||||
@@ -213,6 +215,30 @@ def list_flood_hazard_products(project_id: UUID, db: Session = Depends(get_db)):
|
||||
return envelope({"items": items, "total": len(items)})
|
||||
|
||||
|
||||
@router.get("/datasets/bathymetry/sources", response_model=dict)
|
||||
def list_bathymetry_sources(project_id: UUID, db: Session = Depends(get_db)):
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
items = BathymetryProfileAcquisitionService.list_sources()
|
||||
return envelope({"items": items, "total": len(items)})
|
||||
|
||||
|
||||
@router.post("/datasets/bathymetry/profiles/acquire", response_model=dict)
|
||||
def acquire_bounded_bathymetry_profiles(
|
||||
project_id: UUID,
|
||||
payload: BathymetryProfileAcquireRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
job = JobService.run_sync_job(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
job_type="vector.bathymetry_profiles.acquire",
|
||||
parameters=payload.model_dump(mode="json"),
|
||||
operation=lambda: BathymetryProfileAcquisitionService.acquire(db, project_id, payload),
|
||||
)
|
||||
return envelope(job)
|
||||
|
||||
|
||||
@router.post("/datasets/thematic-raster/acquire", response_model=dict)
|
||||
def acquire_bounded_thematic_raster(
|
||||
project_id: UUID,
|
||||
|
||||
@@ -90,6 +90,39 @@ class Settings(BaseSettings):
|
||||
flood_hazard_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="FLOOD_HAZARD_MAX_PIXELS")
|
||||
flood_hazard_timeout_seconds: int = Field(default=300, ge=1, validation_alias="FLOOD_HAZARD_TIMEOUT_SECONDS")
|
||||
flood_hazard_max_response_mb: int = Field(default=160, ge=1, validation_alias="FLOOD_HAZARD_MAX_RESPONSE_MB")
|
||||
bathymetry_profiles_enabled: bool = Field(default=True, validation_alias="BATHYMETRY_PROFILES_ENABLED")
|
||||
bathymetry_profiles_layer_url: str = Field(
|
||||
default="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
|
||||
validation_alias="BATHYMETRY_PROFILES_LAYER_URL",
|
||||
)
|
||||
bathymetry_watercourse_layer_url: str = Field(
|
||||
default="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1",
|
||||
validation_alias="BATHYMETRY_WATERCOURSE_LAYER_URL",
|
||||
)
|
||||
bathymetry_profiles_page_size: int = Field(
|
||||
default=1000,
|
||||
ge=1,
|
||||
le=2000,
|
||||
validation_alias="BATHYMETRY_PROFILES_PAGE_SIZE",
|
||||
)
|
||||
bathymetry_profiles_max_features: int = Field(
|
||||
default=50_000,
|
||||
ge=1,
|
||||
le=250_000,
|
||||
validation_alias="BATHYMETRY_PROFILES_MAX_FEATURES",
|
||||
)
|
||||
bathymetry_profiles_timeout_seconds: int = Field(
|
||||
default=120,
|
||||
ge=1,
|
||||
le=600,
|
||||
validation_alias="BATHYMETRY_PROFILES_TIMEOUT_SECONDS",
|
||||
)
|
||||
bathymetry_profiles_max_response_mb: int = Field(
|
||||
default=32,
|
||||
ge=1,
|
||||
le=256,
|
||||
validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB",
|
||||
)
|
||||
thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED")
|
||||
thematic_raster_wcs_url: str = Field(
|
||||
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
|
||||
|
||||
@@ -65,6 +65,11 @@ from .flood_hazard import (
|
||||
FloodHazardSelectionResponse,
|
||||
FloodHazardSelectionSummary,
|
||||
)
|
||||
from .bathymetry import (
|
||||
BathymetryProfileAcquireRequest,
|
||||
BathymetryProfileAcquisitionResult,
|
||||
BathymetrySourceRead,
|
||||
)
|
||||
from .thematic_raster import (
|
||||
ThematicRasterAcquireRequest,
|
||||
ThematicRasterAcquisitionResult,
|
||||
@@ -202,6 +207,9 @@ __all__ = [
|
||||
"FloodHazardSelectionRequest",
|
||||
"FloodHazardSelectionResponse",
|
||||
"FloodHazardSelectionSummary",
|
||||
"BathymetryProfileAcquireRequest",
|
||||
"BathymetryProfileAcquisitionResult",
|
||||
"BathymetrySourceRead",
|
||||
"ThematicRasterAcquireRequest",
|
||||
"ThematicRasterAcquisitionResult",
|
||||
"ThematicRasterMetric",
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class BathymetryProfileAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class BathymetrySourceRead(BaseModel):
|
||||
key: str
|
||||
display_name: str
|
||||
owner: str
|
||||
authority_level: Literal["authoritative", "contextual"]
|
||||
geographic_coverage: str
|
||||
data_kind: str
|
||||
query_modes: list[str]
|
||||
vertical_reference: str
|
||||
horizontal_crs: str
|
||||
native_resolution: str | None = None
|
||||
integration_status: Literal["operational", "available_not_integrated", "catalog_only"]
|
||||
acquisition_supported: bool
|
||||
configured: bool
|
||||
service_url: str | None = None
|
||||
catalog_url: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class BathymetryProfileAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
profile_count: int = Field(ge=0)
|
||||
document_count: int = Field(ge=0)
|
||||
structured_depth_count: int = Field(ge=0)
|
||||
structured_width_count: int = Field(ge=0)
|
||||
watercourse_count: int = Field(ge=0)
|
||||
bbox_epsg4326: list[float]
|
||||
clipped_to_area_id: UUID | None = None
|
||||
measurement_date_min: str | None = None
|
||||
measurement_date_max: str | None = None
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
@@ -0,0 +1,671 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from typing import Any, Callable
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import Point, box, mapping
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.schemas.bathymetry import (
|
||||
BathymetryProfileAcquireRequest,
|
||||
BathymetryProfileAcquisitionResult,
|
||||
BathymetrySourceRead,
|
||||
)
|
||||
from app.services.dataset_service import DatasetService
|
||||
|
||||
|
||||
class BathymetryProfileAcquisitionService:
|
||||
PROVIDER = "vmm_vha_bathymetry_profiles"
|
||||
SOURCE_VERSION = "VHA digitale atlas ArcGIS MapServer"
|
||||
PROFILE_OUT_FIELDS = (
|
||||
"OBJECTID,vhag,atlaspunt,opg_kruinb,opg_vloerb,d_opmeti,"
|
||||
"hyperlink,bron,kunstwerkid,opg_diepte"
|
||||
)
|
||||
ATTRIBUTION = "Vlaamse Milieumaatschappij (VMM), Vlaamse Hydrografische Atlas"
|
||||
LICENSE_NOTE = "Hergebruik volgens de voorwaarden van de Vlaamse overheid en de bronmetadata."
|
||||
LIMITATION = (
|
||||
"Dwarsprofielen zijn historische puntmetingen met bronafhankelijke meetdatum en verticale referentie. "
|
||||
"Ze vormen geen continue actuele bodemkaart en ondersteunen zonder gelijktijdig waterpeil geen "
|
||||
"gebiedsdekkend of actueel watervolume."
|
||||
)
|
||||
_SOURCES = (
|
||||
{
|
||||
"key": "vha_inland_profiles",
|
||||
"display_name": "VHA dwarsprofielen binnenwater",
|
||||
"owner": "Vlaamse Milieumaatschappij",
|
||||
"authority_level": "authoritative",
|
||||
"geographic_coverage": "Vlaanderen, puntlocaties op gekarteerde waterlopen",
|
||||
"data_kind": "dwarsprofielpunten met meetvelden en brondocumenten",
|
||||
"query_modes": ["bbox", "persisted_area"],
|
||||
"vertical_reference": "document-specific; niet uniform als één peilreferentie te behandelen",
|
||||
"horizontal_crs": "EPSG:4326",
|
||||
"native_resolution": None,
|
||||
"integration_status": "operational",
|
||||
"acquisition_supported": True,
|
||||
"configured": True,
|
||||
"service_url": "https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
|
||||
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/vlaamse-hydrografische-atlas-waterlopen",
|
||||
"attribution": ATTRIBUTION,
|
||||
"license_note": LICENSE_NOTE,
|
||||
"limitation_message": LIMITATION,
|
||||
},
|
||||
{
|
||||
"key": "mdk_bcp_bathymetry",
|
||||
"display_name": "Dieptemodel Belgisch Continentaal Plat",
|
||||
"owner": "Agentschap Maritieme Dienstverlening en Kust",
|
||||
"authority_level": "authoritative",
|
||||
"geographic_coverage": "Belgisch Continentaal Plat en Noordzee",
|
||||
"data_kind": "continu bathymetrisch raster",
|
||||
"query_modes": ["wcs", "wmts", "bounded_raster"],
|
||||
"vertical_reference": "LAT",
|
||||
"horizontal_crs": "bronafhankelijk; expliciet per WCS-respons",
|
||||
"native_resolution": "20 x 20 m",
|
||||
"integration_status": "available_not_integrated",
|
||||
"acquisition_supported": False,
|
||||
"configured": False,
|
||||
"service_url": "https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/WCS_Public",
|
||||
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee",
|
||||
"attribution": "Agentschap Maritieme Dienstverlening en Kust",
|
||||
"license_note": "Zie de officiële datasetmetadata en gebruiksvoorwaarden.",
|
||||
"limitation_message": (
|
||||
"Niet operationeel in GeoIntel totdat WCS, maritieme begrenzing, tegels, LAT-semantiek en "
|
||||
"servercertificaten in een live integratietest zijn gevalideerd."
|
||||
),
|
||||
},
|
||||
{
|
||||
"key": "spw_walloon_waterway_bathymetry",
|
||||
"display_name": "Bathymétrie des voies navigables et lacs-réservoirs",
|
||||
"owner": "Service public de Wallonie",
|
||||
"authority_level": "authoritative",
|
||||
"geographic_coverage": "Waalse bevaarbare waterwegen en stuwmeren met uitgevoerde opmetingen",
|
||||
"data_kind": "bodemhoogteraster en XYZ-puntenwolk",
|
||||
"query_modes": ["download", "arcgis_map_service"],
|
||||
"vertical_reference": "mDNG",
|
||||
"horizontal_crs": "EPSG:3812; visualisatieservice kan EPSG:31370 aanbieden",
|
||||
"native_resolution": "0,5 m",
|
||||
"integration_status": "available_not_integrated",
|
||||
"acquisition_supported": False,
|
||||
"configured": False,
|
||||
"service_url": "https://geoservices.wallonie.be/arcgis/rest/services/EAU/BATHY/MapServer",
|
||||
"catalog_url": "https://geoportail.wallonie.be/catalogue/c450c28f-d357-48af-8423-62d524632cf9.html",
|
||||
"attribution": "Service public de Wallonie",
|
||||
"license_note": "CC BY 4.0 volgens de officiële Geoportail-metadata.",
|
||||
"limitation_message": (
|
||||
"Dekking en meetjaar verschillen per vaarweg of reservoir. Integratie vereist een beheerde "
|
||||
"download- en mosaïekstroom plus expliciete omzetting van mDNG."
|
||||
),
|
||||
},
|
||||
{
|
||||
"key": "port_antwerp_bathymetry",
|
||||
"display_name": "Havenbathymetrie Antwerpen-Brugge",
|
||||
"owner": "Port of Antwerp-Bruges",
|
||||
"authority_level": "contextual",
|
||||
"geographic_coverage": "Gepubliceerde havenzones en meetcampagnes",
|
||||
"data_kind": "periodieke peilingen",
|
||||
"query_modes": ["catalog"],
|
||||
"vertical_reference": "product-specific",
|
||||
"horizontal_crs": "product-specific",
|
||||
"native_resolution": None,
|
||||
"integration_status": "catalog_only",
|
||||
"acquisition_supported": False,
|
||||
"configured": False,
|
||||
"service_url": None,
|
||||
"catalog_url": "https://data.gov.be/nl/datasets",
|
||||
"attribution": "Port of Antwerp-Bruges",
|
||||
"license_note": "Per publicatie te verifiëren.",
|
||||
"limitation_message": (
|
||||
"Alleen als cataloguskandidaat geregistreerd; er is nog geen stabiel, publiek en machineleesbaar "
|
||||
"acquisitiecontract in GeoIntel gevalideerd."
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def list_sources() -> list[dict[str, Any]]:
|
||||
return [BathymetrySourceRead(**item).model_dump() for item in BathymetryProfileAcquisitionService._SOURCES]
|
||||
|
||||
@staticmethod
|
||||
def _validate_bbox(payload: BathymetryProfileAcquireRequest) -> tuple[float, float, float, float]:
|
||||
bbox = payload.bbox
|
||||
if bbox.crs.upper() != "EPSG:4326":
|
||||
raise AppError(
|
||||
code="BATHYMETRY_INVALID_CRS",
|
||||
message="Bathymetry profile acquisition requires EPSG:4326",
|
||||
status_code=400,
|
||||
)
|
||||
values = (bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
|
||||
if not all(math.isfinite(value) for value in values):
|
||||
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box values must be finite", status_code=400)
|
||||
if bbox.min_x >= bbox.max_x or bbox.min_y >= bbox.max_y:
|
||||
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box has no area", status_code=400)
|
||||
if bbox.min_x < -180 or bbox.max_x > 180 or bbox.min_y < -90 or bbox.max_y > 90:
|
||||
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box is outside EPSG:4326", status_code=400)
|
||||
return values
|
||||
|
||||
@staticmethod
|
||||
def _scope_geometry(db, project_id: UUID, area_id: UUID | None, bbox_values: tuple[float, float, float, float]):
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
selection = box(*bbox_values)
|
||||
if area_id is None:
|
||||
return selection
|
||||
area = db.get(Area, area_id)
|
||||
if area is None:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
if area.project_id != project_id:
|
||||
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
|
||||
area_geometry = area.geometry if hasattr(area.geometry, "__geo_interface__") else to_shape(area.geometry)
|
||||
intersection = area_geometry.intersection(selection)
|
||||
if intersection.is_empty:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_SCOPE_EMPTY",
|
||||
message="The requested bounding box does not intersect the selected area",
|
||||
status_code=400,
|
||||
)
|
||||
return intersection
|
||||
|
||||
@staticmethod
|
||||
def _query_url(base_url: str, parameters: dict[str, Any]) -> str:
|
||||
return f"{base_url}?{urlencode(parameters)}"
|
||||
|
||||
@staticmethod
|
||||
def _fetch_json(
|
||||
url: str,
|
||||
settings: Settings,
|
||||
opener: Callable[..., Any] | None,
|
||||
) -> tuple[dict[str, Any], str]:
|
||||
request = Request(
|
||||
url,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "GeoIntel/1.0 bathymetry-profile-acquisition",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with (opener or urlopen)(request, timeout=settings.bathymetry_profiles_timeout_seconds) as response:
|
||||
limit = settings.bathymetry_profiles_max_response_mb * 1024 * 1024
|
||||
content = response.read(limit + 1)
|
||||
except HTTPError as exc:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_HTTP_ERROR",
|
||||
message="VHA profile service returned an HTTP error",
|
||||
details={"status_code": exc.code},
|
||||
status_code=502,
|
||||
) from exc
|
||||
except (TimeoutError, URLError, OSError) as exc:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_UNAVAILABLE",
|
||||
message="VHA profile service is unavailable",
|
||||
status_code=502,
|
||||
) from exc
|
||||
if len(content) > limit:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_RESPONSE_TOO_LARGE",
|
||||
message="VHA profile response exceeded the configured size limit",
|
||||
status_code=502,
|
||||
)
|
||||
try:
|
||||
payload = json.loads(content.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
|
||||
message="VHA profile service returned invalid JSON",
|
||||
status_code=502,
|
||||
) from exc
|
||||
if not isinstance(payload, dict) or payload.get("error"):
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
|
||||
message="VHA profile service returned an ArcGIS error",
|
||||
details={"provider_error": payload.get("error") if isinstance(payload, dict) else None},
|
||||
status_code=502,
|
||||
)
|
||||
return payload, hashlib.sha256(content).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _base_spatial_parameters(bbox_values: tuple[float, float, float, float]) -> dict[str, str]:
|
||||
return {
|
||||
"f": "json",
|
||||
"where": "1=1",
|
||||
"geometry": ",".join(f"{value:.12g}" for value in bbox_values),
|
||||
"geometryType": "esriGeometryEnvelope",
|
||||
"inSR": "4326",
|
||||
"outSR": "4326",
|
||||
"spatialRel": "esriSpatialRelIntersects",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _fetch_profiles(
|
||||
bbox_values: tuple[float, float, float, float],
|
||||
settings: Settings,
|
||||
opener: Callable[..., Any] | None,
|
||||
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
base = settings.bathymetry_profiles_layer_url.rstrip("/") + "/query"
|
||||
count_url = BathymetryProfileAcquisitionService._query_url(
|
||||
base,
|
||||
{
|
||||
**BathymetryProfileAcquisitionService._base_spatial_parameters(bbox_values),
|
||||
"returnCountOnly": "true",
|
||||
"returnGeometry": "false",
|
||||
},
|
||||
)
|
||||
count_payload, count_sha = BathymetryProfileAcquisitionService._fetch_json(count_url, settings, opener)
|
||||
candidate_count = int(count_payload.get("count") or 0)
|
||||
if candidate_count > settings.bathymetry_profiles_max_features:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_SCOPE_TOO_LARGE",
|
||||
message="The requested profile scope exceeds the configured feature limit; acquire smaller area partitions",
|
||||
details={
|
||||
"candidate_count": candidate_count,
|
||||
"max_features": settings.bathymetry_profiles_max_features,
|
||||
},
|
||||
status_code=413,
|
||||
)
|
||||
|
||||
features: list[dict[str, Any]] = []
|
||||
response_hashes: list[str] = []
|
||||
request_urls: list[str] = [count_url]
|
||||
offset = 0
|
||||
while offset < candidate_count:
|
||||
page_url = BathymetryProfileAcquisitionService._query_url(
|
||||
base,
|
||||
{
|
||||
**BathymetryProfileAcquisitionService._base_spatial_parameters(bbox_values),
|
||||
"outFields": BathymetryProfileAcquisitionService.PROFILE_OUT_FIELDS,
|
||||
"returnGeometry": "true",
|
||||
"orderByFields": "OBJECTID",
|
||||
"resultOffset": str(offset),
|
||||
"resultRecordCount": str(settings.bathymetry_profiles_page_size),
|
||||
},
|
||||
)
|
||||
page, page_sha = BathymetryProfileAcquisitionService._fetch_json(page_url, settings, opener)
|
||||
page_features = page.get("features")
|
||||
if not isinstance(page_features, list):
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
|
||||
message="VHA profile response does not contain a feature list",
|
||||
status_code=502,
|
||||
)
|
||||
features.extend(item for item in page_features if isinstance(item, dict))
|
||||
response_hashes.append(page_sha)
|
||||
request_urls.append(page_url)
|
||||
if not page_features:
|
||||
break
|
||||
offset += len(page_features)
|
||||
if len(features) != candidate_count:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_INCOMPLETE_RESPONSE",
|
||||
message="VHA profile pagination did not return the announced number of records",
|
||||
details={"expected": candidate_count, "received": len(features)},
|
||||
status_code=502,
|
||||
)
|
||||
return features, {
|
||||
"candidate_count": candidate_count,
|
||||
"request_urls": request_urls,
|
||||
"response_sha256": [count_sha, *response_hashes],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _fetch_watercourse_names(
|
||||
vhag_codes: set[int],
|
||||
settings: Settings,
|
||||
opener: Callable[..., Any] | None,
|
||||
) -> tuple[dict[int, dict[str, str | None]], list[str], list[str]]:
|
||||
if not vhag_codes:
|
||||
return {}, [], []
|
||||
base = settings.bathymetry_watercourse_layer_url.rstrip("/") + "/query"
|
||||
names: dict[int, dict[str, str | None]] = {}
|
||||
urls: list[str] = []
|
||||
hashes: list[str] = []
|
||||
ordered_codes = sorted(vhag_codes)
|
||||
for start in range(0, len(ordered_codes), 100):
|
||||
chunk = ordered_codes[start : start + 100]
|
||||
offset = 0
|
||||
while True:
|
||||
url = BathymetryProfileAcquisitionService._query_url(
|
||||
base,
|
||||
{
|
||||
"f": "json",
|
||||
"where": f"\"wlasvl.vhag\" IN ({','.join(str(code) for code in chunk)})",
|
||||
"outFields": "wlasvl.vhag,VHAG_TABEL.naam,VHAG_TABEL.namen",
|
||||
"returnGeometry": "false",
|
||||
"orderByFields": "wlasvl.vhag",
|
||||
"resultOffset": str(offset),
|
||||
"resultRecordCount": str(settings.bathymetry_profiles_page_size),
|
||||
},
|
||||
)
|
||||
payload, response_sha = BathymetryProfileAcquisitionService._fetch_json(url, settings, opener)
|
||||
urls.append(url)
|
||||
hashes.append(response_sha)
|
||||
page_features = payload.get("features")
|
||||
if not isinstance(page_features, list):
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
|
||||
message="VHA watercourse response does not contain a feature list",
|
||||
status_code=502,
|
||||
)
|
||||
for feature in page_features:
|
||||
attributes = feature.get("attributes") if isinstance(feature, dict) else None
|
||||
if not isinstance(attributes, dict):
|
||||
continue
|
||||
raw_code = attributes.get("wlasvl.vhag")
|
||||
if raw_code is None:
|
||||
continue
|
||||
code = int(raw_code)
|
||||
if code not in names:
|
||||
names[code] = {
|
||||
"name": attributes.get("VHAG_TABEL.naam"),
|
||||
"alternative_names": attributes.get("VHAG_TABEL.namen"),
|
||||
}
|
||||
if not payload.get("exceededTransferLimit") or not page_features:
|
||||
break
|
||||
offset += len(page_features)
|
||||
return names, urls, hashes
|
||||
|
||||
@staticmethod
|
||||
def _date_from_arcgis(value: Any) -> str | None:
|
||||
if not isinstance(value, (int, float)) or not math.isfinite(float(value)):
|
||||
return None
|
||||
try:
|
||||
return datetime.fromtimestamp(float(value) / 1000.0, tz=UTC).date().isoformat()
|
||||
except (OverflowError, OSError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _numeric_or_none(value: Any) -> float | None:
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
normalized = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return normalized if math.isfinite(normalized) else None
|
||||
|
||||
@staticmethod
|
||||
def _document_url(value: Any) -> str | None:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
normalized = value.strip()
|
||||
if normalized.startswith("http://vha.waterinfo.be/"):
|
||||
normalized = "https://" + normalized[len("http://") :]
|
||||
return normalized if normalized.startswith("https://vha.waterinfo.be/") else None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_features(
|
||||
raw_features: list[dict[str, Any]],
|
||||
scope_geometry,
|
||||
watercourse_names: dict[int, dict[str, str | None]],
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
normalized: list[dict[str, Any]] = []
|
||||
dates: list[str] = []
|
||||
watercourse_codes: set[int] = set()
|
||||
document_count = 0
|
||||
depth_count = 0
|
||||
width_count = 0
|
||||
for raw_feature in raw_features:
|
||||
attributes = raw_feature.get("attributes")
|
||||
geometry = raw_feature.get("geometry")
|
||||
if not isinstance(attributes, dict) or not isinstance(geometry, dict):
|
||||
continue
|
||||
x = BathymetryProfileAcquisitionService._numeric_or_none(geometry.get("x"))
|
||||
y = BathymetryProfileAcquisitionService._numeric_or_none(geometry.get("y"))
|
||||
if x is None or y is None:
|
||||
continue
|
||||
point = Point(x, y)
|
||||
if not scope_geometry.covers(point):
|
||||
continue
|
||||
raw_vhag = attributes.get("vhag")
|
||||
vhag = int(raw_vhag) if isinstance(raw_vhag, (int, float)) else None
|
||||
if vhag is not None:
|
||||
watercourse_codes.add(vhag)
|
||||
names = watercourse_names.get(vhag or -1, {})
|
||||
measurement_date = BathymetryProfileAcquisitionService._date_from_arcgis(attributes.get("d_opmeti"))
|
||||
if measurement_date:
|
||||
dates.append(measurement_date)
|
||||
document_url = BathymetryProfileAcquisitionService._document_url(attributes.get("hyperlink"))
|
||||
depth = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_diepte"))
|
||||
crown_width = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_kruinb"))
|
||||
floor_width = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_vloerb"))
|
||||
if document_url:
|
||||
document_count += 1
|
||||
if depth is not None:
|
||||
depth_count += 1
|
||||
if crown_width is not None or floor_width is not None:
|
||||
width_count += 1
|
||||
object_id = str(attributes.get("OBJECTID"))
|
||||
normalized.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": object_id,
|
||||
"properties": {
|
||||
"source_feature_id": object_id,
|
||||
"provider_record_id": object_id,
|
||||
"watercourse_vhag": vhag,
|
||||
"watercourse_name": names.get("name") or (f"VHA-waterloop {vhag}" if vhag else "Onbekende waterloop"),
|
||||
"watercourse_alternative_names": names.get("alternative_names"),
|
||||
"profile_number": attributes.get("atlaspunt"),
|
||||
"measurement_date": measurement_date,
|
||||
"recorded_depth_m": depth,
|
||||
"recorded_crown_width_m": crown_width,
|
||||
"recorded_floor_width_m": floor_width,
|
||||
"source_document_url": document_url,
|
||||
"document_available": document_url is not None,
|
||||
"structured_depth_available": depth is not None,
|
||||
"source_code": attributes.get("bron"),
|
||||
"structure_id": attributes.get("kunstwerkid"),
|
||||
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
||||
"measurement_semantics": "historical_cross_section_profile_point",
|
||||
"vertical_reference": "document-specific",
|
||||
},
|
||||
"geometry": mapping(point),
|
||||
}
|
||||
)
|
||||
return (
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"name": "vha_bathymetry_profiles",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||
"features": normalized,
|
||||
},
|
||||
{
|
||||
"profile_count": len(normalized),
|
||||
"document_count": document_count,
|
||||
"structured_depth_count": depth_count,
|
||||
"structured_width_count": width_count,
|
||||
"watercourse_count": len(watercourse_codes),
|
||||
"measurement_date_min": min(dates) if dates else None,
|
||||
"measurement_date_max": max(dates) if dates else None,
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
|
||||
return (
|
||||
db.query(Dataset)
|
||||
.filter(
|
||||
Dataset.project_id == project_id,
|
||||
Dataset.name == filename,
|
||||
Dataset.source_name == BathymetryProfileAcquisitionService.PROVIDER,
|
||||
Dataset.status == "ready",
|
||||
)
|
||||
.order_by(Dataset.created_at.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _result(dataset: Dataset, *, reused: bool) -> dict[str, Any]:
|
||||
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||
return BathymetryProfileAcquisitionResult(
|
||||
output_dataset_id=dataset.id,
|
||||
reused=reused,
|
||||
provider=BathymetryProfileAcquisitionService.PROVIDER,
|
||||
profile_count=int(metadata.get("profile_count") or 0),
|
||||
document_count=int(metadata.get("document_count") or 0),
|
||||
structured_depth_count=int(metadata.get("structured_depth_count") or 0),
|
||||
structured_width_count=int(metadata.get("structured_width_count") or 0),
|
||||
watercourse_count=int(metadata.get("watercourse_count") or 0),
|
||||
bbox_epsg4326=list(metadata.get("bbox_epsg4326") or []),
|
||||
clipped_to_area_id=dataset.area_id,
|
||||
measurement_date_min=metadata.get("measurement_date_min"),
|
||||
measurement_date_max=metadata.get("measurement_date_max"),
|
||||
attribution=BathymetryProfileAcquisitionService.ATTRIBUTION,
|
||||
limitation_message=BathymetryProfileAcquisitionService.LIMITATION,
|
||||
).model_dump(mode="json")
|
||||
|
||||
@staticmethod
|
||||
def acquire(
|
||||
db,
|
||||
project_id: UUID,
|
||||
payload: BathymetryProfileAcquireRequest,
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
opener: Callable[..., Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
resolved_settings = settings or get_settings()
|
||||
if not resolved_settings.bathymetry_profiles_enabled:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_NOT_CONFIGURED",
|
||||
message="VHA bathymetry profile acquisition is disabled",
|
||||
status_code=503,
|
||||
)
|
||||
bbox_values = BathymetryProfileAcquisitionService._validate_bbox(payload)
|
||||
scope_geometry = BathymetryProfileAcquisitionService._scope_geometry(
|
||||
db, project_id, payload.area_id, bbox_values
|
||||
)
|
||||
exact_bbox = tuple(float(value) for value in scope_geometry.bounds)
|
||||
request_identity = {
|
||||
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
||||
"bbox_epsg4326": list(exact_bbox),
|
||||
"area_id": str(payload.area_id) if payload.area_id else None,
|
||||
"source_version": BathymetryProfileAcquisitionService.SOURCE_VERSION,
|
||||
}
|
||||
request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode()).hexdigest()
|
||||
filename = f"vha_bathymetry_profiles_{request_hash[:12]}.geojson"
|
||||
if not payload.force_refresh:
|
||||
cached = BathymetryProfileAcquisitionService._cached_dataset(db, project_id, filename)
|
||||
if cached is not None:
|
||||
return BathymetryProfileAcquisitionService._result(cached, reused=True)
|
||||
|
||||
raw_features, fetch_provenance = BathymetryProfileAcquisitionService._fetch_profiles(
|
||||
exact_bbox, resolved_settings, opener
|
||||
)
|
||||
vhag_codes = {
|
||||
int(feature["attributes"]["vhag"])
|
||||
for feature in raw_features
|
||||
if isinstance(feature.get("attributes"), dict)
|
||||
and isinstance(feature["attributes"].get("vhag"), (int, float))
|
||||
}
|
||||
names, name_urls, name_hashes = BathymetryProfileAcquisitionService._fetch_watercourse_names(
|
||||
vhag_codes, resolved_settings, opener
|
||||
)
|
||||
feature_collection, summary = BathymetryProfileAcquisitionService._normalize_features(
|
||||
raw_features, scope_geometry, names
|
||||
)
|
||||
if summary["profile_count"] == 0:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_NO_PROFILES",
|
||||
message="No VHA bathymetry profiles intersect the requested area",
|
||||
status_code=404,
|
||||
)
|
||||
artifact = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
acquired_at = datetime.now(UTC)
|
||||
source_metadata = {
|
||||
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
||||
"service": "ArcGIS MapServer",
|
||||
"source_version": BathymetryProfileAcquisitionService.SOURCE_VERSION,
|
||||
"theme": "bathymetry",
|
||||
"layer_name": "bathymetry_profiles",
|
||||
"coverage_scope": "municipality" if payload.area_id else "bounded_selection",
|
||||
"bbox_epsg4326": list(exact_bbox),
|
||||
**summary,
|
||||
"selection_aggregation": {
|
||||
"metric_key": "profile_count",
|
||||
"method": "feature_count",
|
||||
"label": "Dwarsprofielen",
|
||||
"unit": "profielen",
|
||||
},
|
||||
"selection_metrics": [
|
||||
{
|
||||
"metric_key": "recorded_depth_mean_m",
|
||||
"method": "mean",
|
||||
"property": "recorded_depth_m",
|
||||
"label": "Gemiddelde geregistreerde diepte",
|
||||
"unit": "m",
|
||||
"warning": "Alleen profielen met een gestructureerde dieptewaarde; meetdata kunnen verschillen.",
|
||||
},
|
||||
{
|
||||
"metric_key": "recorded_depth_min_m",
|
||||
"method": "min",
|
||||
"property": "recorded_depth_m",
|
||||
"label": "Kleinste geregistreerde diepte",
|
||||
"unit": "m",
|
||||
},
|
||||
{
|
||||
"metric_key": "recorded_depth_max_m",
|
||||
"method": "max",
|
||||
"property": "recorded_depth_m",
|
||||
"label": "Grootste geregistreerde diepte",
|
||||
"unit": "m",
|
||||
},
|
||||
{
|
||||
"metric_key": "recorded_crown_width_mean_m",
|
||||
"method": "mean",
|
||||
"property": "recorded_crown_width_m",
|
||||
"label": "Gemiddelde geregistreerde kruinbreedte",
|
||||
"unit": "m",
|
||||
},
|
||||
{
|
||||
"metric_key": "recorded_floor_width_mean_m",
|
||||
"method": "mean",
|
||||
"property": "recorded_floor_width_m",
|
||||
"label": "Gemiddelde geregistreerde vloerbreedte",
|
||||
"unit": "m",
|
||||
},
|
||||
],
|
||||
"attribution": BathymetryProfileAcquisitionService.ATTRIBUTION,
|
||||
"license_note": BathymetryProfileAcquisitionService.LICENSE_NOTE,
|
||||
"limitation_message": BathymetryProfileAcquisitionService.LIMITATION,
|
||||
"volume_supported": False,
|
||||
}
|
||||
dataset = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project_id,
|
||||
area_id=payload.area_id,
|
||||
filename=filename,
|
||||
content=artifact,
|
||||
source="VHA digitale atlas dwarsprofielen",
|
||||
source_name=BathymetryProfileAcquisitionService.PROVIDER,
|
||||
dataset_role="reference",
|
||||
reference_layer_name="bathymetry_profiles",
|
||||
source_version=BathymetryProfileAcquisitionService.SOURCE_VERSION,
|
||||
source_metadata=source_metadata,
|
||||
provenance_metadata={
|
||||
"acquisition": "explicit_bounded_arcgis_feature_query",
|
||||
"acquired_at": acquired_at.isoformat(),
|
||||
"request_hash": request_hash,
|
||||
"profile_query_urls": fetch_provenance["request_urls"],
|
||||
"watercourse_name_query_urls": name_urls,
|
||||
"response_sha256": [*fetch_provenance["response_sha256"], *name_hashes],
|
||||
"artifact_sha256": hashlib.sha256(artifact).hexdigest(),
|
||||
"candidate_count": fetch_provenance["candidate_count"],
|
||||
"exact_profile_count": summary["profile_count"],
|
||||
"clipped_to_area_id": str(payload.area_id) if payload.area_id else None,
|
||||
"scope_geometry_type": scope_geometry.geom_type,
|
||||
"vertical_reference": "document-specific",
|
||||
"bathymetric_surface_available": False,
|
||||
"water_surface_level_available": False,
|
||||
"water_volume_available": False,
|
||||
"limitation_message": BathymetryProfileAcquisitionService.LIMITATION,
|
||||
},
|
||||
)
|
||||
persisted = db.get(Dataset, dataset.id)
|
||||
return BathymetryProfileAcquisitionService._result(persisted, reused=False)
|
||||
@@ -410,6 +410,126 @@ class DatasetService:
|
||||
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
@staticmethod
|
||||
def import_vector_bytes(
|
||||
db: Session,
|
||||
*,
|
||||
project_id: UUID,
|
||||
filename: str,
|
||||
content: bytes,
|
||||
source: str,
|
||||
source_name: str,
|
||||
dataset_role: str,
|
||||
reference_layer_name: str | None,
|
||||
source_metadata: dict[str, Any],
|
||||
provenance_metadata: dict[str, Any],
|
||||
area_id: UUID | None = None,
|
||||
temporal_series_key: str | None = None,
|
||||
observed_at: datetime | None = None,
|
||||
valid_from: datetime | None = None,
|
||||
valid_to: datetime | None = None,
|
||||
temporal_granularity: str | None = None,
|
||||
source_version: str | None = None,
|
||||
content_type: str = "application/geo+json",
|
||||
) -> DatasetCreateResponse:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
if area_id is not None:
|
||||
area = db.get(Area, area_id)
|
||||
if not area:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
if area.project_id != project_id:
|
||||
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
|
||||
if not content:
|
||||
raise AppError(code="INVALID_UPLOAD", message="Vector artifact is empty", status_code=400)
|
||||
safe_filename = DatasetService._validate_upload_filename(filename)
|
||||
if DatasetService._extension_for_path(safe_filename) not in DatasetService.VECTOR_EXTENSIONS:
|
||||
raise AppError(code="INVALID_UPLOAD", message="Vector artifacts require .geojson or .json files", status_code=415)
|
||||
normalized_role = DatasetService._normalize_dataset_role(dataset_role)
|
||||
normalized_source_name = (source_name or "").strip() or ("manual" if normalized_role == "reference" else None)
|
||||
temporal = DatasetService._validate_temporal_metadata(
|
||||
temporal_series_key=temporal_series_key,
|
||||
observed_at=observed_at,
|
||||
valid_from=valid_from,
|
||||
valid_to=valid_to,
|
||||
temporal_granularity=temporal_granularity,
|
||||
source_version=source_version,
|
||||
)
|
||||
try:
|
||||
text = content.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise AppError(code="INVALID_UPLOAD", message="Vector artifact must be UTF-8 encoded", status_code=400) from exc
|
||||
try:
|
||||
metadata = parse_geojson_payload(text)
|
||||
vector_payload = json.loads(text)
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc
|
||||
|
||||
dataset_id = uuid.uuid4()
|
||||
storage_info = StorageService.persist_dataset_file(
|
||||
project_id=str(project_id),
|
||||
dataset_id=str(dataset_id),
|
||||
dataset_type="vector",
|
||||
original_filename=safe_filename,
|
||||
content=content,
|
||||
content_type=content_type,
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
name=safe_filename,
|
||||
dataset_type="vector",
|
||||
source=source,
|
||||
dataset_role=normalized_role,
|
||||
source_name=normalized_source_name,
|
||||
reference_layer_name=reference_layer_name if normalized_role == "reference" else None,
|
||||
source_metadata=source_metadata,
|
||||
provenance_metadata=provenance_metadata,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
**temporal,
|
||||
storage_path=storage_info["storage_path"],
|
||||
original_filename=storage_info["original_filename"],
|
||||
stored_filename=storage_info["stored_filename"],
|
||||
content_type=storage_info["content_type"],
|
||||
size_bytes=storage_info["size_bytes"],
|
||||
checksum_sha256=storage_info["checksum_sha256"],
|
||||
crs=metadata.get("crs"),
|
||||
bounds_json=metadata.get("bounds_json"),
|
||||
metadata_json=metadata,
|
||||
status="ready",
|
||||
)
|
||||
try:
|
||||
db.add(dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
source_version=dataset.source_version,
|
||||
observed_at=dataset.observed_at,
|
||||
valid_from=dataset.valid_from,
|
||||
valid_to=dataset.valid_to,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
db=db,
|
||||
dataset_id=dataset.id,
|
||||
payload=vector_payload,
|
||||
feature_class=reference_layer_name if normalized_role == "reference" else None,
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
StorageService.remove_dataset_file(storage_info["storage_path"])
|
||||
raise
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
@staticmethod
|
||||
def import_raster_bytes(
|
||||
db: Session,
|
||||
|
||||
@@ -36,6 +36,8 @@ SEMANTIC_METRICS_DISABLED_OPERATOR_TOOLS = {
|
||||
# line metrics (road/watercourse length) would therefore be meaningless.
|
||||
"provision_regional_historical_landuse.py",
|
||||
}
|
||||
PROPERTY_AGGREGATION_METHODS = {"sum", "mean", "area_weighted_sum"}
|
||||
PROPERTY_EXTREMA_METHODS = {"min", "max"}
|
||||
|
||||
PRECLIPPED_MUNICIPALITY_PARTITION_OPERATOR_TOOLS = {
|
||||
"provision_regional_bwk_natura2000.py",
|
||||
@@ -581,7 +583,7 @@ class VectorFeatureService:
|
||||
length_m = db.query(func.coalesce(func.sum(length_expression), 0.0)).filter(*metric_filter).scalar()
|
||||
divisor = 1_000.0 if unit == "km" else 1.0
|
||||
metric_value = float(length_m or 0.0) / divisor
|
||||
elif method in {"sum", "mean", "area_weighted_sum"}:
|
||||
elif method in PROPERTY_AGGREGATION_METHODS | PROPERTY_EXTREMA_METHODS:
|
||||
property_name = str(config.get("property") or "").strip()
|
||||
if not property_name:
|
||||
raise AppError(
|
||||
@@ -599,7 +601,11 @@ class VectorFeatureService:
|
||||
)
|
||||
coverage_ratio = intersection_area / func.nullif(source_area, 0.0)
|
||||
value_expression = numeric_value * coverage_ratio
|
||||
aggregate_function = func.avg if method == "mean" else func.sum
|
||||
aggregate_function = {
|
||||
"mean": func.avg,
|
||||
"min": func.min,
|
||||
"max": func.max,
|
||||
}.get(method, func.sum)
|
||||
aggregate_value = (
|
||||
db.query(func.coalesce(aggregate_function(value_expression), 0.0))
|
||||
.filter(*metric_filter)
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from geoalchemy2.shape import from_shape
|
||||
from shapely.geometry import MultiPolygon, Polygon
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, Job, Project
|
||||
from app.schemas.bathymetry import BathymetryProfileAcquireRequest
|
||||
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
|
||||
from app.services.dataset_service import DatasetService
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, result=None):
|
||||
self.result = result
|
||||
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
|
||||
def order_by(self, *_args):
|
||||
return self
|
||||
|
||||
def first(self):
|
||||
return self.result
|
||||
|
||||
def all(self):
|
||||
return self.result if isinstance(self.result, list) else []
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, rows=None, query_result=None):
|
||||
self.rows = rows or {}
|
||||
self.query_result = query_result
|
||||
self.added = []
|
||||
|
||||
def get(self, model, row_id):
|
||||
row = self.rows.get((model, row_id))
|
||||
if row is not None:
|
||||
return row
|
||||
return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
|
||||
|
||||
def add(self, row):
|
||||
self.added.append(row)
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
def rollback(self):
|
||||
return None
|
||||
|
||||
def refresh(self, row):
|
||||
return row
|
||||
|
||||
def query(self, _model):
|
||||
return FakeQuery(self.query_result)
|
||||
|
||||
|
||||
class JsonResponse:
|
||||
def __init__(self, payload):
|
||||
self.content = json.dumps(payload).encode("utf-8")
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self, size=-1):
|
||||
return self.content if size < 0 else self.content[:size]
|
||||
|
||||
|
||||
def request(*, area_id=None, force_refresh=True) -> BathymetryProfileAcquireRequest:
|
||||
return BathymetryProfileAcquireRequest(
|
||||
bbox={
|
||||
"min_x": 5.0,
|
||||
"min_y": 51.0,
|
||||
"max_x": 6.0,
|
||||
"max_y": 52.0,
|
||||
"crs": "EPSG:4326",
|
||||
},
|
||||
area_id=area_id,
|
||||
force_refresh=force_refresh,
|
||||
)
|
||||
|
||||
|
||||
def profile(object_id, vhag, x, y, *, depth=None, document=None, measured_at=951868800000):
|
||||
return {
|
||||
"attributes": {
|
||||
"OBJECTID": object_id,
|
||||
"vhag": vhag,
|
||||
"atlaspunt": str(object_id),
|
||||
"opg_kruinb": 4.5 if depth is not None else None,
|
||||
"opg_vloerb": 1.2 if depth is not None else None,
|
||||
"d_opmeti": measured_at,
|
||||
"hyperlink": document,
|
||||
"bron": 4,
|
||||
"kunstwerkid": f"structure-{object_id}",
|
||||
"opg_diepte": depth,
|
||||
},
|
||||
"geometry": {"x": x, "y": y},
|
||||
}
|
||||
|
||||
|
||||
def provider_opener(*, count=3):
|
||||
profiles = [
|
||||
profile(
|
||||
1,
|
||||
8506,
|
||||
5.2,
|
||||
51.2,
|
||||
depth=1.8,
|
||||
document="http://vha.waterinfo.be/download/dwarsprofielen/Molse_Nete/8506_DP_1.pdf",
|
||||
),
|
||||
profile(2, 8634, 5.8, 51.8, depth=2.4),
|
||||
profile(3, 8506, 5.2, 51.8),
|
||||
]
|
||||
|
||||
def opener(raw_request, timeout):
|
||||
assert timeout == 120
|
||||
url = raw_request.full_url
|
||||
query = parse_qs(urlparse(url).query)
|
||||
if query.get("returnCountOnly") == ["true"]:
|
||||
return JsonResponse({"count": count})
|
||||
if "MapServer/1/query" in url:
|
||||
return JsonResponse(
|
||||
{
|
||||
"features": [
|
||||
{
|
||||
"attributes": {
|
||||
"wlasvl.vhag": 8506,
|
||||
"VHAG_TABEL.naam": "Molse Nete",
|
||||
"VHAG_TABEL.namen": "Molse Nete - Mol Neet",
|
||||
}
|
||||
},
|
||||
{
|
||||
"attributes": {
|
||||
"wlasvl.vhag": 8634,
|
||||
"VHAG_TABEL.naam": "Scheppelijke Nete",
|
||||
"VHAG_TABEL.namen": "Scheppelijke Nete - Stevensloop",
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
return JsonResponse({"features": profiles[:count]})
|
||||
|
||||
return opener
|
||||
|
||||
|
||||
def test_bathymetry_source_registry_is_honest_and_nationally_extensible() -> None:
|
||||
sources = BathymetryProfileAcquisitionService.list_sources()
|
||||
by_key = {item["key"]: item for item in sources}
|
||||
|
||||
assert set(by_key) == {
|
||||
"vha_inland_profiles",
|
||||
"mdk_bcp_bathymetry",
|
||||
"spw_walloon_waterway_bathymetry",
|
||||
"port_antwerp_bathymetry",
|
||||
}
|
||||
assert by_key["vha_inland_profiles"]["integration_status"] == "operational"
|
||||
assert by_key["vha_inland_profiles"]["acquisition_supported"] is True
|
||||
assert by_key["mdk_bcp_bathymetry"]["vertical_reference"] == "LAT"
|
||||
assert by_key["mdk_bcp_bathymetry"]["acquisition_supported"] is False
|
||||
assert by_key["spw_walloon_waterway_bathymetry"]["vertical_reference"] == "mDNG"
|
||||
assert by_key["spw_walloon_waterway_bathymetry"]["license_note"].startswith("CC BY 4.0")
|
||||
|
||||
|
||||
def test_bathymetry_normalization_exactly_clips_area_and_preserves_evidence() -> None:
|
||||
l_shape = Polygon(
|
||||
[
|
||||
(5.0, 51.0),
|
||||
(6.0, 51.0),
|
||||
(6.0, 51.4),
|
||||
(5.4, 51.4),
|
||||
(5.4, 52.0),
|
||||
(5.0, 52.0),
|
||||
(5.0, 51.0),
|
||||
]
|
||||
)
|
||||
raw, _provenance = BathymetryProfileAcquisitionService._fetch_profiles(
|
||||
(5.0, 51.0, 6.0, 52.0),
|
||||
Settings(_env_file=None),
|
||||
provider_opener(),
|
||||
)
|
||||
names, _urls, _hashes = BathymetryProfileAcquisitionService._fetch_watercourse_names(
|
||||
{8506, 8634},
|
||||
Settings(_env_file=None),
|
||||
provider_opener(),
|
||||
)
|
||||
collection, summary = BathymetryProfileAcquisitionService._normalize_features(raw, l_shape, names)
|
||||
|
||||
assert summary == {
|
||||
"profile_count": 2,
|
||||
"document_count": 1,
|
||||
"structured_depth_count": 1,
|
||||
"structured_width_count": 1,
|
||||
"watercourse_count": 1,
|
||||
"measurement_date_min": "2000-03-01",
|
||||
"measurement_date_max": "2000-03-01",
|
||||
}
|
||||
assert {feature["id"] for feature in collection["features"]} == {"1", "3"}
|
||||
first = collection["features"][0]["properties"]
|
||||
assert first["watercourse_name"] == "Molse Nete"
|
||||
assert first["recorded_depth_m"] == 1.8
|
||||
assert first["source_document_url"].startswith("https://vha.waterinfo.be/")
|
||||
assert first["vertical_reference"] == "document-specific"
|
||||
|
||||
|
||||
def test_bathymetry_acquisition_persists_reference_dataset_through_dataset_service(monkeypatch) -> None:
|
||||
project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4()
|
||||
l_shape = MultiPolygon(
|
||||
[
|
||||
Polygon(
|
||||
[
|
||||
(5.0, 51.0),
|
||||
(6.0, 51.0),
|
||||
(6.0, 51.4),
|
||||
(5.4, 51.4),
|
||||
(5.4, 52.0),
|
||||
(5.0, 52.0),
|
||||
(5.0, 51.0),
|
||||
]
|
||||
)
|
||||
]
|
||||
)
|
||||
project = Project(id=project_id, name="Mol")
|
||||
area = Area(
|
||||
id=area_id,
|
||||
project_id=project_id,
|
||||
name="Gemeente Mol - officieel",
|
||||
geometry=from_shape(l_shape, srid=4326),
|
||||
)
|
||||
db = FakeSession({(Project, project_id): project, (Area, area_id): area})
|
||||
captured = {}
|
||||
|
||||
def persist(_db, **kwargs):
|
||||
captured.update(kwargs)
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
name=kwargs["filename"],
|
||||
dataset_type="vector",
|
||||
source=kwargs["source"],
|
||||
dataset_role=kwargs["dataset_role"],
|
||||
source_name=kwargs["source_name"],
|
||||
reference_layer_name=kwargs["reference_layer_name"],
|
||||
source_metadata=kwargs["source_metadata"],
|
||||
provenance_metadata=kwargs["provenance_metadata"],
|
||||
status="ready",
|
||||
)
|
||||
db.rows[(Dataset, dataset_id)] = dataset
|
||||
return SimpleNamespace(id=dataset_id)
|
||||
|
||||
monkeypatch.setattr(DatasetService, "import_vector_bytes", persist)
|
||||
result = BathymetryProfileAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
request(area_id=area_id),
|
||||
settings=Settings(_env_file=None),
|
||||
opener=provider_opener(),
|
||||
)
|
||||
|
||||
assert result["output_dataset_id"] == str(dataset_id)
|
||||
assert result["profile_count"] == 2
|
||||
assert captured["dataset_role"] == "reference"
|
||||
assert captured["source_name"] == BathymetryProfileAcquisitionService.PROVIDER
|
||||
assert captured["reference_layer_name"] == "bathymetry_profiles"
|
||||
assert captured["source_metadata"]["theme"] == "bathymetry"
|
||||
assert captured["source_metadata"]["volume_supported"] is False
|
||||
assert captured["provenance_metadata"]["water_volume_available"] is False
|
||||
payload = json.loads(captured["content"])
|
||||
assert len(payload["features"]) == 2
|
||||
|
||||
|
||||
def test_bathymetry_acquisition_rejects_unbounded_feature_volume() -> None:
|
||||
settings = Settings(_env_file=None, BATHYMETRY_PROFILES_MAX_FEATURES=2)
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
BathymetryProfileAcquisitionService._fetch_profiles(
|
||||
(5.0, 51.0, 6.0, 52.0),
|
||||
settings,
|
||||
provider_opener(count=3),
|
||||
)
|
||||
assert exc_info.value.code == "BATHYMETRY_SCOPE_TOO_LARGE"
|
||||
assert exc_info.value.details["candidate_count"] == 3
|
||||
|
||||
|
||||
def test_bathymetry_source_api_uses_canonical_envelope() -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
try:
|
||||
response = TestClient(app).get(f"/api/v1/projects/{project_id}/datasets/bathymetry/sources")
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert response.status_code == 200
|
||||
assert set(response.json()) == {"data"}
|
||||
assert response.json()["data"]["total"] == 4
|
||||
assert response.json()["data"]["items"][0]["integration_status"] == "operational"
|
||||
|
||||
|
||||
def test_bathymetry_acquisition_route_stays_inside_existing_job_envelope(monkeypatch) -> None:
|
||||
project_id, dataset_id = uuid4(), uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
|
||||
monkeypatch.setattr(
|
||||
BathymetryProfileAcquisitionService,
|
||||
"acquire",
|
||||
lambda *_args, **_kwargs: {
|
||||
"output_dataset_id": str(dataset_id),
|
||||
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
||||
"profile_count": 2,
|
||||
},
|
||||
)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
try:
|
||||
response = TestClient(app).post(
|
||||
f"/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire",
|
||||
json=request().model_dump(mode="json"),
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert response.status_code == 200
|
||||
assert set(response.json()) == {"data"}
|
||||
assert response.json()["data"]["job_type"] == "vector.bathymetry_profiles.acquire"
|
||||
assert response.json()["data"]["output_dataset_id"] == str(dataset_id)
|
||||
assert any(isinstance(item, Job) for item in db.added)
|
||||
|
||||
|
||||
def test_bathymetry_contract_and_expansion_roadmap_are_documented() -> None:
|
||||
api_contracts = (ROOT / "docs" / "API_CONTRACTS.md").read_text(encoding="utf-8")
|
||||
roadmap = ROOT / "docs" / "BATHYMETRY_EXPANSION_ROADMAP.md"
|
||||
assert "bathymetry/profiles/acquire" in api_contracts
|
||||
assert roadmap.exists()
|
||||
contents = roadmap.read_text(encoding="utf-8")
|
||||
assert "LAT" in contents and "mDNG" in contents and "territoriale zee" in contents
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
assert "py_compile scripts/provision_mol_bathymetry_profiles.py" in readiness
|
||||
assert "COPY scripts/provision_mol_bathymetry_profiles.py" in dockerfile
|
||||
@@ -88,6 +88,7 @@ COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_hist
|
||||
COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_regional_historical_landuse.py
|
||||
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
|
||||
COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py
|
||||
COPY scripts/provision_mol_bathymetry_profiles.py /app/scripts/provision_mol_bathymetry_profiles.py
|
||||
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
|
||||
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
|
||||
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
|
||||
|
||||
@@ -54,6 +54,13 @@ services:
|
||||
FLOOD_HAZARD_MAX_PIXELS: ${FLOOD_HAZARD_MAX_PIXELS:-12000000}
|
||||
FLOOD_HAZARD_TIMEOUT_SECONDS: ${FLOOD_HAZARD_TIMEOUT_SECONDS:-300}
|
||||
FLOOD_HAZARD_MAX_RESPONSE_MB: ${FLOOD_HAZARD_MAX_RESPONSE_MB:-160}
|
||||
BATHYMETRY_PROFILES_ENABLED: ${BATHYMETRY_PROFILES_ENABLED:-true}
|
||||
BATHYMETRY_PROFILES_LAYER_URL: ${BATHYMETRY_PROFILES_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0}
|
||||
BATHYMETRY_WATERCOURSE_LAYER_URL: ${BATHYMETRY_WATERCOURSE_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1}
|
||||
BATHYMETRY_PROFILES_PAGE_SIZE: ${BATHYMETRY_PROFILES_PAGE_SIZE:-1000}
|
||||
BATHYMETRY_PROFILES_MAX_FEATURES: ${BATHYMETRY_PROFILES_MAX_FEATURES:-50000}
|
||||
BATHYMETRY_PROFILES_TIMEOUT_SECONDS: ${BATHYMETRY_PROFILES_TIMEOUT_SECONDS:-120}
|
||||
BATHYMETRY_PROFILES_MAX_RESPONSE_MB: ${BATHYMETRY_PROFILES_MAX_RESPONSE_MB:-32}
|
||||
THEMATIC_RASTER_ENABLED: ${THEMATIC_RASTER_ENABLED:-true}
|
||||
THEMATIC_RASTER_WCS_URL: ${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}
|
||||
THEMATIC_RASTER_MIN_SIDE_M: ${THEMATIC_RASTER_MIN_SIDE_M:-100}
|
||||
|
||||
@@ -2055,3 +2055,37 @@ specifically, no water volume is inferred from 2D water geometry. The backend
|
||||
sets an explicit Ollama context window and returns
|
||||
`OLLAMA_RESPONSE_TRUNCATED` instead of accepting a response with
|
||||
`done_reason=length` as a complete answer.
|
||||
|
||||
## Bathymetry and VHA profile acquisition
|
||||
|
||||
### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources`
|
||||
|
||||
Returns the governed bathymetry source registry in the canonical envelope.
|
||||
VHA inland profiles are `operational`. MDK Belgian Continental Shelf and SPW
|
||||
Walloon bathymetry remain `available_not_integrated`; they cannot be acquired
|
||||
through this contract until their raster/download and vertical-datum flows
|
||||
pass live validation.
|
||||
|
||||
### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire`
|
||||
|
||||
```json
|
||||
{
|
||||
"bbox": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.3, "max_y": 51.3, "crs": "EPSG:4326"},
|
||||
"area_id": "optional persisted Area uuid",
|
||||
"force_refresh": false
|
||||
}
|
||||
```
|
||||
|
||||
The endpoint creates a synchronous `vector.bathymetry_profiles.acquire` Job.
|
||||
It queries the official VHA Digital Atlas in bounded pages, intersects every
|
||||
point with the persisted Area when supplied and persists one ordinary
|
||||
reference Dataset plus VectorFeature rows. The result reports exact profile,
|
||||
document, structured-depth/width and watercourse counts and the source
|
||||
measurement-date range.
|
||||
|
||||
The acquired Dataset uses
|
||||
`source_name=vmm_vha_bathymetry_profiles`,
|
||||
`dataset_role=reference` and
|
||||
`reference_layer_name=bathymetry_profiles`. Existing vector content and
|
||||
selection endpoints provide GeoJSON and metrics. The contract does not expose
|
||||
a continuous bathymetric surface, current water depth or water volume.
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
# Bathymetry expansion roadmap
|
||||
|
||||
## Purpose
|
||||
|
||||
GeoIntel must distinguish three different questions:
|
||||
|
||||
1. Where were cross-sections measured and what does the source document say?
|
||||
2. What is the continuous elevation of the bed at a specific survey epoch?
|
||||
3. What is the water depth or volume at a specific moment?
|
||||
|
||||
Only the first question is operational for Mol through the VHA cross-section
|
||||
profile layer. A bed model does not provide water depth without a compatible
|
||||
water-surface elevation. Flood-hazard maximum depth is a scenario result and
|
||||
must not be reused as current water level.
|
||||
|
||||
## Governed source matrix
|
||||
|
||||
| Source | Coverage | Data | Vertical reference | GeoIntel status |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| VMM VHA Digital Atlas | Flanders | Point locations, structured profile fields, PDF evidence | Document-specific | Operational, bounded vector acquisition |
|
||||
| MDK Belgian Continental Shelf model | Belgian North Sea | Continuous 20 x 20 m bathymetric raster | LAT | Available, not integrated |
|
||||
| SPW navigable waterways and reservoir lakes | Wallonia | 0.5 m bed-elevation raster and XYZ cloud | mDNG | Available, not integrated |
|
||||
| Port of Antwerp-Bruges publications | Port survey areas | Periodic soundings | Product-specific | Catalog candidate |
|
||||
|
||||
VHA contains approximately 129,643 profile points across Flanders at the
|
||||
observed catalog state. This is a scale indication, not a fixed contractual
|
||||
count. Mol contained 828 exact in-boundary points during source validation,
|
||||
715 with document links and 112 with a structured depth field. Production
|
||||
provisioning always records the live counts and checksums.
|
||||
|
||||
## Operational tier 1: Mol
|
||||
|
||||
- Query only an explicit EPSG:4326 bbox.
|
||||
- Intersect the bbox with the exact persisted Mol Area.
|
||||
- Page the official ArcGIS FeatureServer response without truncation.
|
||||
- Resolve VHA watercourse names from the official atlas layer.
|
||||
- Normalize profile points to EPSG:4326.
|
||||
- Persist the artifact through `DatasetService.import_vector_bytes`.
|
||||
- Persist every point through `VectorFeatureService`; the provider never
|
||||
writes directly to `vector_features`.
|
||||
- Expose document, measurement date, depth and width fields without parsing or
|
||||
inventing values from scanned PDFs.
|
||||
- Keep volume unsupported.
|
||||
|
||||
## Tier 2: all of Flanders
|
||||
|
||||
Flanders must be provisioned as exact municipality or other approved Area
|
||||
partitions, not as one monolithic request. The backend feature limit protects
|
||||
the provider and the application. A regional logical layer may group complete
|
||||
partitions, but each Dataset retains its Area id, query URLs, checksums, exact
|
||||
count and measurement-date range.
|
||||
|
||||
Before regional activation:
|
||||
|
||||
- add a governed Flanders boundary manifest and partition coordinator;
|
||||
- prove idempotent resume and no duplicate VHA `OBJECTID` within a partition;
|
||||
- benchmark PostGIS point selection and viewport delivery;
|
||||
- add freshness/version probing for the VHA MapServer;
|
||||
- keep individual profile dates instead of fabricating one Dataset
|
||||
`observed_at`.
|
||||
|
||||
## Tier 3: Belgium
|
||||
|
||||
Belgian coverage is a federation of source adapters with one normalized
|
||||
contract, not one assumed national dataset:
|
||||
|
||||
- Flanders: VHA profiles and future validated bed rasters;
|
||||
- Wallonia: SPW bathymetry for measured navigable waterways/reservoirs;
|
||||
- Brussels: hydrological context until an authoritative public bathymetric
|
||||
product is identified;
|
||||
- federal/maritime: MDK and legally appropriate maritime boundaries.
|
||||
|
||||
Every adapter must emit:
|
||||
|
||||
- authority and owner;
|
||||
- exact geographic and temporal coverage;
|
||||
- horizontal and vertical CRS/datum;
|
||||
- survey/acquisition time;
|
||||
- resolution or sample density;
|
||||
- source URL, request identity, checksum, attribution and license;
|
||||
- explicit supported and unsupported metrics.
|
||||
|
||||
LAT, TAW and mDNG values must never be merged or compared without a documented,
|
||||
tested vertical transformation and uncertainty statement.
|
||||
|
||||
## Tier 4: the Belgian North Sea
|
||||
|
||||
The map must distinguish:
|
||||
|
||||
- the Belgian land boundary and baseline;
|
||||
- the territoriale zee (up to 12 nautical miles);
|
||||
- the Belgian EEZ and continental shelf, which are jurisdictional maritime
|
||||
zones and should not be labelled ordinary municipal or provincial
|
||||
"grondgebied".
|
||||
|
||||
The MDK bathymetry WCS is the preferred continuous source candidate. Activation
|
||||
requires a live Docker validation of TLS/certificates, GetCapabilities,
|
||||
coverage identifiers, bounded GeoTIFF retrieval, CRS, LAT, nodata, pixel size,
|
||||
response limits and maritime clipping. WMTS can support visual context but is
|
||||
not the analytical source.
|
||||
|
||||
## Depth and volume rules
|
||||
|
||||
For a compatible bed raster and water-surface raster at the same time and
|
||||
vertical datum:
|
||||
|
||||
`volume_m3 = sum(max(0, water_surface_z - bed_z) * cell_area_m2)`
|
||||
|
||||
For surveyed cross-sections along a connected reach:
|
||||
|
||||
`volume_m3 = sum(((section_area_i + section_area_i+1) / 2) * reach_length_i)`
|
||||
|
||||
The second method requires complete profile geometry, ordered chainage,
|
||||
contemporaneous water level and defensible interpolation. VHA profile points
|
||||
alone do not satisfy those prerequisites.
|
||||
|
||||
Historical evolution compares only survey epochs with documented compatible
|
||||
coverage, datum and method. A changed raster footprint is not automatically
|
||||
bed evolution.
|
||||
|
||||
## Implementation order
|
||||
|
||||
1. Operate and validate the Mol VHA profile Dataset and map flow.
|
||||
2. Add VHA municipal partition orchestration for Flanders.
|
||||
3. Implement a bounded MDK WCS probe, then acquisition behind live evidence.
|
||||
4. Implement SPW download staging and vertical-datum metadata validation.
|
||||
5. Add maritime boundaries as separate authoritative scope layers.
|
||||
6. Add cross-source vertical-datum transformation only with authoritative
|
||||
grids/parameters and uncertainty tests.
|
||||
7. Add volume only after a compatible measured or modeled water-surface source
|
||||
is part of the same analysis contract.
|
||||
@@ -10010,3 +10010,27 @@ Validation:
|
||||
stays within its 375 px client width; the theme inventory is bounded to a
|
||||
compact scrollable selector so it no longer pushes the map behind all 15
|
||||
theme cards.
|
||||
|
||||
## Sprint 235 - Governed bathymetry profiles and Belgian expansion model (2026-07-17)
|
||||
|
||||
Implemented:
|
||||
- Audited official VHA, MDK Belgian Continental Shelf and SPW Walloon
|
||||
bathymetry services and separated profile evidence, continuous bed models
|
||||
and time-specific water depth/volume.
|
||||
- Added bounded, paged VHA profile acquisition with exact persisted-Area
|
||||
clipping, official watercourse names, document links, checksums and standard
|
||||
Dataset/VectorFeature persistence.
|
||||
- Added reusable vector-byte import and governed `min`/`max` property metrics
|
||||
to the existing selection aggregation path.
|
||||
- Added the bathymetry source and acquisition API, environment controls, Mol
|
||||
operator command, map theme, profile inspector and source inventory.
|
||||
- Documented partitioned Flanders scaling and federated Belgium/maritime
|
||||
scaling with explicit territorial sea, EEZ/continental shelf and
|
||||
TAW/LAT/mDNG semantics.
|
||||
|
||||
Validation:
|
||||
- Backend compilation, 914 backend tests, documentation/contract audits,
|
||||
Alembic head `202607160001`, frontend TypeScript typecheck and the production
|
||||
Vite build passed in the complete readiness gate.
|
||||
- Full readiness and live Mol acceptance are recorded after final validation
|
||||
and deployment below.
|
||||
|
||||
@@ -360,3 +360,18 @@ the composite expression index
|
||||
It supports exact preclipped municipality selection without changing the
|
||||
canonical `vector_features` schema or introducing operator-specific tables.
|
||||
Free rectangle queries continue to use the geometry GiST index.
|
||||
|
||||
## Bathymetry profile persistence
|
||||
|
||||
VHA profile points require no new table or migration. The immutable GeoJSON
|
||||
artifact is stored as one normal reference `datasets` row and
|
||||
`dataset_versions` row; each exact point is a normal `vector_features` row with
|
||||
EPSG:4326 geometry and source properties. Existing dataset, source-feature and
|
||||
GiST indexes support selection.
|
||||
|
||||
Profile measurement dates remain feature properties because a bounded Dataset
|
||||
can contain many historical campaigns. No artificial Dataset `observed_at` or
|
||||
temporal series is assigned. A future bed raster remains file/object storage
|
||||
plus Dataset metadata, not raster-in-database storage. Any national/maritime
|
||||
extension keeps source vertical datum, survey epoch and Area partition
|
||||
explicit and does not create a provider-specific shadow schema.
|
||||
|
||||
@@ -709,3 +709,26 @@ thematic rasters for space occupation, open space, population density, node
|
||||
value and service level. The digital soil map should follow the existing
|
||||
canonical vector persistence path. Watercourse/runoff additions remain in the
|
||||
roadmap but no longer precede these cross-domain gaps.
|
||||
|
||||
## Bathymetry, inland profiles and maritime scope
|
||||
|
||||
The official VHA Digital Atlas profile-point layer is the first operational
|
||||
bathymetry-adjacent source. GeoIntel requests an explicit bbox, clips against
|
||||
the exact persisted Area and retains VHA point identifiers, watercourse names,
|
||||
profile numbers, measurement dates, available structured depth/width values
|
||||
and official document URLs. Scanned documents remain evidence; missing fields
|
||||
are not filled by fabricated OCR output.
|
||||
|
||||
The following sources are audited but not yet operational:
|
||||
|
||||
- MDK Dieptemodel Belgisch Continentaal Plat/Noordzee: 20 x 20 m continuous
|
||||
raster in LAT, exposed through WCS/WMTS.
|
||||
- SPW bathymetry of navigable waterways and reservoir lakes: 0.5 m bed
|
||||
elevation and XYZ data in mDNG.
|
||||
- Port of Antwerp-Bruges periodic soundings: catalog candidate pending a
|
||||
stable public machine contract.
|
||||
|
||||
See `docs/BATHYMETRY_EXPANSION_ROADMAP.md`. TAW, LAT and mDNG remain separate
|
||||
until an authoritative vertical transformation is implemented and tested.
|
||||
The territorial sea, EEZ and continental shelf are separate scope layers and
|
||||
must be labelled according to their legal meaning.
|
||||
|
||||
@@ -438,3 +438,22 @@ Sprint 7B makes provider architecture operationally visible without performing l
|
||||
- status: `configured`
|
||||
- dataset mapping: `dataset_role=reference`, `source_name=fixture`
|
||||
- write path: checked-in demo/test fixture flow
|
||||
|
||||
## Bathymetry profile vector contract
|
||||
|
||||
The operational VHA layer is an EPSG:4326 point FeatureCollection. Required
|
||||
normalized properties are:
|
||||
|
||||
- `provider_record_id` and `source_feature_id`;
|
||||
- `watercourse_vhag`, `watercourse_name` and alternative names;
|
||||
- `profile_number` and `measurement_date`;
|
||||
- nullable `recorded_depth_m`, `recorded_crown_width_m` and
|
||||
`recorded_floor_width_m`;
|
||||
- nullable allowlisted `source_document_url`;
|
||||
- `document_available`, `structured_depth_available`;
|
||||
- `measurement_semantics=historical_cross_section_profile_point`;
|
||||
- `vertical_reference=document-specific`.
|
||||
|
||||
Null means the provider did not expose a structured value. It is never
|
||||
converted to zero. Dataset metadata records exact counts, measurement range,
|
||||
scope, attribution and `volume_supported=false`.
|
||||
|
||||
@@ -749,3 +749,18 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
- [x] Keep orthophoto pixel refresh manual through a separate
|
||||
plan-stage-review-apply flow that retains the passed preflight identity and
|
||||
creates a new immutable Dataset with official `YYYY.NN` source version.
|
||||
# Bathymetry follow-up
|
||||
|
||||
- [ ] Add governed municipality partition orchestration for all of Flanders
|
||||
after the Mol VHA operator passes live acceptance.
|
||||
- [ ] Add read-only MDK WCS capability/TLS probe and maritime scope metadata.
|
||||
- [ ] Add bounded MDK GeoTIFF acquisition only after live CRS, LAT, nodata and
|
||||
response-limit validation.
|
||||
- [ ] Add SPW staged-download adapter with mDNG metadata and survey-epoch
|
||||
coverage validation.
|
||||
- [ ] Add authoritative territorial-sea, EEZ and continental-shelf boundary
|
||||
layers with legally accurate labels.
|
||||
- [ ] Add vertical-datum conversion only when authoritative transforms and
|
||||
uncertainty tests exist; never merge TAW, LAT and mDNG implicitly.
|
||||
- [ ] Add water volume only when bed and water-surface inputs share a governed
|
||||
time, datum and coverage contract.
|
||||
|
||||
@@ -635,3 +635,16 @@ validation remain inside the collapsed follow-up list with an explicit
|
||||
priority and examples of the measurements they should eventually support.
|
||||
Theme/time-series detail and active-source limitations are also collapsed by
|
||||
default so the Data workspace stays readable.
|
||||
|
||||
## Bathymetry profile map theme
|
||||
|
||||
Ready `vmm_vha_bathymetry_profiles` Datasets activate the `Waterbodem` map
|
||||
theme. The layer renders persisted profile points through the existing
|
||||
MapLibre/GeoJSON path. Clicking a point shows the watercourse, profile number,
|
||||
measurement date, nullable registered depth and an allowlisted official VHA
|
||||
profile-document link.
|
||||
|
||||
The UI always labels the points as historical cross-sections. It does not
|
||||
present them as a continuous bed raster and does not calculate water volume.
|
||||
The Sources workspace lists the future MDK North Sea and SPW Walloon
|
||||
bathymetry integrations as planned until backend acquisition is operational.
|
||||
|
||||
@@ -19,6 +19,7 @@ const THEME_LABELS: Record<string, string> = {
|
||||
nature_value: 'Natuurwaarde',
|
||||
agriculture: 'Landbouw',
|
||||
water: 'Water',
|
||||
bathymetry: 'Waterbodem en dwarsprofielen',
|
||||
flood_hazard: 'Overstromingsgevaar',
|
||||
elevation: 'Hoogte en reliëf',
|
||||
roads: 'Wegen en transport',
|
||||
@@ -81,6 +82,9 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
|
||||
)
|
||||
const dhmvDatasets = ready.filter((dataset) => dataset.source_name === 'digitaal_vlaanderen_dhmv')
|
||||
const floodHazardDatasets = ready.filter((dataset) => dataset.source_name === 'vmm_flood_hazard')
|
||||
const bathymetryProfileDatasets = ready.filter(
|
||||
(dataset) => dataset.source_name === 'vmm_vha_bathymetry_profiles',
|
||||
)
|
||||
const latestBuildingsRegister = [...buildingsRegisterDatasets].sort(
|
||||
(left, right) => new Date(right.observed_at ?? 0).getTime() - new Date(left.observed_at ?? 0).getTime(),
|
||||
)[0]
|
||||
@@ -184,7 +188,7 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
|
||||
</div>
|
||||
</details>
|
||||
|
||||
{waterinfoDatasets.length > 0 || historicalOrthophotos.length > 0 || bwkDatasets.length > 0 || agricultureDatasets.length > 0 || buildingsRegisterDatasets.length > 0 || dhmvDatasets.length > 0 || floodHazardDatasets.length > 0 ? (
|
||||
{waterinfoDatasets.length > 0 || historicalOrthophotos.length > 0 || bwkDatasets.length > 0 || agricultureDatasets.length > 0 || buildingsRegisterDatasets.length > 0 || dhmvDatasets.length > 0 || floodHazardDatasets.length > 0 || bathymetryProfileDatasets.length > 0 ? (
|
||||
<details className="source-loaded-disclosure">
|
||||
<summary>Actieve broncollecties en hun beperkingen</summary>
|
||||
<div className="source-catalog-loaded" aria-label="Aanvullende ingeladen bronnen">
|
||||
@@ -244,6 +248,18 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
|
||||
<p>Gemodelleerde maximumdiepte per kansscenario. Geen actuele waterstand, bathymetrie of permanent watervolume.</p>
|
||||
</article>
|
||||
) : null}
|
||||
{bathymetryProfileDatasets.length > 0 ? (
|
||||
<article>
|
||||
<strong>VHA-dwarsprofielen waterbodem</strong>
|
||||
<span>
|
||||
{bathymetryProfileDatasets.reduce(
|
||||
(total, dataset) => total + Number(dataset.feature_count ?? 0),
|
||||
0,
|
||||
).toLocaleString('nl-BE')} profielpunten · historische meetcampagnes
|
||||
</span>
|
||||
<p>Meetvelden en officiële profielbladen per punt. Geen continue actuele bathymetrie of volume zonder een gelijktijdig waterpeil.</p>
|
||||
</article>
|
||||
) : null}
|
||||
</div>
|
||||
</details>
|
||||
) : null}
|
||||
|
||||
@@ -36,7 +36,7 @@ const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson'
|
||||
const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson'
|
||||
const EMPTY_TEMPORAL_SERIES: DatasetCreateResponse[] = []
|
||||
|
||||
type DataThemeId = 'buildings' | 'space_occupation' | 'open_space' | 'population' | 'forest' | 'nature_value' | 'agriculture' | 'soil' | 'water' | 'flood_hazard' | 'elevation' | 'accessibility' | 'services' | 'roads' | 'parcels'
|
||||
type DataThemeId = 'buildings' | 'space_occupation' | 'open_space' | 'population' | 'forest' | 'nature_value' | 'agriculture' | 'soil' | 'water' | 'bathymetry' | 'flood_hazard' | 'elevation' | 'accessibility' | 'services' | 'roads' | 'parcels'
|
||||
|
||||
interface DataTheme {
|
||||
id: DataThemeId
|
||||
@@ -116,6 +116,13 @@ const DATA_THEMES: DataTheme[] = [
|
||||
description: 'Waterlopen, grachten, kanalen en wateroppervlakken.',
|
||||
tokens: ['waterways', 'waterway', 'water', 'hydro', 'river', 'stream', 'canal', 'waterloop'],
|
||||
},
|
||||
{
|
||||
id: 'bathymetry',
|
||||
label: 'Waterbodem',
|
||||
shortLabel: 'Dwarsprofielen',
|
||||
description: 'Officiële historische VHA-dwarsprofielen met meetvelden en brondocumenten.',
|
||||
tokens: ['bathymetry', 'bathymetry_profiles', 'dwarsprofielen', 'waterbodem'],
|
||||
},
|
||||
{
|
||||
id: 'flood_hazard',
|
||||
label: 'Overstroming',
|
||||
@@ -170,6 +177,7 @@ const DATA_THEME_MAP_STYLES: Record<DataThemeId, { fill: string; line: string }>
|
||||
agriculture: { fill: '#7b8f32', line: '#53671d' },
|
||||
soil: { fill: '#9a7040', line: '#6f4c27' },
|
||||
water: { fill: '#2676a8', line: '#155b85' },
|
||||
bathymetry: { fill: '#0e7490', line: '#164e63' },
|
||||
flood_hazard: { fill: '#1597c2', line: '#075985' },
|
||||
elevation: { fill: '#a57a4b', line: '#315f59' },
|
||||
accessibility: { fill: '#0f766e', line: '#115e59' },
|
||||
@@ -188,6 +196,11 @@ function datasetAvailabilityLabel(dataset: DatasetCreateResponse, partitionCount
|
||||
const resolution = Number(dataset.source_metadata?.['analysis_resolution_m'])
|
||||
return `${Number.isFinite(resolution) ? `${resolution.toLocaleString('nl-BE')} m` : 'Raster'} overstromingsscenario${regionalSuffix}`
|
||||
}
|
||||
if (dataset.source_name === 'vmm_vha_bathymetry_profiles') {
|
||||
const profiles = dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0
|
||||
const documents = Number(dataset.source_metadata?.['document_count'] ?? 0)
|
||||
return `${profiles.toLocaleString('nl-BE')} profielen · ${documents.toLocaleString('nl-BE')} bronbladen`
|
||||
}
|
||||
if (dataset.dataset_type === 'raster' && dataset.source_name === 'department_omgeving_thematic_raster') {
|
||||
const resolution = Number(dataset.source_metadata?.['analysis_resolution_m'])
|
||||
const year = Number(dataset.source_metadata?.['observation_year'])
|
||||
@@ -221,6 +234,9 @@ function datasetMatchesTheme(dataset: DatasetCreateResponse, theme: DataTheme):
|
||||
if (dataset.source_name === 'vmm_flood_hazard') {
|
||||
return theme.id === 'flood_hazard'
|
||||
}
|
||||
if (dataset.source_name === 'vmm_vha_bathymetry_profiles') {
|
||||
return theme.id === 'bathymetry'
|
||||
}
|
||||
if (dataset.source_name === 'digitaal_vlaanderen_dhmv') {
|
||||
return theme.id === 'elevation'
|
||||
}
|
||||
@@ -310,6 +326,7 @@ function pickThemeDataset(
|
||||
(dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register' ? 5_000_000 : 0) +
|
||||
(dataset.source_name === 'digitaal_vlaanderen_dhmv' ? 5_000_000 : 0) +
|
||||
(dataset.source_name === 'vmm_flood_hazard' ? 5_000_000 : 0) +
|
||||
(dataset.source_name === 'vmm_vha_bathymetry_profiles' ? 5_000_000 : 0) +
|
||||
(dataset.source_metadata?.['product_key'] === 'dtm_1m' ? 1_000_000 : 0) +
|
||||
(dataset.source_metadata?.['product_key'] === 'pluviaal_current_t100' ? 1_000_000 : 0) +
|
||||
(dataset.dataset_role === 'reference' ? 10_000 : 0)
|
||||
@@ -618,6 +635,12 @@ export function MapWorkspace({
|
||||
const selectedMapArea = areas.find((area) => area.id === selectedMapAreaId)
|
||||
const regionalScopeSelected = Boolean(selectedMapArea && !isMunicipalityAreaName(selectedMapArea.name))
|
||||
const featureProperties = selectedMapFeature?.properties ?? null
|
||||
const isBathymetryProfile = featureProperties?.['provider'] === 'vmm_vha_bathymetry_profiles'
|
||||
|| featureProperties?.['measurement_semantics'] === 'historical_cross_section_profile_point'
|
||||
const bathymetryDocumentUrl = typeof featureProperties?.['source_document_url'] === 'string'
|
||||
&& featureProperties['source_document_url'].startsWith('https://vha.waterinfo.be/')
|
||||
? featureProperties['source_document_url']
|
||||
: null
|
||||
const featureSummaryEntries = featureProperties
|
||||
? Object.entries(featureProperties)
|
||||
.filter(([, value]) => value !== null && value !== undefined && typeof value !== 'object')
|
||||
@@ -2555,6 +2578,41 @@ export function MapWorkspace({
|
||||
</div>
|
||||
{selectedMapFeature ? (
|
||||
<>
|
||||
{isBathymetryProfile ? (
|
||||
<div className="bathymetry-profile-summary" aria-label="Samenvatting VHA-dwarsprofiel">
|
||||
<div>
|
||||
<span>Waterloop</span>
|
||||
<strong>{String(featureProperties?.['watercourse_name'] ?? 'Onbekende waterloop')}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>Profiel</span>
|
||||
<strong>{String(featureProperties?.['profile_number'] ?? 'n.v.t.')}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>Meetdatum</span>
|
||||
<strong>{String(featureProperties?.['measurement_date'] ?? 'Niet geregistreerd')}</strong>
|
||||
</div>
|
||||
<div>
|
||||
<span>Geregistreerde diepte</span>
|
||||
<strong>
|
||||
{typeof featureProperties?.['recorded_depth_m'] === 'number'
|
||||
? `${featureProperties['recorded_depth_m'].toLocaleString('nl-BE')} m`
|
||||
: 'Niet als veld beschikbaar'}
|
||||
</strong>
|
||||
</div>
|
||||
{bathymetryDocumentUrl ? (
|
||||
<a href={bathymetryDocumentUrl} target="_blank" rel="noreferrer">
|
||||
Officieel profielblad openen
|
||||
</a>
|
||||
) : (
|
||||
<small>Voor dit meetpunt is geen digitaal profielblad gekoppeld.</small>
|
||||
)}
|
||||
<p>
|
||||
Historisch dwarsprofiel. Dit punt is geen continue actuele bodemkaart en levert zonder
|
||||
gelijktijdig waterpeil geen actueel watervolume.
|
||||
</p>
|
||||
</div>
|
||||
) : null}
|
||||
<div className="feature-extract-grid" aria-label="Geometrie van het geselecteerde object">
|
||||
<div>
|
||||
<span>Geometrie</span>
|
||||
|
||||
@@ -4,6 +4,7 @@ const DATASET_LABEL_BY_LAYER: Record<string, string> = {
|
||||
buildings: 'Bebouwing',
|
||||
roads: 'Wegen',
|
||||
water: 'Water',
|
||||
bathymetry_profiles: 'Dwarsprofielen waterbodem',
|
||||
parcels: 'Percelen',
|
||||
population: 'Bevolking',
|
||||
forest: 'Bos en groen',
|
||||
@@ -34,6 +35,7 @@ const DATASET_SOURCE_LABELS: Record<string, string> = {
|
||||
vrbg: 'Digitaal Vlaanderen',
|
||||
waterinfo: 'Waterinfo Vlaanderen',
|
||||
vmm_flood_hazard: 'Vlaamse Milieumaatschappij',
|
||||
vmm_vha_bathymetry_profiles: 'VMM / Vlaamse Hydrografische Atlas',
|
||||
department_omgeving_thematic_raster: 'Departement Omgeving',
|
||||
dov_soil_map: 'Databank Ondergrond Vlaanderen',
|
||||
}
|
||||
@@ -52,6 +54,9 @@ export function getDatasetDisplayName(dataset: DatasetCreateResponse): string {
|
||||
const productName = dataset.source_metadata?.['product_display_name']
|
||||
return typeof productName === 'string' && productName.trim() ? productName : 'VMM-overstromingsscenario'
|
||||
}
|
||||
if (dataset.source_name === 'vmm_vha_bathymetry_profiles') {
|
||||
return 'VHA-dwarsprofielen waterbodem'
|
||||
}
|
||||
if (dataset.source_name === 'department_omgeving_thematic_raster') {
|
||||
const productName = dataset.source_metadata?.['product_display_name']
|
||||
return typeof productName === 'string' && productName.trim() ? productName : 'Officieel Vlaams themaraster'
|
||||
|
||||
@@ -384,6 +384,42 @@ export const OFFICIAL_SOURCE_PORTFOLIO: OfficialSourceDefinition[] = [
|
||||
url: 'https://www.vlaanderen.be/datavindplaats/catalogus/ogc-api-features-vlaamse-hydrografische-atlas-waterlopen',
|
||||
matches: (dataset) => sourceNameIs(dataset, 'vmm_vha'),
|
||||
},
|
||||
{
|
||||
key: 'vha_bathymetry_profiles',
|
||||
domain: 'climate',
|
||||
name: 'VHA-dwarsprofielen waterbodem',
|
||||
owner: 'Vlaamse Milieumaatschappij',
|
||||
coverage: 'Historische profielpunten op Vlaamse waterlopen',
|
||||
value: 'Gemeten profielinformatie en officiële bronbladen op exacte kaartlocaties.',
|
||||
metricExamples: 'aantal profielen, geregistreerde diepte en breedte per meetpunt',
|
||||
priority: 'next',
|
||||
url: 'https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer',
|
||||
matches: (dataset) => sourceNameIs(dataset, 'vmm_vha_bathymetry_profiles'),
|
||||
},
|
||||
{
|
||||
key: 'mdk_bcp_bathymetry',
|
||||
domain: 'climate',
|
||||
name: 'Dieptemodel Belgisch Continentaal Plat',
|
||||
owner: 'Agentschap Maritieme Dienstverlening en Kust',
|
||||
coverage: 'Belgische Noordzee, 20 x 20 m raster in LAT',
|
||||
value: 'Continue zeebodemhoogte voor mariene selectie en evolutie.',
|
||||
metricExamples: 'minimum/maximum diepte, diepteklassen en bodemvolume per peilcampagne',
|
||||
priority: 'planned',
|
||||
url: 'https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee',
|
||||
matches: (dataset) => sourceNameIs(dataset, 'mdk_bcp_bathymetry'),
|
||||
},
|
||||
{
|
||||
key: 'spw_waterway_bathymetry',
|
||||
domain: 'climate',
|
||||
name: 'Waalse vaarweg- en stuwmeerbathymetrie',
|
||||
owner: 'Service public de Wallonie',
|
||||
coverage: 'Gemeten Waalse vaarwegen en stuwmeren, 0,5 m in mDNG',
|
||||
value: 'Hoogwaardige bodemrasters voor federale uitbreiding buiten Vlaanderen.',
|
||||
metricExamples: 'bodemhoogte, diepteprofiel en vergelijkbare meetcampagnes',
|
||||
priority: 'planned',
|
||||
url: 'https://geoportail.wallonie.be/catalogue/c450c28f-d357-48af-8423-62d524632cf9.html',
|
||||
matches: (dataset) => sourceNameIs(dataset, 'spw_walloon_waterway_bathymetry'),
|
||||
},
|
||||
]
|
||||
|
||||
export function operationalSources(
|
||||
|
||||
@@ -22,6 +22,8 @@ import type {
|
||||
FloodHazardAcquireRequest,
|
||||
FloodHazardProductRead,
|
||||
FloodHazardSelectionResponse,
|
||||
BathymetryProfileAcquireRequest,
|
||||
BathymetrySourceRead,
|
||||
DhmvProductRead,
|
||||
TerrainSelectionResponse,
|
||||
ThematicRasterAcquireRequest,
|
||||
@@ -167,6 +169,12 @@ export const datasetsApi = {
|
||||
payload: { bbox: VectorSelectionRequest['bbox']; area_id?: string; product_key: string },
|
||||
): Promise<FloodHazardSelectionResponse> =>
|
||||
apiPost<FloodHazardSelectionResponse>(`/api/v1/projects/${projectId}/datasets/raster/flood-hazard/select`, payload),
|
||||
listBathymetrySources: (projectId: string): Promise<{ items: BathymetrySourceRead[]; total: number }> =>
|
||||
apiGet<{ items: BathymetrySourceRead[]; total: number }>(
|
||||
`/api/v1/projects/${projectId}/datasets/bathymetry/sources`,
|
||||
),
|
||||
acquireBathymetryProfiles: (projectId: string, payload: BathymetryProfileAcquireRequest): Promise<JobRead> =>
|
||||
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/bathymetry/profiles/acquire`, payload),
|
||||
acquireThematicRaster: (projectId: string, payload: ThematicRasterAcquireRequest): Promise<JobRead> =>
|
||||
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/thematic-raster/acquire`, payload),
|
||||
listThematicRasterProducts: (projectId: string): Promise<{ items: ThematicRasterProductRead[]; total: number }> =>
|
||||
|
||||
@@ -5966,6 +5966,11 @@ section {
|
||||
background: rgba(38, 118, 168, 0.24);
|
||||
}
|
||||
|
||||
.geo-map-legend .geo-legend-layer-bathymetry {
|
||||
background: #0e7490;
|
||||
border-color: #164e63;
|
||||
}
|
||||
|
||||
.geo-map-legend .geo-legend-layer-soil {
|
||||
border-color: #6f4c27;
|
||||
background: rgba(154, 112, 64, 0.24);
|
||||
@@ -6036,6 +6041,60 @@ section {
|
||||
.geo-legend-ramp-population { background: linear-gradient(90deg, #eee7f6, #673a97); }
|
||||
.geo-legend-ramp-accessibility { background: linear-gradient(90deg, #e9f1f4, #0f766e); }
|
||||
.geo-legend-ramp-services { background: linear-gradient(90deg, #fff4bf, #b66d16); }
|
||||
.geo-legend-ramp-bathymetry { background: linear-gradient(90deg, #cffafe, #0e7490); }
|
||||
|
||||
.bathymetry-profile-summary {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(4, minmax(0, 1fr));
|
||||
gap: 0.65rem;
|
||||
margin-bottom: 0.9rem;
|
||||
padding: 0.8rem;
|
||||
border: 1px solid #a5d8e6;
|
||||
border-radius: 6px;
|
||||
background: #effbff;
|
||||
}
|
||||
|
||||
.bathymetry-profile-summary > div {
|
||||
display: grid;
|
||||
gap: 0.2rem;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.bathymetry-profile-summary span,
|
||||
.bathymetry-profile-summary small {
|
||||
color: #46646f;
|
||||
font-size: 0.75rem;
|
||||
}
|
||||
|
||||
.bathymetry-profile-summary strong {
|
||||
overflow-wrap: anywhere;
|
||||
color: #113d4b;
|
||||
}
|
||||
|
||||
.bathymetry-profile-summary a,
|
||||
.bathymetry-profile-summary p,
|
||||
.bathymetry-profile-summary small {
|
||||
grid-column: 1 / -1;
|
||||
}
|
||||
|
||||
.bathymetry-profile-summary a {
|
||||
width: fit-content;
|
||||
color: #075985;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.bathymetry-profile-summary p {
|
||||
margin: 0;
|
||||
color: #315867;
|
||||
font-size: 0.78rem;
|
||||
line-height: 1.45;
|
||||
}
|
||||
|
||||
@media (max-width: 760px) {
|
||||
.bathymetry-profile-summary {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
}
|
||||
|
||||
.geo-map-legend .geo-legend-added {
|
||||
border-color: #15803d;
|
||||
|
||||
@@ -442,6 +442,33 @@ export interface FloodHazardSelectionResponse {
|
||||
generated_at: string
|
||||
}
|
||||
|
||||
export interface BathymetryProfileAcquireRequest {
|
||||
bbox: VectorSelectionBBox
|
||||
area_id?: string | null
|
||||
force_refresh?: boolean
|
||||
}
|
||||
|
||||
export interface BathymetrySourceRead {
|
||||
key: string
|
||||
display_name: string
|
||||
owner: string
|
||||
authority_level: 'authoritative' | 'contextual'
|
||||
geographic_coverage: string
|
||||
data_kind: string
|
||||
query_modes: string[]
|
||||
vertical_reference: string
|
||||
horizontal_crs: string
|
||||
native_resolution?: string | null
|
||||
integration_status: 'operational' | 'available_not_integrated' | 'catalog_only'
|
||||
acquisition_supported: boolean
|
||||
configured: boolean
|
||||
service_url?: string | null
|
||||
catalog_url: string
|
||||
attribution: string
|
||||
license_note: string
|
||||
limitation_message: string
|
||||
}
|
||||
|
||||
export interface ThematicRasterAcquireRequest {
|
||||
bbox: VectorSelectionBBox
|
||||
area_id?: string | null
|
||||
|
||||
@@ -1882,3 +1882,30 @@ boundary artifact. Defaults cap each municipality at 30,000 source features and
|
||||
the assembled snapshot at 300,000 features. It never truncates silently, never
|
||||
writes `vector_features` directly and never turns the single 2025 state into a
|
||||
fabricated historical series.
|
||||
|
||||
## Mol VHA bathymetry profiles
|
||||
|
||||
Provision and verify the official profile points for the exact persisted Mol
|
||||
Area:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_mol_bathymetry_profiles.py
|
||||
```
|
||||
|
||||
Use `--force` only for a deliberate fresh provider snapshot. The command finds
|
||||
`Mol Municipality Workbench` and `Gemeente Mol`, calls the canonical
|
||||
bathymetry acquisition endpoint and then verifies that vector selection count
|
||||
and semantic metrics match the persisted Job result. It never writes directly
|
||||
to PostGIS.
|
||||
|
||||
For another approved workspace or Area:
|
||||
|
||||
```bash
|
||||
python scripts/provision_mol_bathymetry_profiles.py \
|
||||
--base-url http://127.0.0.1:8000 \
|
||||
--project-name "Project name" \
|
||||
--area-name "Area name fragment"
|
||||
```
|
||||
|
||||
The output is a point dataset with historical evidence. It is not a continuous
|
||||
water-bottom raster and cannot calculate current water volume.
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Provision official VHA cross-section profile points for the persisted Mol Area.
|
||||
|
||||
The operator uses only canonical GeoIntel API endpoints. The backend queries the
|
||||
official VHA service, clips points against the exact persisted Area geometry,
|
||||
stores the GeoJSON artifact and persists VectorFeature rows through DatasetService.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from typing import Any, Iterable
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_PROJECT_NAME = "Mol Municipality Workbench"
|
||||
DEFAULT_AREA_FRAGMENT = "Gemeente Mol"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision official VHA cross-section profiles for Mol.")
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME)
|
||||
parser.add_argument("--area-name", default=DEFAULT_AREA_FRAGMENT)
|
||||
parser.add_argument("--timeout", type=int, default=900)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def unwrap(response: requests.Response) -> Any:
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if not isinstance(payload, dict) or "data" not in payload:
|
||||
raise RuntimeError(f"Non-canonical API response from {response.url}")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def coordinates(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
|
||||
def walk(value: Any):
|
||||
if isinstance(value, list) and len(value) >= 2 and all(isinstance(item, (int, float)) for item in value[:2]):
|
||||
yield float(value[0]), float(value[1])
|
||||
return
|
||||
if isinstance(value, list):
|
||||
for child in value:
|
||||
yield from walk(child)
|
||||
|
||||
yield from walk(geometry.get("coordinates", []))
|
||||
|
||||
|
||||
def geometry_bbox(geometry: dict[str, Any]) -> dict[str, float | str]:
|
||||
points = list(coordinates(geometry))
|
||||
if not points:
|
||||
raise RuntimeError("Persisted Area geometry contains no coordinates")
|
||||
xs = [point[0] for point in points]
|
||||
ys = [point[1] for point in points]
|
||||
return {
|
||||
"min_x": min(xs),
|
||||
"min_y": min(ys),
|
||||
"max_x": max(xs),
|
||||
"max_y": max(ys),
|
||||
"crs": "EPSG:4326",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
base_url = args.base_url.rstrip("/")
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": "GeoIntel-Bathymetry-Operator/1.0"})
|
||||
|
||||
projects = unwrap(session.get(f"{base_url}/api/v1/projects", params={"limit": 200, "offset": 0}, timeout=60))["items"]
|
||||
project = next((item for item in projects if item["name"] == args.project_name), None)
|
||||
if project is None:
|
||||
raise RuntimeError(f"Project {args.project_name!r} was not found")
|
||||
|
||||
areas = unwrap(
|
||||
session.get(
|
||||
f"{base_url}/api/v1/projects/{project['id']}/areas",
|
||||
params={"limit": 200, "offset": 0},
|
||||
timeout=60,
|
||||
)
|
||||
)["items"]
|
||||
fragment = args.area_name.casefold()
|
||||
area = next((item for item in areas if fragment in item["name"].casefold()), None)
|
||||
if area is None:
|
||||
raise RuntimeError(f"Area containing {args.area_name!r} was not found")
|
||||
bbox = geometry_bbox(area["geometry"])
|
||||
|
||||
job = unwrap(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects/{project['id']}/datasets/bathymetry/profiles/acquire",
|
||||
json={
|
||||
"bbox": bbox,
|
||||
"area_id": area["id"],
|
||||
"force_refresh": args.force,
|
||||
},
|
||||
timeout=args.timeout,
|
||||
)
|
||||
)
|
||||
if job.get("status") != "success" or not job.get("output_dataset_id"):
|
||||
raise RuntimeError(f"Bathymetry profile acquisition failed: {job.get('error_message') or job}")
|
||||
result = job.get("result_json") or {}
|
||||
dataset_id = job["output_dataset_id"]
|
||||
selection = unwrap(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects/{project['id']}/datasets/{dataset_id}/vector/select",
|
||||
json={"bbox": bbox, "area_id": area["id"], "limit": 1000},
|
||||
timeout=args.timeout,
|
||||
)
|
||||
)
|
||||
if selection.get("total_feature_count") != result.get("profile_count"):
|
||||
raise RuntimeError(
|
||||
"Persisted vector selection count does not match the exact acquired profile count "
|
||||
f"({selection.get('total_feature_count')} != {result.get('profile_count')})"
|
||||
)
|
||||
metrics = {
|
||||
item["metric_key"]: item
|
||||
for item in (selection.get("summary") or {}).get("metrics", [])
|
||||
if isinstance(item, dict) and item.get("metric_key")
|
||||
}
|
||||
if metrics.get("profile_count", {}).get("metric_value") != result.get("profile_count"):
|
||||
raise RuntimeError("Bathymetry profile summary does not expose the persisted profile count")
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"project_id": project["id"],
|
||||
"area_id": area["id"],
|
||||
"area_name": area["name"],
|
||||
"bbox": bbox,
|
||||
"dataset_id": dataset_id,
|
||||
"reused": bool(result.get("reused")),
|
||||
"profile_count": result.get("profile_count"),
|
||||
"document_count": result.get("document_count"),
|
||||
"structured_depth_count": result.get("structured_depth_count"),
|
||||
"watercourse_count": result.get("watercourse_count"),
|
||||
"measurement_date_min": result.get("measurement_date_min"),
|
||||
"measurement_date_max": result.get("measurement_date_max"),
|
||||
"metrics": list(metrics.values()),
|
||||
"limitation_message": result.get("limitation_message"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -59,6 +59,7 @@ ${PYTHON_BIN} -m py_compile scripts/orthophoto_release_preflight.py
|
||||
${PYTHON_BIN} -m py_compile scripts/manage_orthophoto_release.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_bathymetry_profiles.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_dhmv.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_flood_hazards.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_flood_hazards.py
|
||||
|
||||
Reference in New Issue
Block a user