Add governed bathymetry profile workflow
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-17 15:25:00 +02:00
parent c0cab9e3c5
commit 5b4e18059a
31 changed files with 1990 additions and 4 deletions
+7
View File
@@ -36,6 +36,13 @@ FLOOD_HAZARD_MAX_SIDE_M=20000
FLOOD_HAZARD_MAX_PIXELS=12000000
FLOOD_HAZARD_TIMEOUT_SECONDS=300
FLOOD_HAZARD_MAX_RESPONSE_MB=160
BATHYMETRY_PROFILES_ENABLED=true
BATHYMETRY_PROFILES_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0
BATHYMETRY_WATERCOURSE_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1
BATHYMETRY_PROFILES_PAGE_SIZE=1000
BATHYMETRY_PROFILES_MAX_FEATURES=50000
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
THEMATIC_RASTER_ENABLED=true
THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs
THEMATIC_RASTER_MIN_SIDE_M=100
+15
View File
@@ -7,6 +7,21 @@
# Changelog
## Sprint 235 Governed bathymetry profiles and Belgian scale architecture (2026-07-17)
- Added bounded official VHA cross-section acquisition with exact Area
clipping, complete paging, checksums and ordinary Dataset/VectorFeature
persistence.
- Added nullable structured depth/width metrics, official profile-document
evidence and explicit unsupported volume semantics.
- Added a bathymetry source registry covering operational VHA profiles and
audited MDK North Sea, SPW Walloon and port candidates without pretending
those future connectors are configured.
- Added the `Waterbodem` Map theme and a concise profile inspector.
- Added the Mol operator command and a staged expansion roadmap for Flanders,
Belgium, the territorial sea, EEZ and continental shelf with strict
TAW/LAT/mDNG separation.
## Sprint 234 Full audit closure and workspace lifecycle cleanup (2026-07-17)
- Added reversible active/archived project lifecycle handling and made active
+20
View File
@@ -1708,3 +1708,23 @@ The command never deletes projects or related datasets, jobs, analyses,
quality checks and exports. It always preserves `Kempen Regional Workbench`
and `Mol Municipality Workbench`, defaults to dry-run and can print every
matched name with `--show-names`.
## Governed VHA bathymetry profiles
`POST /api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire`
performs a bounded official VHA ArcGIS query, exact persisted-Area clipping,
watercourse-name normalization and ordinary Dataset/VectorFeature persistence.
`GET /api/v1/projects/{project_id}/datasets/bathymetry/sources` reports VHA as
operational and the audited MDK/SPW candidates as unavailable for acquisition.
Runtime controls are `BATHYMETRY_PROFILES_ENABLED`,
`BATHYMETRY_PROFILES_LAYER_URL`, `BATHYMETRY_WATERCOURSE_LAYER_URL`,
`BATHYMETRY_PROFILES_PAGE_SIZE`, `BATHYMETRY_PROFILES_MAX_FEATURES`,
`BATHYMETRY_PROFILES_TIMEOUT_SECONDS` and
`BATHYMETRY_PROFILES_MAX_RESPONSE_MB`. The feature limit intentionally forces
large Flemish scopes into exact Area partitions.
The Dataset exposes profile count and nullable structured depth/width metrics.
It does not claim a continuous bed model, current depth or volume. Use
`scripts/provision_mol_bathymetry_profiles.py` for the canonical Mol operator
flow.
+26
View File
@@ -27,6 +27,7 @@ from app.schemas import (
FloodHazardAcquireRequest,
FloodHazardPartitionSelectionRequest,
FloodHazardSelectionRequest,
BathymetryProfileAcquireRequest,
ThematicRasterAcquireRequest,
ThematicRasterSelectionRequest,
VectorBBoxResponse,
@@ -52,6 +53,7 @@ from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.terrain_analysis_service import TerrainAnalysisService
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
from app.utils.response import envelope
@@ -213,6 +215,30 @@ def list_flood_hazard_products(project_id: UUID, db: Session = Depends(get_db)):
return envelope({"items": items, "total": len(items)})
@router.get("/datasets/bathymetry/sources", response_model=dict)
def list_bathymetry_sources(project_id: UUID, db: Session = Depends(get_db)):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
items = BathymetryProfileAcquisitionService.list_sources()
return envelope({"items": items, "total": len(items)})
@router.post("/datasets/bathymetry/profiles/acquire", response_model=dict)
def acquire_bounded_bathymetry_profiles(
project_id: UUID,
payload: BathymetryProfileAcquireRequest,
db: Session = Depends(get_db),
):
job = JobService.run_sync_job(
db=db,
project_id=project_id,
job_type="vector.bathymetry_profiles.acquire",
parameters=payload.model_dump(mode="json"),
operation=lambda: BathymetryProfileAcquisitionService.acquire(db, project_id, payload),
)
return envelope(job)
@router.post("/datasets/thematic-raster/acquire", response_model=dict)
def acquire_bounded_thematic_raster(
project_id: UUID,
+33
View File
@@ -90,6 +90,39 @@ class Settings(BaseSettings):
flood_hazard_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="FLOOD_HAZARD_MAX_PIXELS")
flood_hazard_timeout_seconds: int = Field(default=300, ge=1, validation_alias="FLOOD_HAZARD_TIMEOUT_SECONDS")
flood_hazard_max_response_mb: int = Field(default=160, ge=1, validation_alias="FLOOD_HAZARD_MAX_RESPONSE_MB")
bathymetry_profiles_enabled: bool = Field(default=True, validation_alias="BATHYMETRY_PROFILES_ENABLED")
bathymetry_profiles_layer_url: str = Field(
default="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
validation_alias="BATHYMETRY_PROFILES_LAYER_URL",
)
bathymetry_watercourse_layer_url: str = Field(
default="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1",
validation_alias="BATHYMETRY_WATERCOURSE_LAYER_URL",
)
bathymetry_profiles_page_size: int = Field(
default=1000,
ge=1,
le=2000,
validation_alias="BATHYMETRY_PROFILES_PAGE_SIZE",
)
bathymetry_profiles_max_features: int = Field(
default=50_000,
ge=1,
le=250_000,
validation_alias="BATHYMETRY_PROFILES_MAX_FEATURES",
)
bathymetry_profiles_timeout_seconds: int = Field(
default=120,
ge=1,
le=600,
validation_alias="BATHYMETRY_PROFILES_TIMEOUT_SECONDS",
)
bathymetry_profiles_max_response_mb: int = Field(
default=32,
ge=1,
le=256,
validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB",
)
thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED")
thematic_raster_wcs_url: str = Field(
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
+8
View File
@@ -65,6 +65,11 @@ from .flood_hazard import (
FloodHazardSelectionResponse,
FloodHazardSelectionSummary,
)
from .bathymetry import (
BathymetryProfileAcquireRequest,
BathymetryProfileAcquisitionResult,
BathymetrySourceRead,
)
from .thematic_raster import (
ThematicRasterAcquireRequest,
ThematicRasterAcquisitionResult,
@@ -202,6 +207,9 @@ __all__ = [
"FloodHazardSelectionRequest",
"FloodHazardSelectionResponse",
"FloodHazardSelectionSummary",
"BathymetryProfileAcquireRequest",
"BathymetryProfileAcquisitionResult",
"BathymetrySourceRead",
"ThematicRasterAcquireRequest",
"ThematicRasterAcquisitionResult",
"ThematicRasterMetric",
+52
View File
@@ -0,0 +1,52 @@
from __future__ import annotations
from typing import Literal
from uuid import UUID
from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox
class BathymetryProfileAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
force_refresh: bool = False
class BathymetrySourceRead(BaseModel):
key: str
display_name: str
owner: str
authority_level: Literal["authoritative", "contextual"]
geographic_coverage: str
data_kind: str
query_modes: list[str]
vertical_reference: str
horizontal_crs: str
native_resolution: str | None = None
integration_status: Literal["operational", "available_not_integrated", "catalog_only"]
acquisition_supported: bool
configured: bool
service_url: str | None = None
catalog_url: str
attribution: str
license_note: str
limitation_message: str
class BathymetryProfileAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
profile_count: int = Field(ge=0)
document_count: int = Field(ge=0)
structured_depth_count: int = Field(ge=0)
structured_width_count: int = Field(ge=0)
watercourse_count: int = Field(ge=0)
bbox_epsg4326: list[float]
clipped_to_area_id: UUID | None = None
measurement_date_min: str | None = None
measurement_date_max: str | None = None
attribution: str
limitation_message: str
@@ -0,0 +1,671 @@
from __future__ import annotations
from datetime import UTC, datetime
import hashlib
import json
import math
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from uuid import UUID
from geoalchemy2.shape import to_shape
from shapely.geometry import Point, box, mapping
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import Area, Dataset, Project
from app.schemas.bathymetry import (
BathymetryProfileAcquireRequest,
BathymetryProfileAcquisitionResult,
BathymetrySourceRead,
)
from app.services.dataset_service import DatasetService
class BathymetryProfileAcquisitionService:
PROVIDER = "vmm_vha_bathymetry_profiles"
SOURCE_VERSION = "VHA digitale atlas ArcGIS MapServer"
PROFILE_OUT_FIELDS = (
"OBJECTID,vhag,atlaspunt,opg_kruinb,opg_vloerb,d_opmeti,"
"hyperlink,bron,kunstwerkid,opg_diepte"
)
ATTRIBUTION = "Vlaamse Milieumaatschappij (VMM), Vlaamse Hydrografische Atlas"
LICENSE_NOTE = "Hergebruik volgens de voorwaarden van de Vlaamse overheid en de bronmetadata."
LIMITATION = (
"Dwarsprofielen zijn historische puntmetingen met bronafhankelijke meetdatum en verticale referentie. "
"Ze vormen geen continue actuele bodemkaart en ondersteunen zonder gelijktijdig waterpeil geen "
"gebiedsdekkend of actueel watervolume."
)
_SOURCES = (
{
"key": "vha_inland_profiles",
"display_name": "VHA dwarsprofielen binnenwater",
"owner": "Vlaamse Milieumaatschappij",
"authority_level": "authoritative",
"geographic_coverage": "Vlaanderen, puntlocaties op gekarteerde waterlopen",
"data_kind": "dwarsprofielpunten met meetvelden en brondocumenten",
"query_modes": ["bbox", "persisted_area"],
"vertical_reference": "document-specific; niet uniform als één peilreferentie te behandelen",
"horizontal_crs": "EPSG:4326",
"native_resolution": None,
"integration_status": "operational",
"acquisition_supported": True,
"configured": True,
"service_url": "https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0",
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/vlaamse-hydrografische-atlas-waterlopen",
"attribution": ATTRIBUTION,
"license_note": LICENSE_NOTE,
"limitation_message": LIMITATION,
},
{
"key": "mdk_bcp_bathymetry",
"display_name": "Dieptemodel Belgisch Continentaal Plat",
"owner": "Agentschap Maritieme Dienstverlening en Kust",
"authority_level": "authoritative",
"geographic_coverage": "Belgisch Continentaal Plat en Noordzee",
"data_kind": "continu bathymetrisch raster",
"query_modes": ["wcs", "wmts", "bounded_raster"],
"vertical_reference": "LAT",
"horizontal_crs": "bronafhankelijk; expliciet per WCS-respons",
"native_resolution": "20 x 20 m",
"integration_status": "available_not_integrated",
"acquisition_supported": False,
"configured": False,
"service_url": "https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/WCS_Public",
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee",
"attribution": "Agentschap Maritieme Dienstverlening en Kust",
"license_note": "Zie de officiële datasetmetadata en gebruiksvoorwaarden.",
"limitation_message": (
"Niet operationeel in GeoIntel totdat WCS, maritieme begrenzing, tegels, LAT-semantiek en "
"servercertificaten in een live integratietest zijn gevalideerd."
),
},
{
"key": "spw_walloon_waterway_bathymetry",
"display_name": "Bathymétrie des voies navigables et lacs-réservoirs",
"owner": "Service public de Wallonie",
"authority_level": "authoritative",
"geographic_coverage": "Waalse bevaarbare waterwegen en stuwmeren met uitgevoerde opmetingen",
"data_kind": "bodemhoogteraster en XYZ-puntenwolk",
"query_modes": ["download", "arcgis_map_service"],
"vertical_reference": "mDNG",
"horizontal_crs": "EPSG:3812; visualisatieservice kan EPSG:31370 aanbieden",
"native_resolution": "0,5 m",
"integration_status": "available_not_integrated",
"acquisition_supported": False,
"configured": False,
"service_url": "https://geoservices.wallonie.be/arcgis/rest/services/EAU/BATHY/MapServer",
"catalog_url": "https://geoportail.wallonie.be/catalogue/c450c28f-d357-48af-8423-62d524632cf9.html",
"attribution": "Service public de Wallonie",
"license_note": "CC BY 4.0 volgens de officiële Geoportail-metadata.",
"limitation_message": (
"Dekking en meetjaar verschillen per vaarweg of reservoir. Integratie vereist een beheerde "
"download- en mosaïekstroom plus expliciete omzetting van mDNG."
),
},
{
"key": "port_antwerp_bathymetry",
"display_name": "Havenbathymetrie Antwerpen-Brugge",
"owner": "Port of Antwerp-Bruges",
"authority_level": "contextual",
"geographic_coverage": "Gepubliceerde havenzones en meetcampagnes",
"data_kind": "periodieke peilingen",
"query_modes": ["catalog"],
"vertical_reference": "product-specific",
"horizontal_crs": "product-specific",
"native_resolution": None,
"integration_status": "catalog_only",
"acquisition_supported": False,
"configured": False,
"service_url": None,
"catalog_url": "https://data.gov.be/nl/datasets",
"attribution": "Port of Antwerp-Bruges",
"license_note": "Per publicatie te verifiëren.",
"limitation_message": (
"Alleen als cataloguskandidaat geregistreerd; er is nog geen stabiel, publiek en machineleesbaar "
"acquisitiecontract in GeoIntel gevalideerd."
),
},
)
@staticmethod
def list_sources() -> list[dict[str, Any]]:
return [BathymetrySourceRead(**item).model_dump() for item in BathymetryProfileAcquisitionService._SOURCES]
@staticmethod
def _validate_bbox(payload: BathymetryProfileAcquireRequest) -> tuple[float, float, float, float]:
bbox = payload.bbox
if bbox.crs.upper() != "EPSG:4326":
raise AppError(
code="BATHYMETRY_INVALID_CRS",
message="Bathymetry profile acquisition requires EPSG:4326",
status_code=400,
)
values = (bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
if not all(math.isfinite(value) for value in values):
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box values must be finite", status_code=400)
if bbox.min_x >= bbox.max_x or bbox.min_y >= bbox.max_y:
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box has no area", status_code=400)
if bbox.min_x < -180 or bbox.max_x > 180 or bbox.min_y < -90 or bbox.max_y > 90:
raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box is outside EPSG:4326", status_code=400)
return values
@staticmethod
def _scope_geometry(db, project_id: UUID, area_id: UUID | None, bbox_values: tuple[float, float, float, float]):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
selection = box(*bbox_values)
if area_id is None:
return selection
area = db.get(Area, area_id)
if area is None:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
area_geometry = area.geometry if hasattr(area.geometry, "__geo_interface__") else to_shape(area.geometry)
intersection = area_geometry.intersection(selection)
if intersection.is_empty:
raise AppError(
code="BATHYMETRY_SCOPE_EMPTY",
message="The requested bounding box does not intersect the selected area",
status_code=400,
)
return intersection
@staticmethod
def _query_url(base_url: str, parameters: dict[str, Any]) -> str:
return f"{base_url}?{urlencode(parameters)}"
@staticmethod
def _fetch_json(
url: str,
settings: Settings,
opener: Callable[..., Any] | None,
) -> tuple[dict[str, Any], str]:
request = Request(
url,
headers={
"Accept": "application/json",
"User-Agent": "GeoIntel/1.0 bathymetry-profile-acquisition",
},
)
try:
with (opener or urlopen)(request, timeout=settings.bathymetry_profiles_timeout_seconds) as response:
limit = settings.bathymetry_profiles_max_response_mb * 1024 * 1024
content = response.read(limit + 1)
except HTTPError as exc:
raise AppError(
code="BATHYMETRY_PROVIDER_HTTP_ERROR",
message="VHA profile service returned an HTTP error",
details={"status_code": exc.code},
status_code=502,
) from exc
except (TimeoutError, URLError, OSError) as exc:
raise AppError(
code="BATHYMETRY_PROVIDER_UNAVAILABLE",
message="VHA profile service is unavailable",
status_code=502,
) from exc
if len(content) > limit:
raise AppError(
code="BATHYMETRY_PROVIDER_RESPONSE_TOO_LARGE",
message="VHA profile response exceeded the configured size limit",
status_code=502,
)
try:
payload = json.loads(content.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise AppError(
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
message="VHA profile service returned invalid JSON",
status_code=502,
) from exc
if not isinstance(payload, dict) or payload.get("error"):
raise AppError(
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
message="VHA profile service returned an ArcGIS error",
details={"provider_error": payload.get("error") if isinstance(payload, dict) else None},
status_code=502,
)
return payload, hashlib.sha256(content).hexdigest()
@staticmethod
def _base_spatial_parameters(bbox_values: tuple[float, float, float, float]) -> dict[str, str]:
return {
"f": "json",
"where": "1=1",
"geometry": ",".join(f"{value:.12g}" for value in bbox_values),
"geometryType": "esriGeometryEnvelope",
"inSR": "4326",
"outSR": "4326",
"spatialRel": "esriSpatialRelIntersects",
}
@staticmethod
def _fetch_profiles(
bbox_values: tuple[float, float, float, float],
settings: Settings,
opener: Callable[..., Any] | None,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
base = settings.bathymetry_profiles_layer_url.rstrip("/") + "/query"
count_url = BathymetryProfileAcquisitionService._query_url(
base,
{
**BathymetryProfileAcquisitionService._base_spatial_parameters(bbox_values),
"returnCountOnly": "true",
"returnGeometry": "false",
},
)
count_payload, count_sha = BathymetryProfileAcquisitionService._fetch_json(count_url, settings, opener)
candidate_count = int(count_payload.get("count") or 0)
if candidate_count > settings.bathymetry_profiles_max_features:
raise AppError(
code="BATHYMETRY_SCOPE_TOO_LARGE",
message="The requested profile scope exceeds the configured feature limit; acquire smaller area partitions",
details={
"candidate_count": candidate_count,
"max_features": settings.bathymetry_profiles_max_features,
},
status_code=413,
)
features: list[dict[str, Any]] = []
response_hashes: list[str] = []
request_urls: list[str] = [count_url]
offset = 0
while offset < candidate_count:
page_url = BathymetryProfileAcquisitionService._query_url(
base,
{
**BathymetryProfileAcquisitionService._base_spatial_parameters(bbox_values),
"outFields": BathymetryProfileAcquisitionService.PROFILE_OUT_FIELDS,
"returnGeometry": "true",
"orderByFields": "OBJECTID",
"resultOffset": str(offset),
"resultRecordCount": str(settings.bathymetry_profiles_page_size),
},
)
page, page_sha = BathymetryProfileAcquisitionService._fetch_json(page_url, settings, opener)
page_features = page.get("features")
if not isinstance(page_features, list):
raise AppError(
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
message="VHA profile response does not contain a feature list",
status_code=502,
)
features.extend(item for item in page_features if isinstance(item, dict))
response_hashes.append(page_sha)
request_urls.append(page_url)
if not page_features:
break
offset += len(page_features)
if len(features) != candidate_count:
raise AppError(
code="BATHYMETRY_PROVIDER_INCOMPLETE_RESPONSE",
message="VHA profile pagination did not return the announced number of records",
details={"expected": candidate_count, "received": len(features)},
status_code=502,
)
return features, {
"candidate_count": candidate_count,
"request_urls": request_urls,
"response_sha256": [count_sha, *response_hashes],
}
@staticmethod
def _fetch_watercourse_names(
vhag_codes: set[int],
settings: Settings,
opener: Callable[..., Any] | None,
) -> tuple[dict[int, dict[str, str | None]], list[str], list[str]]:
if not vhag_codes:
return {}, [], []
base = settings.bathymetry_watercourse_layer_url.rstrip("/") + "/query"
names: dict[int, dict[str, str | None]] = {}
urls: list[str] = []
hashes: list[str] = []
ordered_codes = sorted(vhag_codes)
for start in range(0, len(ordered_codes), 100):
chunk = ordered_codes[start : start + 100]
offset = 0
while True:
url = BathymetryProfileAcquisitionService._query_url(
base,
{
"f": "json",
"where": f"\"wlasvl.vhag\" IN ({','.join(str(code) for code in chunk)})",
"outFields": "wlasvl.vhag,VHAG_TABEL.naam,VHAG_TABEL.namen",
"returnGeometry": "false",
"orderByFields": "wlasvl.vhag",
"resultOffset": str(offset),
"resultRecordCount": str(settings.bathymetry_profiles_page_size),
},
)
payload, response_sha = BathymetryProfileAcquisitionService._fetch_json(url, settings, opener)
urls.append(url)
hashes.append(response_sha)
page_features = payload.get("features")
if not isinstance(page_features, list):
raise AppError(
code="BATHYMETRY_PROVIDER_INVALID_RESPONSE",
message="VHA watercourse response does not contain a feature list",
status_code=502,
)
for feature in page_features:
attributes = feature.get("attributes") if isinstance(feature, dict) else None
if not isinstance(attributes, dict):
continue
raw_code = attributes.get("wlasvl.vhag")
if raw_code is None:
continue
code = int(raw_code)
if code not in names:
names[code] = {
"name": attributes.get("VHAG_TABEL.naam"),
"alternative_names": attributes.get("VHAG_TABEL.namen"),
}
if not payload.get("exceededTransferLimit") or not page_features:
break
offset += len(page_features)
return names, urls, hashes
@staticmethod
def _date_from_arcgis(value: Any) -> str | None:
if not isinstance(value, (int, float)) or not math.isfinite(float(value)):
return None
try:
return datetime.fromtimestamp(float(value) / 1000.0, tz=UTC).date().isoformat()
except (OverflowError, OSError, ValueError):
return None
@staticmethod
def _numeric_or_none(value: Any) -> float | None:
if value is None:
return None
try:
normalized = float(value)
except (TypeError, ValueError):
return None
return normalized if math.isfinite(normalized) else None
@staticmethod
def _document_url(value: Any) -> str | None:
if not isinstance(value, str) or not value.strip():
return None
normalized = value.strip()
if normalized.startswith("http://vha.waterinfo.be/"):
normalized = "https://" + normalized[len("http://") :]
return normalized if normalized.startswith("https://vha.waterinfo.be/") else None
@staticmethod
def _normalize_features(
raw_features: list[dict[str, Any]],
scope_geometry,
watercourse_names: dict[int, dict[str, str | None]],
) -> tuple[dict[str, Any], dict[str, Any]]:
normalized: list[dict[str, Any]] = []
dates: list[str] = []
watercourse_codes: set[int] = set()
document_count = 0
depth_count = 0
width_count = 0
for raw_feature in raw_features:
attributes = raw_feature.get("attributes")
geometry = raw_feature.get("geometry")
if not isinstance(attributes, dict) or not isinstance(geometry, dict):
continue
x = BathymetryProfileAcquisitionService._numeric_or_none(geometry.get("x"))
y = BathymetryProfileAcquisitionService._numeric_or_none(geometry.get("y"))
if x is None or y is None:
continue
point = Point(x, y)
if not scope_geometry.covers(point):
continue
raw_vhag = attributes.get("vhag")
vhag = int(raw_vhag) if isinstance(raw_vhag, (int, float)) else None
if vhag is not None:
watercourse_codes.add(vhag)
names = watercourse_names.get(vhag or -1, {})
measurement_date = BathymetryProfileAcquisitionService._date_from_arcgis(attributes.get("d_opmeti"))
if measurement_date:
dates.append(measurement_date)
document_url = BathymetryProfileAcquisitionService._document_url(attributes.get("hyperlink"))
depth = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_diepte"))
crown_width = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_kruinb"))
floor_width = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_vloerb"))
if document_url:
document_count += 1
if depth is not None:
depth_count += 1
if crown_width is not None or floor_width is not None:
width_count += 1
object_id = str(attributes.get("OBJECTID"))
normalized.append(
{
"type": "Feature",
"id": object_id,
"properties": {
"source_feature_id": object_id,
"provider_record_id": object_id,
"watercourse_vhag": vhag,
"watercourse_name": names.get("name") or (f"VHA-waterloop {vhag}" if vhag else "Onbekende waterloop"),
"watercourse_alternative_names": names.get("alternative_names"),
"profile_number": attributes.get("atlaspunt"),
"measurement_date": measurement_date,
"recorded_depth_m": depth,
"recorded_crown_width_m": crown_width,
"recorded_floor_width_m": floor_width,
"source_document_url": document_url,
"document_available": document_url is not None,
"structured_depth_available": depth is not None,
"source_code": attributes.get("bron"),
"structure_id": attributes.get("kunstwerkid"),
"provider": BathymetryProfileAcquisitionService.PROVIDER,
"measurement_semantics": "historical_cross_section_profile_point",
"vertical_reference": "document-specific",
},
"geometry": mapping(point),
}
)
return (
{
"type": "FeatureCollection",
"name": "vha_bathymetry_profiles",
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
"features": normalized,
},
{
"profile_count": len(normalized),
"document_count": document_count,
"structured_depth_count": depth_count,
"structured_width_count": width_count,
"watercourse_count": len(watercourse_codes),
"measurement_date_min": min(dates) if dates else None,
"measurement_date_max": max(dates) if dates else None,
},
)
@staticmethod
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
return (
db.query(Dataset)
.filter(
Dataset.project_id == project_id,
Dataset.name == filename,
Dataset.source_name == BathymetryProfileAcquisitionService.PROVIDER,
Dataset.status == "ready",
)
.order_by(Dataset.created_at.desc())
.first()
)
@staticmethod
def _result(dataset: Dataset, *, reused: bool) -> dict[str, Any]:
metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
return BathymetryProfileAcquisitionResult(
output_dataset_id=dataset.id,
reused=reused,
provider=BathymetryProfileAcquisitionService.PROVIDER,
profile_count=int(metadata.get("profile_count") or 0),
document_count=int(metadata.get("document_count") or 0),
structured_depth_count=int(metadata.get("structured_depth_count") or 0),
structured_width_count=int(metadata.get("structured_width_count") or 0),
watercourse_count=int(metadata.get("watercourse_count") or 0),
bbox_epsg4326=list(metadata.get("bbox_epsg4326") or []),
clipped_to_area_id=dataset.area_id,
measurement_date_min=metadata.get("measurement_date_min"),
measurement_date_max=metadata.get("measurement_date_max"),
attribution=BathymetryProfileAcquisitionService.ATTRIBUTION,
limitation_message=BathymetryProfileAcquisitionService.LIMITATION,
).model_dump(mode="json")
@staticmethod
def acquire(
db,
project_id: UUID,
payload: BathymetryProfileAcquireRequest,
*,
settings: Settings | None = None,
opener: Callable[..., Any] | None = None,
) -> dict[str, Any]:
resolved_settings = settings or get_settings()
if not resolved_settings.bathymetry_profiles_enabled:
raise AppError(
code="BATHYMETRY_NOT_CONFIGURED",
message="VHA bathymetry profile acquisition is disabled",
status_code=503,
)
bbox_values = BathymetryProfileAcquisitionService._validate_bbox(payload)
scope_geometry = BathymetryProfileAcquisitionService._scope_geometry(
db, project_id, payload.area_id, bbox_values
)
exact_bbox = tuple(float(value) for value in scope_geometry.bounds)
request_identity = {
"provider": BathymetryProfileAcquisitionService.PROVIDER,
"bbox_epsg4326": list(exact_bbox),
"area_id": str(payload.area_id) if payload.area_id else None,
"source_version": BathymetryProfileAcquisitionService.SOURCE_VERSION,
}
request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode()).hexdigest()
filename = f"vha_bathymetry_profiles_{request_hash[:12]}.geojson"
if not payload.force_refresh:
cached = BathymetryProfileAcquisitionService._cached_dataset(db, project_id, filename)
if cached is not None:
return BathymetryProfileAcquisitionService._result(cached, reused=True)
raw_features, fetch_provenance = BathymetryProfileAcquisitionService._fetch_profiles(
exact_bbox, resolved_settings, opener
)
vhag_codes = {
int(feature["attributes"]["vhag"])
for feature in raw_features
if isinstance(feature.get("attributes"), dict)
and isinstance(feature["attributes"].get("vhag"), (int, float))
}
names, name_urls, name_hashes = BathymetryProfileAcquisitionService._fetch_watercourse_names(
vhag_codes, resolved_settings, opener
)
feature_collection, summary = BathymetryProfileAcquisitionService._normalize_features(
raw_features, scope_geometry, names
)
if summary["profile_count"] == 0:
raise AppError(
code="BATHYMETRY_NO_PROFILES",
message="No VHA bathymetry profiles intersect the requested area",
status_code=404,
)
artifact = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
acquired_at = datetime.now(UTC)
source_metadata = {
"provider": BathymetryProfileAcquisitionService.PROVIDER,
"service": "ArcGIS MapServer",
"source_version": BathymetryProfileAcquisitionService.SOURCE_VERSION,
"theme": "bathymetry",
"layer_name": "bathymetry_profiles",
"coverage_scope": "municipality" if payload.area_id else "bounded_selection",
"bbox_epsg4326": list(exact_bbox),
**summary,
"selection_aggregation": {
"metric_key": "profile_count",
"method": "feature_count",
"label": "Dwarsprofielen",
"unit": "profielen",
},
"selection_metrics": [
{
"metric_key": "recorded_depth_mean_m",
"method": "mean",
"property": "recorded_depth_m",
"label": "Gemiddelde geregistreerde diepte",
"unit": "m",
"warning": "Alleen profielen met een gestructureerde dieptewaarde; meetdata kunnen verschillen.",
},
{
"metric_key": "recorded_depth_min_m",
"method": "min",
"property": "recorded_depth_m",
"label": "Kleinste geregistreerde diepte",
"unit": "m",
},
{
"metric_key": "recorded_depth_max_m",
"method": "max",
"property": "recorded_depth_m",
"label": "Grootste geregistreerde diepte",
"unit": "m",
},
{
"metric_key": "recorded_crown_width_mean_m",
"method": "mean",
"property": "recorded_crown_width_m",
"label": "Gemiddelde geregistreerde kruinbreedte",
"unit": "m",
},
{
"metric_key": "recorded_floor_width_mean_m",
"method": "mean",
"property": "recorded_floor_width_m",
"label": "Gemiddelde geregistreerde vloerbreedte",
"unit": "m",
},
],
"attribution": BathymetryProfileAcquisitionService.ATTRIBUTION,
"license_note": BathymetryProfileAcquisitionService.LICENSE_NOTE,
"limitation_message": BathymetryProfileAcquisitionService.LIMITATION,
"volume_supported": False,
}
dataset = DatasetService.import_vector_bytes(
db,
project_id=project_id,
area_id=payload.area_id,
filename=filename,
content=artifact,
source="VHA digitale atlas dwarsprofielen",
source_name=BathymetryProfileAcquisitionService.PROVIDER,
dataset_role="reference",
reference_layer_name="bathymetry_profiles",
source_version=BathymetryProfileAcquisitionService.SOURCE_VERSION,
source_metadata=source_metadata,
provenance_metadata={
"acquisition": "explicit_bounded_arcgis_feature_query",
"acquired_at": acquired_at.isoformat(),
"request_hash": request_hash,
"profile_query_urls": fetch_provenance["request_urls"],
"watercourse_name_query_urls": name_urls,
"response_sha256": [*fetch_provenance["response_sha256"], *name_hashes],
"artifact_sha256": hashlib.sha256(artifact).hexdigest(),
"candidate_count": fetch_provenance["candidate_count"],
"exact_profile_count": summary["profile_count"],
"clipped_to_area_id": str(payload.area_id) if payload.area_id else None,
"scope_geometry_type": scope_geometry.geom_type,
"vertical_reference": "document-specific",
"bathymetric_surface_available": False,
"water_surface_level_available": False,
"water_volume_available": False,
"limitation_message": BathymetryProfileAcquisitionService.LIMITATION,
},
)
persisted = db.get(Dataset, dataset.id)
return BathymetryProfileAcquisitionService._result(persisted, reused=False)
+120
View File
@@ -410,6 +410,126 @@ class DatasetService:
return DatasetService._to_response(dataset)
@staticmethod
def import_vector_bytes(
db: Session,
*,
project_id: UUID,
filename: str,
content: bytes,
source: str,
source_name: str,
dataset_role: str,
reference_layer_name: str | None,
source_metadata: dict[str, Any],
provenance_metadata: dict[str, Any],
area_id: UUID | None = None,
temporal_series_key: str | None = None,
observed_at: datetime | None = None,
valid_from: datetime | None = None,
valid_to: datetime | None = None,
temporal_granularity: str | None = None,
source_version: str | None = None,
content_type: str = "application/geo+json",
) -> DatasetCreateResponse:
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
if area_id is not None:
area = db.get(Area, area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
if not content:
raise AppError(code="INVALID_UPLOAD", message="Vector artifact is empty", status_code=400)
safe_filename = DatasetService._validate_upload_filename(filename)
if DatasetService._extension_for_path(safe_filename) not in DatasetService.VECTOR_EXTENSIONS:
raise AppError(code="INVALID_UPLOAD", message="Vector artifacts require .geojson or .json files", status_code=415)
normalized_role = DatasetService._normalize_dataset_role(dataset_role)
normalized_source_name = (source_name or "").strip() or ("manual" if normalized_role == "reference" else None)
temporal = DatasetService._validate_temporal_metadata(
temporal_series_key=temporal_series_key,
observed_at=observed_at,
valid_from=valid_from,
valid_to=valid_to,
temporal_granularity=temporal_granularity,
source_version=source_version,
)
try:
text = content.decode("utf-8")
except UnicodeDecodeError as exc:
raise AppError(code="INVALID_UPLOAD", message="Vector artifact must be UTF-8 encoded", status_code=400) from exc
try:
metadata = parse_geojson_payload(text)
vector_payload = json.loads(text)
except (ValueError, json.JSONDecodeError) as exc:
raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc
dataset_id = uuid.uuid4()
storage_info = StorageService.persist_dataset_file(
project_id=str(project_id),
dataset_id=str(dataset_id),
dataset_type="vector",
original_filename=safe_filename,
content=content,
content_type=content_type,
)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
area_id=area_id,
name=safe_filename,
dataset_type="vector",
source=source,
dataset_role=normalized_role,
source_name=normalized_source_name,
reference_layer_name=reference_layer_name if normalized_role == "reference" else None,
source_metadata=source_metadata,
provenance_metadata=provenance_metadata,
imported_at=datetime.now(timezone.utc),
**temporal,
storage_path=storage_info["storage_path"],
original_filename=storage_info["original_filename"],
stored_filename=storage_info["stored_filename"],
content_type=storage_info["content_type"],
size_bytes=storage_info["size_bytes"],
checksum_sha256=storage_info["checksum_sha256"],
crs=metadata.get("crs"),
bounds_json=metadata.get("bounds_json"),
metadata_json=metadata,
status="ready",
)
try:
db.add(dataset)
db.add(
DatasetVersion(
dataset_id=dataset.id,
version=1,
storage_path=dataset.storage_path,
source_version=dataset.source_version,
observed_at=dataset.observed_at,
valid_from=dataset.valid_from,
valid_to=dataset.valid_to,
checksum_sha256=dataset.checksum_sha256,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
)
)
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=dataset.id,
payload=vector_payload,
feature_class=reference_layer_name if normalized_role == "reference" else None,
commit=False,
)
db.commit()
db.refresh(dataset)
except Exception:
db.rollback()
StorageService.remove_dataset_file(storage_info["storage_path"])
raise
return DatasetService._to_response(dataset)
@staticmethod
def import_raster_bytes(
db: Session,
@@ -36,6 +36,8 @@ SEMANTIC_METRICS_DISABLED_OPERATOR_TOOLS = {
# line metrics (road/watercourse length) would therefore be meaningless.
"provision_regional_historical_landuse.py",
}
PROPERTY_AGGREGATION_METHODS = {"sum", "mean", "area_weighted_sum"}
PROPERTY_EXTREMA_METHODS = {"min", "max"}
PRECLIPPED_MUNICIPALITY_PARTITION_OPERATOR_TOOLS = {
"provision_regional_bwk_natura2000.py",
@@ -581,7 +583,7 @@ class VectorFeatureService:
length_m = db.query(func.coalesce(func.sum(length_expression), 0.0)).filter(*metric_filter).scalar()
divisor = 1_000.0 if unit == "km" else 1.0
metric_value = float(length_m or 0.0) / divisor
elif method in {"sum", "mean", "area_weighted_sum"}:
elif method in PROPERTY_AGGREGATION_METHODS | PROPERTY_EXTREMA_METHODS:
property_name = str(config.get("property") or "").strip()
if not property_name:
raise AppError(
@@ -599,7 +601,11 @@ class VectorFeatureService:
)
coverage_ratio = intersection_area / func.nullif(source_area, 0.0)
value_expression = numeric_value * coverage_ratio
aggregate_function = func.avg if method == "mean" else func.sum
aggregate_function = {
"mean": func.avg,
"min": func.min,
"max": func.max,
}.get(method, func.sum)
aggregate_value = (
db.query(func.coalesce(aggregate_function(value_expression), 0.0))
.filter(*metric_filter)
@@ -0,0 +1,355 @@
from __future__ import annotations
import json
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import parse_qs, urlparse
from uuid import uuid4
import pytest
from fastapi.testclient import TestClient
from geoalchemy2.shape import from_shape
from shapely.geometry import MultiPolygon, Polygon
from app.core.config import Settings
from app.core.errors import AppError
from app.db.session import get_db
from app.main import app
from app.models import Area, Dataset, Job, Project
from app.schemas.bathymetry import BathymetryProfileAcquireRequest
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
from app.services.dataset_service import DatasetService
ROOT = Path(__file__).resolve().parents[2]
class FakeQuery:
def __init__(self, result=None):
self.result = result
def filter(self, *_args):
return self
def order_by(self, *_args):
return self
def first(self):
return self.result
def all(self):
return self.result if isinstance(self.result, list) else []
class FakeSession:
def __init__(self, rows=None, query_result=None):
self.rows = rows or {}
self.query_result = query_result
self.added = []
def get(self, model, row_id):
row = self.rows.get((model, row_id))
if row is not None:
return row
return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
def add(self, row):
self.added.append(row)
def commit(self):
return None
def rollback(self):
return None
def refresh(self, row):
return row
def query(self, _model):
return FakeQuery(self.query_result)
class JsonResponse:
def __init__(self, payload):
self.content = json.dumps(payload).encode("utf-8")
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, size=-1):
return self.content if size < 0 else self.content[:size]
def request(*, area_id=None, force_refresh=True) -> BathymetryProfileAcquireRequest:
return BathymetryProfileAcquireRequest(
bbox={
"min_x": 5.0,
"min_y": 51.0,
"max_x": 6.0,
"max_y": 52.0,
"crs": "EPSG:4326",
},
area_id=area_id,
force_refresh=force_refresh,
)
def profile(object_id, vhag, x, y, *, depth=None, document=None, measured_at=951868800000):
return {
"attributes": {
"OBJECTID": object_id,
"vhag": vhag,
"atlaspunt": str(object_id),
"opg_kruinb": 4.5 if depth is not None else None,
"opg_vloerb": 1.2 if depth is not None else None,
"d_opmeti": measured_at,
"hyperlink": document,
"bron": 4,
"kunstwerkid": f"structure-{object_id}",
"opg_diepte": depth,
},
"geometry": {"x": x, "y": y},
}
def provider_opener(*, count=3):
profiles = [
profile(
1,
8506,
5.2,
51.2,
depth=1.8,
document="http://vha.waterinfo.be/download/dwarsprofielen/Molse_Nete/8506_DP_1.pdf",
),
profile(2, 8634, 5.8, 51.8, depth=2.4),
profile(3, 8506, 5.2, 51.8),
]
def opener(raw_request, timeout):
assert timeout == 120
url = raw_request.full_url
query = parse_qs(urlparse(url).query)
if query.get("returnCountOnly") == ["true"]:
return JsonResponse({"count": count})
if "MapServer/1/query" in url:
return JsonResponse(
{
"features": [
{
"attributes": {
"wlasvl.vhag": 8506,
"VHAG_TABEL.naam": "Molse Nete",
"VHAG_TABEL.namen": "Molse Nete - Mol Neet",
}
},
{
"attributes": {
"wlasvl.vhag": 8634,
"VHAG_TABEL.naam": "Scheppelijke Nete",
"VHAG_TABEL.namen": "Scheppelijke Nete - Stevensloop",
}
},
]
}
)
return JsonResponse({"features": profiles[:count]})
return opener
def test_bathymetry_source_registry_is_honest_and_nationally_extensible() -> None:
sources = BathymetryProfileAcquisitionService.list_sources()
by_key = {item["key"]: item for item in sources}
assert set(by_key) == {
"vha_inland_profiles",
"mdk_bcp_bathymetry",
"spw_walloon_waterway_bathymetry",
"port_antwerp_bathymetry",
}
assert by_key["vha_inland_profiles"]["integration_status"] == "operational"
assert by_key["vha_inland_profiles"]["acquisition_supported"] is True
assert by_key["mdk_bcp_bathymetry"]["vertical_reference"] == "LAT"
assert by_key["mdk_bcp_bathymetry"]["acquisition_supported"] is False
assert by_key["spw_walloon_waterway_bathymetry"]["vertical_reference"] == "mDNG"
assert by_key["spw_walloon_waterway_bathymetry"]["license_note"].startswith("CC BY 4.0")
def test_bathymetry_normalization_exactly_clips_area_and_preserves_evidence() -> None:
l_shape = Polygon(
[
(5.0, 51.0),
(6.0, 51.0),
(6.0, 51.4),
(5.4, 51.4),
(5.4, 52.0),
(5.0, 52.0),
(5.0, 51.0),
]
)
raw, _provenance = BathymetryProfileAcquisitionService._fetch_profiles(
(5.0, 51.0, 6.0, 52.0),
Settings(_env_file=None),
provider_opener(),
)
names, _urls, _hashes = BathymetryProfileAcquisitionService._fetch_watercourse_names(
{8506, 8634},
Settings(_env_file=None),
provider_opener(),
)
collection, summary = BathymetryProfileAcquisitionService._normalize_features(raw, l_shape, names)
assert summary == {
"profile_count": 2,
"document_count": 1,
"structured_depth_count": 1,
"structured_width_count": 1,
"watercourse_count": 1,
"measurement_date_min": "2000-03-01",
"measurement_date_max": "2000-03-01",
}
assert {feature["id"] for feature in collection["features"]} == {"1", "3"}
first = collection["features"][0]["properties"]
assert first["watercourse_name"] == "Molse Nete"
assert first["recorded_depth_m"] == 1.8
assert first["source_document_url"].startswith("https://vha.waterinfo.be/")
assert first["vertical_reference"] == "document-specific"
def test_bathymetry_acquisition_persists_reference_dataset_through_dataset_service(monkeypatch) -> None:
project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4()
l_shape = MultiPolygon(
[
Polygon(
[
(5.0, 51.0),
(6.0, 51.0),
(6.0, 51.4),
(5.4, 51.4),
(5.4, 52.0),
(5.0, 52.0),
(5.0, 51.0),
]
)
]
)
project = Project(id=project_id, name="Mol")
area = Area(
id=area_id,
project_id=project_id,
name="Gemeente Mol - officieel",
geometry=from_shape(l_shape, srid=4326),
)
db = FakeSession({(Project, project_id): project, (Area, area_id): area})
captured = {}
def persist(_db, **kwargs):
captured.update(kwargs)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
area_id=area_id,
name=kwargs["filename"],
dataset_type="vector",
source=kwargs["source"],
dataset_role=kwargs["dataset_role"],
source_name=kwargs["source_name"],
reference_layer_name=kwargs["reference_layer_name"],
source_metadata=kwargs["source_metadata"],
provenance_metadata=kwargs["provenance_metadata"],
status="ready",
)
db.rows[(Dataset, dataset_id)] = dataset
return SimpleNamespace(id=dataset_id)
monkeypatch.setattr(DatasetService, "import_vector_bytes", persist)
result = BathymetryProfileAcquisitionService.acquire(
db,
project_id,
request(area_id=area_id),
settings=Settings(_env_file=None),
opener=provider_opener(),
)
assert result["output_dataset_id"] == str(dataset_id)
assert result["profile_count"] == 2
assert captured["dataset_role"] == "reference"
assert captured["source_name"] == BathymetryProfileAcquisitionService.PROVIDER
assert captured["reference_layer_name"] == "bathymetry_profiles"
assert captured["source_metadata"]["theme"] == "bathymetry"
assert captured["source_metadata"]["volume_supported"] is False
assert captured["provenance_metadata"]["water_volume_available"] is False
payload = json.loads(captured["content"])
assert len(payload["features"]) == 2
def test_bathymetry_acquisition_rejects_unbounded_feature_volume() -> None:
settings = Settings(_env_file=None, BATHYMETRY_PROFILES_MAX_FEATURES=2)
with pytest.raises(AppError) as exc_info:
BathymetryProfileAcquisitionService._fetch_profiles(
(5.0, 51.0, 6.0, 52.0),
settings,
provider_opener(count=3),
)
assert exc_info.value.code == "BATHYMETRY_SCOPE_TOO_LARGE"
assert exc_info.value.details["candidate_count"] == 3
def test_bathymetry_source_api_uses_canonical_envelope() -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
app.dependency_overrides[get_db] = lambda: db
try:
response = TestClient(app).get(f"/api/v1/projects/{project_id}/datasets/bathymetry/sources")
finally:
app.dependency_overrides.clear()
assert response.status_code == 200
assert set(response.json()) == {"data"}
assert response.json()["data"]["total"] == 4
assert response.json()["data"]["items"][0]["integration_status"] == "operational"
def test_bathymetry_acquisition_route_stays_inside_existing_job_envelope(monkeypatch) -> None:
project_id, dataset_id = uuid4(), uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
monkeypatch.setattr(
BathymetryProfileAcquisitionService,
"acquire",
lambda *_args, **_kwargs: {
"output_dataset_id": str(dataset_id),
"provider": BathymetryProfileAcquisitionService.PROVIDER,
"profile_count": 2,
},
)
app.dependency_overrides[get_db] = lambda: db
try:
response = TestClient(app).post(
f"/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire",
json=request().model_dump(mode="json"),
)
finally:
app.dependency_overrides.clear()
assert response.status_code == 200
assert set(response.json()) == {"data"}
assert response.json()["data"]["job_type"] == "vector.bathymetry_profiles.acquire"
assert response.json()["data"]["output_dataset_id"] == str(dataset_id)
assert any(isinstance(item, Job) for item in db.added)
def test_bathymetry_contract_and_expansion_roadmap_are_documented() -> None:
api_contracts = (ROOT / "docs" / "API_CONTRACTS.md").read_text(encoding="utf-8")
roadmap = ROOT / "docs" / "BATHYMETRY_EXPANSION_ROADMAP.md"
assert "bathymetry/profiles/acquire" in api_contracts
assert roadmap.exists()
contents = roadmap.read_text(encoding="utf-8")
assert "LAT" in contents and "mDNG" in contents and "territoriale zee" in contents
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
assert "py_compile scripts/provision_mol_bathymetry_profiles.py" in readiness
assert "COPY scripts/provision_mol_bathymetry_profiles.py" in dockerfile
+1
View File
@@ -88,6 +88,7 @@ COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_hist
COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_regional_historical_landuse.py
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py
COPY scripts/provision_mol_bathymetry_profiles.py /app/scripts/provision_mol_bathymetry_profiles.py
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
+7
View File
@@ -54,6 +54,13 @@ services:
FLOOD_HAZARD_MAX_PIXELS: ${FLOOD_HAZARD_MAX_PIXELS:-12000000}
FLOOD_HAZARD_TIMEOUT_SECONDS: ${FLOOD_HAZARD_TIMEOUT_SECONDS:-300}
FLOOD_HAZARD_MAX_RESPONSE_MB: ${FLOOD_HAZARD_MAX_RESPONSE_MB:-160}
BATHYMETRY_PROFILES_ENABLED: ${BATHYMETRY_PROFILES_ENABLED:-true}
BATHYMETRY_PROFILES_LAYER_URL: ${BATHYMETRY_PROFILES_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0}
BATHYMETRY_WATERCOURSE_LAYER_URL: ${BATHYMETRY_WATERCOURSE_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1}
BATHYMETRY_PROFILES_PAGE_SIZE: ${BATHYMETRY_PROFILES_PAGE_SIZE:-1000}
BATHYMETRY_PROFILES_MAX_FEATURES: ${BATHYMETRY_PROFILES_MAX_FEATURES:-50000}
BATHYMETRY_PROFILES_TIMEOUT_SECONDS: ${BATHYMETRY_PROFILES_TIMEOUT_SECONDS:-120}
BATHYMETRY_PROFILES_MAX_RESPONSE_MB: ${BATHYMETRY_PROFILES_MAX_RESPONSE_MB:-32}
THEMATIC_RASTER_ENABLED: ${THEMATIC_RASTER_ENABLED:-true}
THEMATIC_RASTER_WCS_URL: ${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}
THEMATIC_RASTER_MIN_SIDE_M: ${THEMATIC_RASTER_MIN_SIDE_M:-100}
+34
View File
@@ -2055,3 +2055,37 @@ specifically, no water volume is inferred from 2D water geometry. The backend
sets an explicit Ollama context window and returns
`OLLAMA_RESPONSE_TRUNCATED` instead of accepting a response with
`done_reason=length` as a complete answer.
## Bathymetry and VHA profile acquisition
### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources`
Returns the governed bathymetry source registry in the canonical envelope.
VHA inland profiles are `operational`. MDK Belgian Continental Shelf and SPW
Walloon bathymetry remain `available_not_integrated`; they cannot be acquired
through this contract until their raster/download and vertical-datum flows
pass live validation.
### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire`
```json
{
"bbox": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.3, "max_y": 51.3, "crs": "EPSG:4326"},
"area_id": "optional persisted Area uuid",
"force_refresh": false
}
```
The endpoint creates a synchronous `vector.bathymetry_profiles.acquire` Job.
It queries the official VHA Digital Atlas in bounded pages, intersects every
point with the persisted Area when supplied and persists one ordinary
reference Dataset plus VectorFeature rows. The result reports exact profile,
document, structured-depth/width and watercourse counts and the source
measurement-date range.
The acquired Dataset uses
`source_name=vmm_vha_bathymetry_profiles`,
`dataset_role=reference` and
`reference_layer_name=bathymetry_profiles`. Existing vector content and
selection endpoints provide GeoJSON and metrics. The contract does not expose
a continuous bathymetric surface, current water depth or water volume.
+131
View File
@@ -0,0 +1,131 @@
# Bathymetry expansion roadmap
## Purpose
GeoIntel must distinguish three different questions:
1. Where were cross-sections measured and what does the source document say?
2. What is the continuous elevation of the bed at a specific survey epoch?
3. What is the water depth or volume at a specific moment?
Only the first question is operational for Mol through the VHA cross-section
profile layer. A bed model does not provide water depth without a compatible
water-surface elevation. Flood-hazard maximum depth is a scenario result and
must not be reused as current water level.
## Governed source matrix
| Source | Coverage | Data | Vertical reference | GeoIntel status |
| --- | --- | --- | --- | --- |
| VMM VHA Digital Atlas | Flanders | Point locations, structured profile fields, PDF evidence | Document-specific | Operational, bounded vector acquisition |
| MDK Belgian Continental Shelf model | Belgian North Sea | Continuous 20 x 20 m bathymetric raster | LAT | Available, not integrated |
| SPW navigable waterways and reservoir lakes | Wallonia | 0.5 m bed-elevation raster and XYZ cloud | mDNG | Available, not integrated |
| Port of Antwerp-Bruges publications | Port survey areas | Periodic soundings | Product-specific | Catalog candidate |
VHA contains approximately 129,643 profile points across Flanders at the
observed catalog state. This is a scale indication, not a fixed contractual
count. Mol contained 828 exact in-boundary points during source validation,
715 with document links and 112 with a structured depth field. Production
provisioning always records the live counts and checksums.
## Operational tier 1: Mol
- Query only an explicit EPSG:4326 bbox.
- Intersect the bbox with the exact persisted Mol Area.
- Page the official ArcGIS FeatureServer response without truncation.
- Resolve VHA watercourse names from the official atlas layer.
- Normalize profile points to EPSG:4326.
- Persist the artifact through `DatasetService.import_vector_bytes`.
- Persist every point through `VectorFeatureService`; the provider never
writes directly to `vector_features`.
- Expose document, measurement date, depth and width fields without parsing or
inventing values from scanned PDFs.
- Keep volume unsupported.
## Tier 2: all of Flanders
Flanders must be provisioned as exact municipality or other approved Area
partitions, not as one monolithic request. The backend feature limit protects
the provider and the application. A regional logical layer may group complete
partitions, but each Dataset retains its Area id, query URLs, checksums, exact
count and measurement-date range.
Before regional activation:
- add a governed Flanders boundary manifest and partition coordinator;
- prove idempotent resume and no duplicate VHA `OBJECTID` within a partition;
- benchmark PostGIS point selection and viewport delivery;
- add freshness/version probing for the VHA MapServer;
- keep individual profile dates instead of fabricating one Dataset
`observed_at`.
## Tier 3: Belgium
Belgian coverage is a federation of source adapters with one normalized
contract, not one assumed national dataset:
- Flanders: VHA profiles and future validated bed rasters;
- Wallonia: SPW bathymetry for measured navigable waterways/reservoirs;
- Brussels: hydrological context until an authoritative public bathymetric
product is identified;
- federal/maritime: MDK and legally appropriate maritime boundaries.
Every adapter must emit:
- authority and owner;
- exact geographic and temporal coverage;
- horizontal and vertical CRS/datum;
- survey/acquisition time;
- resolution or sample density;
- source URL, request identity, checksum, attribution and license;
- explicit supported and unsupported metrics.
LAT, TAW and mDNG values must never be merged or compared without a documented,
tested vertical transformation and uncertainty statement.
## Tier 4: the Belgian North Sea
The map must distinguish:
- the Belgian land boundary and baseline;
- the territoriale zee (up to 12 nautical miles);
- the Belgian EEZ and continental shelf, which are jurisdictional maritime
zones and should not be labelled ordinary municipal or provincial
"grondgebied".
The MDK bathymetry WCS is the preferred continuous source candidate. Activation
requires a live Docker validation of TLS/certificates, GetCapabilities,
coverage identifiers, bounded GeoTIFF retrieval, CRS, LAT, nodata, pixel size,
response limits and maritime clipping. WMTS can support visual context but is
not the analytical source.
## Depth and volume rules
For a compatible bed raster and water-surface raster at the same time and
vertical datum:
`volume_m3 = sum(max(0, water_surface_z - bed_z) * cell_area_m2)`
For surveyed cross-sections along a connected reach:
`volume_m3 = sum(((section_area_i + section_area_i+1) / 2) * reach_length_i)`
The second method requires complete profile geometry, ordered chainage,
contemporaneous water level and defensible interpolation. VHA profile points
alone do not satisfy those prerequisites.
Historical evolution compares only survey epochs with documented compatible
coverage, datum and method. A changed raster footprint is not automatically
bed evolution.
## Implementation order
1. Operate and validate the Mol VHA profile Dataset and map flow.
2. Add VHA municipal partition orchestration for Flanders.
3. Implement a bounded MDK WCS probe, then acquisition behind live evidence.
4. Implement SPW download staging and vertical-datum metadata validation.
5. Add maritime boundaries as separate authoritative scope layers.
6. Add cross-source vertical-datum transformation only with authoritative
grids/parameters and uncertainty tests.
7. Add volume only after a compatible measured or modeled water-surface source
is part of the same analysis contract.
+24
View File
@@ -10010,3 +10010,27 @@ Validation:
stays within its 375 px client width; the theme inventory is bounded to a
compact scrollable selector so it no longer pushes the map behind all 15
theme cards.
## Sprint 235 - Governed bathymetry profiles and Belgian expansion model (2026-07-17)
Implemented:
- Audited official VHA, MDK Belgian Continental Shelf and SPW Walloon
bathymetry services and separated profile evidence, continuous bed models
and time-specific water depth/volume.
- Added bounded, paged VHA profile acquisition with exact persisted-Area
clipping, official watercourse names, document links, checksums and standard
Dataset/VectorFeature persistence.
- Added reusable vector-byte import and governed `min`/`max` property metrics
to the existing selection aggregation path.
- Added the bathymetry source and acquisition API, environment controls, Mol
operator command, map theme, profile inspector and source inventory.
- Documented partitioned Flanders scaling and federated Belgium/maritime
scaling with explicit territorial sea, EEZ/continental shelf and
TAW/LAT/mDNG semantics.
Validation:
- Backend compilation, 914 backend tests, documentation/contract audits,
Alembic head `202607160001`, frontend TypeScript typecheck and the production
Vite build passed in the complete readiness gate.
- Full readiness and live Mol acceptance are recorded after final validation
and deployment below.
+15
View File
@@ -360,3 +360,18 @@ the composite expression index
It supports exact preclipped municipality selection without changing the
canonical `vector_features` schema or introducing operator-specific tables.
Free rectangle queries continue to use the geometry GiST index.
## Bathymetry profile persistence
VHA profile points require no new table or migration. The immutable GeoJSON
artifact is stored as one normal reference `datasets` row and
`dataset_versions` row; each exact point is a normal `vector_features` row with
EPSG:4326 geometry and source properties. Existing dataset, source-feature and
GiST indexes support selection.
Profile measurement dates remain feature properties because a bounded Dataset
can contain many historical campaigns. No artificial Dataset `observed_at` or
temporal series is assigned. A future bed raster remains file/object storage
plus Dataset metadata, not raster-in-database storage. Any national/maritime
extension keeps source vertical datum, survey epoch and Area partition
explicit and does not create a provider-specific shadow schema.
+23
View File
@@ -709,3 +709,26 @@ thematic rasters for space occupation, open space, population density, node
value and service level. The digital soil map should follow the existing
canonical vector persistence path. Watercourse/runoff additions remain in the
roadmap but no longer precede these cross-domain gaps.
## Bathymetry, inland profiles and maritime scope
The official VHA Digital Atlas profile-point layer is the first operational
bathymetry-adjacent source. GeoIntel requests an explicit bbox, clips against
the exact persisted Area and retains VHA point identifiers, watercourse names,
profile numbers, measurement dates, available structured depth/width values
and official document URLs. Scanned documents remain evidence; missing fields
are not filled by fabricated OCR output.
The following sources are audited but not yet operational:
- MDK Dieptemodel Belgisch Continentaal Plat/Noordzee: 20 x 20 m continuous
raster in LAT, exposed through WCS/WMTS.
- SPW bathymetry of navigable waterways and reservoir lakes: 0.5 m bed
elevation and XYZ data in mDNG.
- Port of Antwerp-Bruges periodic soundings: catalog candidate pending a
stable public machine contract.
See `docs/BATHYMETRY_EXPANSION_ROADMAP.md`. TAW, LAT and mDNG remain separate
until an authoritative vertical transformation is implemented and tested.
The territorial sea, EEZ and continental shelf are separate scope layers and
must be labelled according to their legal meaning.
+19
View File
@@ -438,3 +438,22 @@ Sprint 7B makes provider architecture operationally visible without performing l
- status: `configured`
- dataset mapping: `dataset_role=reference`, `source_name=fixture`
- write path: checked-in demo/test fixture flow
## Bathymetry profile vector contract
The operational VHA layer is an EPSG:4326 point FeatureCollection. Required
normalized properties are:
- `provider_record_id` and `source_feature_id`;
- `watercourse_vhag`, `watercourse_name` and alternative names;
- `profile_number` and `measurement_date`;
- nullable `recorded_depth_m`, `recorded_crown_width_m` and
`recorded_floor_width_m`;
- nullable allowlisted `source_document_url`;
- `document_available`, `structured_depth_available`;
- `measurement_semantics=historical_cross_section_profile_point`;
- `vertical_reference=document-specific`.
Null means the provider did not expose a structured value. It is never
converted to zero. Dataset metadata records exact counts, measurement range,
scope, attribution and `volume_supported=false`.
+15
View File
@@ -749,3 +749,18 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Keep orthophoto pixel refresh manual through a separate
plan-stage-review-apply flow that retains the passed preflight identity and
creates a new immutable Dataset with official `YYYY.NN` source version.
# Bathymetry follow-up
- [ ] Add governed municipality partition orchestration for all of Flanders
after the Mol VHA operator passes live acceptance.
- [ ] Add read-only MDK WCS capability/TLS probe and maritime scope metadata.
- [ ] Add bounded MDK GeoTIFF acquisition only after live CRS, LAT, nodata and
response-limit validation.
- [ ] Add SPW staged-download adapter with mDNG metadata and survey-epoch
coverage validation.
- [ ] Add authoritative territorial-sea, EEZ and continental-shelf boundary
layers with legally accurate labels.
- [ ] Add vertical-datum conversion only when authoritative transforms and
uncertainty tests exist; never merge TAW, LAT and mDNG implicitly.
- [ ] Add water volume only when bed and water-surface inputs share a governed
time, datum and coverage contract.
+13
View File
@@ -635,3 +635,16 @@ validation remain inside the collapsed follow-up list with an explicit
priority and examples of the measurements they should eventually support.
Theme/time-series detail and active-source limitations are also collapsed by
default so the Data workspace stays readable.
## Bathymetry profile map theme
Ready `vmm_vha_bathymetry_profiles` Datasets activate the `Waterbodem` map
theme. The layer renders persisted profile points through the existing
MapLibre/GeoJSON path. Clicking a point shows the watercourse, profile number,
measurement date, nullable registered depth and an allowlisted official VHA
profile-document link.
The UI always labels the points as historical cross-sections. It does not
present them as a continuous bed raster and does not calculate water volume.
The Sources workspace lists the future MDK North Sea and SPW Walloon
bathymetry integrations as planned until backend acquisition is operational.
@@ -19,6 +19,7 @@ const THEME_LABELS: Record<string, string> = {
nature_value: 'Natuurwaarde',
agriculture: 'Landbouw',
water: 'Water',
bathymetry: 'Waterbodem en dwarsprofielen',
flood_hazard: 'Overstromingsgevaar',
elevation: 'Hoogte en reliëf',
roads: 'Wegen en transport',
@@ -81,6 +82,9 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
)
const dhmvDatasets = ready.filter((dataset) => dataset.source_name === 'digitaal_vlaanderen_dhmv')
const floodHazardDatasets = ready.filter((dataset) => dataset.source_name === 'vmm_flood_hazard')
const bathymetryProfileDatasets = ready.filter(
(dataset) => dataset.source_name === 'vmm_vha_bathymetry_profiles',
)
const latestBuildingsRegister = [...buildingsRegisterDatasets].sort(
(left, right) => new Date(right.observed_at ?? 0).getTime() - new Date(left.observed_at ?? 0).getTime(),
)[0]
@@ -184,7 +188,7 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
</div>
</details>
{waterinfoDatasets.length > 0 || historicalOrthophotos.length > 0 || bwkDatasets.length > 0 || agricultureDatasets.length > 0 || buildingsRegisterDatasets.length > 0 || dhmvDatasets.length > 0 || floodHazardDatasets.length > 0 ? (
{waterinfoDatasets.length > 0 || historicalOrthophotos.length > 0 || bwkDatasets.length > 0 || agricultureDatasets.length > 0 || buildingsRegisterDatasets.length > 0 || dhmvDatasets.length > 0 || floodHazardDatasets.length > 0 || bathymetryProfileDatasets.length > 0 ? (
<details className="source-loaded-disclosure">
<summary>Actieve broncollecties en hun beperkingen</summary>
<div className="source-catalog-loaded" aria-label="Aanvullende ingeladen bronnen">
@@ -244,6 +248,18 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
<p>Gemodelleerde maximumdiepte per kansscenario. Geen actuele waterstand, bathymetrie of permanent watervolume.</p>
</article>
) : null}
{bathymetryProfileDatasets.length > 0 ? (
<article>
<strong>VHA-dwarsprofielen waterbodem</strong>
<span>
{bathymetryProfileDatasets.reduce(
(total, dataset) => total + Number(dataset.feature_count ?? 0),
0,
).toLocaleString('nl-BE')} profielpunten · historische meetcampagnes
</span>
<p>Meetvelden en officiële profielbladen per punt. Geen continue actuele bathymetrie of volume zonder een gelijktijdig waterpeil.</p>
</article>
) : null}
</div>
</details>
) : null}
+59 -1
View File
@@ -36,7 +36,7 @@ const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson'
const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson'
const EMPTY_TEMPORAL_SERIES: DatasetCreateResponse[] = []
type DataThemeId = 'buildings' | 'space_occupation' | 'open_space' | 'population' | 'forest' | 'nature_value' | 'agriculture' | 'soil' | 'water' | 'flood_hazard' | 'elevation' | 'accessibility' | 'services' | 'roads' | 'parcels'
type DataThemeId = 'buildings' | 'space_occupation' | 'open_space' | 'population' | 'forest' | 'nature_value' | 'agriculture' | 'soil' | 'water' | 'bathymetry' | 'flood_hazard' | 'elevation' | 'accessibility' | 'services' | 'roads' | 'parcels'
interface DataTheme {
id: DataThemeId
@@ -116,6 +116,13 @@ const DATA_THEMES: DataTheme[] = [
description: 'Waterlopen, grachten, kanalen en wateroppervlakken.',
tokens: ['waterways', 'waterway', 'water', 'hydro', 'river', 'stream', 'canal', 'waterloop'],
},
{
id: 'bathymetry',
label: 'Waterbodem',
shortLabel: 'Dwarsprofielen',
description: 'Officiële historische VHA-dwarsprofielen met meetvelden en brondocumenten.',
tokens: ['bathymetry', 'bathymetry_profiles', 'dwarsprofielen', 'waterbodem'],
},
{
id: 'flood_hazard',
label: 'Overstroming',
@@ -170,6 +177,7 @@ const DATA_THEME_MAP_STYLES: Record<DataThemeId, { fill: string; line: string }>
agriculture: { fill: '#7b8f32', line: '#53671d' },
soil: { fill: '#9a7040', line: '#6f4c27' },
water: { fill: '#2676a8', line: '#155b85' },
bathymetry: { fill: '#0e7490', line: '#164e63' },
flood_hazard: { fill: '#1597c2', line: '#075985' },
elevation: { fill: '#a57a4b', line: '#315f59' },
accessibility: { fill: '#0f766e', line: '#115e59' },
@@ -188,6 +196,11 @@ function datasetAvailabilityLabel(dataset: DatasetCreateResponse, partitionCount
const resolution = Number(dataset.source_metadata?.['analysis_resolution_m'])
return `${Number.isFinite(resolution) ? `${resolution.toLocaleString('nl-BE')} m` : 'Raster'} overstromingsscenario${regionalSuffix}`
}
if (dataset.source_name === 'vmm_vha_bathymetry_profiles') {
const profiles = dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0
const documents = Number(dataset.source_metadata?.['document_count'] ?? 0)
return `${profiles.toLocaleString('nl-BE')} profielen · ${documents.toLocaleString('nl-BE')} bronbladen`
}
if (dataset.dataset_type === 'raster' && dataset.source_name === 'department_omgeving_thematic_raster') {
const resolution = Number(dataset.source_metadata?.['analysis_resolution_m'])
const year = Number(dataset.source_metadata?.['observation_year'])
@@ -221,6 +234,9 @@ function datasetMatchesTheme(dataset: DatasetCreateResponse, theme: DataTheme):
if (dataset.source_name === 'vmm_flood_hazard') {
return theme.id === 'flood_hazard'
}
if (dataset.source_name === 'vmm_vha_bathymetry_profiles') {
return theme.id === 'bathymetry'
}
if (dataset.source_name === 'digitaal_vlaanderen_dhmv') {
return theme.id === 'elevation'
}
@@ -310,6 +326,7 @@ function pickThemeDataset(
(dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register' ? 5_000_000 : 0) +
(dataset.source_name === 'digitaal_vlaanderen_dhmv' ? 5_000_000 : 0) +
(dataset.source_name === 'vmm_flood_hazard' ? 5_000_000 : 0) +
(dataset.source_name === 'vmm_vha_bathymetry_profiles' ? 5_000_000 : 0) +
(dataset.source_metadata?.['product_key'] === 'dtm_1m' ? 1_000_000 : 0) +
(dataset.source_metadata?.['product_key'] === 'pluviaal_current_t100' ? 1_000_000 : 0) +
(dataset.dataset_role === 'reference' ? 10_000 : 0)
@@ -618,6 +635,12 @@ export function MapWorkspace({
const selectedMapArea = areas.find((area) => area.id === selectedMapAreaId)
const regionalScopeSelected = Boolean(selectedMapArea && !isMunicipalityAreaName(selectedMapArea.name))
const featureProperties = selectedMapFeature?.properties ?? null
const isBathymetryProfile = featureProperties?.['provider'] === 'vmm_vha_bathymetry_profiles'
|| featureProperties?.['measurement_semantics'] === 'historical_cross_section_profile_point'
const bathymetryDocumentUrl = typeof featureProperties?.['source_document_url'] === 'string'
&& featureProperties['source_document_url'].startsWith('https://vha.waterinfo.be/')
? featureProperties['source_document_url']
: null
const featureSummaryEntries = featureProperties
? Object.entries(featureProperties)
.filter(([, value]) => value !== null && value !== undefined && typeof value !== 'object')
@@ -2555,6 +2578,41 @@ export function MapWorkspace({
</div>
{selectedMapFeature ? (
<>
{isBathymetryProfile ? (
<div className="bathymetry-profile-summary" aria-label="Samenvatting VHA-dwarsprofiel">
<div>
<span>Waterloop</span>
<strong>{String(featureProperties?.['watercourse_name'] ?? 'Onbekende waterloop')}</strong>
</div>
<div>
<span>Profiel</span>
<strong>{String(featureProperties?.['profile_number'] ?? 'n.v.t.')}</strong>
</div>
<div>
<span>Meetdatum</span>
<strong>{String(featureProperties?.['measurement_date'] ?? 'Niet geregistreerd')}</strong>
</div>
<div>
<span>Geregistreerde diepte</span>
<strong>
{typeof featureProperties?.['recorded_depth_m'] === 'number'
? `${featureProperties['recorded_depth_m'].toLocaleString('nl-BE')} m`
: 'Niet als veld beschikbaar'}
</strong>
</div>
{bathymetryDocumentUrl ? (
<a href={bathymetryDocumentUrl} target="_blank" rel="noreferrer">
Officieel profielblad openen
</a>
) : (
<small>Voor dit meetpunt is geen digitaal profielblad gekoppeld.</small>
)}
<p>
Historisch dwarsprofiel. Dit punt is geen continue actuele bodemkaart en levert zonder
gelijktijdig waterpeil geen actueel watervolume.
</p>
</div>
) : null}
<div className="feature-extract-grid" aria-label="Geometrie van het geselecteerde object">
<div>
<span>Geometrie</span>
+5
View File
@@ -4,6 +4,7 @@ const DATASET_LABEL_BY_LAYER: Record<string, string> = {
buildings: 'Bebouwing',
roads: 'Wegen',
water: 'Water',
bathymetry_profiles: 'Dwarsprofielen waterbodem',
parcels: 'Percelen',
population: 'Bevolking',
forest: 'Bos en groen',
@@ -34,6 +35,7 @@ const DATASET_SOURCE_LABELS: Record<string, string> = {
vrbg: 'Digitaal Vlaanderen',
waterinfo: 'Waterinfo Vlaanderen',
vmm_flood_hazard: 'Vlaamse Milieumaatschappij',
vmm_vha_bathymetry_profiles: 'VMM / Vlaamse Hydrografische Atlas',
department_omgeving_thematic_raster: 'Departement Omgeving',
dov_soil_map: 'Databank Ondergrond Vlaanderen',
}
@@ -52,6 +54,9 @@ export function getDatasetDisplayName(dataset: DatasetCreateResponse): string {
const productName = dataset.source_metadata?.['product_display_name']
return typeof productName === 'string' && productName.trim() ? productName : 'VMM-overstromingsscenario'
}
if (dataset.source_name === 'vmm_vha_bathymetry_profiles') {
return 'VHA-dwarsprofielen waterbodem'
}
if (dataset.source_name === 'department_omgeving_thematic_raster') {
const productName = dataset.source_metadata?.['product_display_name']
return typeof productName === 'string' && productName.trim() ? productName : 'Officieel Vlaams themaraster'
+36
View File
@@ -384,6 +384,42 @@ export const OFFICIAL_SOURCE_PORTFOLIO: OfficialSourceDefinition[] = [
url: 'https://www.vlaanderen.be/datavindplaats/catalogus/ogc-api-features-vlaamse-hydrografische-atlas-waterlopen',
matches: (dataset) => sourceNameIs(dataset, 'vmm_vha'),
},
{
key: 'vha_bathymetry_profiles',
domain: 'climate',
name: 'VHA-dwarsprofielen waterbodem',
owner: 'Vlaamse Milieumaatschappij',
coverage: 'Historische profielpunten op Vlaamse waterlopen',
value: 'Gemeten profielinformatie en officiële bronbladen op exacte kaartlocaties.',
metricExamples: 'aantal profielen, geregistreerde diepte en breedte per meetpunt',
priority: 'next',
url: 'https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer',
matches: (dataset) => sourceNameIs(dataset, 'vmm_vha_bathymetry_profiles'),
},
{
key: 'mdk_bcp_bathymetry',
domain: 'climate',
name: 'Dieptemodel Belgisch Continentaal Plat',
owner: 'Agentschap Maritieme Dienstverlening en Kust',
coverage: 'Belgische Noordzee, 20 x 20 m raster in LAT',
value: 'Continue zeebodemhoogte voor mariene selectie en evolutie.',
metricExamples: 'minimum/maximum diepte, diepteklassen en bodemvolume per peilcampagne',
priority: 'planned',
url: 'https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee',
matches: (dataset) => sourceNameIs(dataset, 'mdk_bcp_bathymetry'),
},
{
key: 'spw_waterway_bathymetry',
domain: 'climate',
name: 'Waalse vaarweg- en stuwmeerbathymetrie',
owner: 'Service public de Wallonie',
coverage: 'Gemeten Waalse vaarwegen en stuwmeren, 0,5 m in mDNG',
value: 'Hoogwaardige bodemrasters voor federale uitbreiding buiten Vlaanderen.',
metricExamples: 'bodemhoogte, diepteprofiel en vergelijkbare meetcampagnes',
priority: 'planned',
url: 'https://geoportail.wallonie.be/catalogue/c450c28f-d357-48af-8423-62d524632cf9.html',
matches: (dataset) => sourceNameIs(dataset, 'spw_walloon_waterway_bathymetry'),
},
]
export function operationalSources(
+8
View File
@@ -22,6 +22,8 @@ import type {
FloodHazardAcquireRequest,
FloodHazardProductRead,
FloodHazardSelectionResponse,
BathymetryProfileAcquireRequest,
BathymetrySourceRead,
DhmvProductRead,
TerrainSelectionResponse,
ThematicRasterAcquireRequest,
@@ -167,6 +169,12 @@ export const datasetsApi = {
payload: { bbox: VectorSelectionRequest['bbox']; area_id?: string; product_key: string },
): Promise<FloodHazardSelectionResponse> =>
apiPost<FloodHazardSelectionResponse>(`/api/v1/projects/${projectId}/datasets/raster/flood-hazard/select`, payload),
listBathymetrySources: (projectId: string): Promise<{ items: BathymetrySourceRead[]; total: number }> =>
apiGet<{ items: BathymetrySourceRead[]; total: number }>(
`/api/v1/projects/${projectId}/datasets/bathymetry/sources`,
),
acquireBathymetryProfiles: (projectId: string, payload: BathymetryProfileAcquireRequest): Promise<JobRead> =>
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/bathymetry/profiles/acquire`, payload),
acquireThematicRaster: (projectId: string, payload: ThematicRasterAcquireRequest): Promise<JobRead> =>
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/thematic-raster/acquire`, payload),
listThematicRasterProducts: (projectId: string): Promise<{ items: ThematicRasterProductRead[]; total: number }> =>
+59
View File
@@ -5966,6 +5966,11 @@ section {
background: rgba(38, 118, 168, 0.24);
}
.geo-map-legend .geo-legend-layer-bathymetry {
background: #0e7490;
border-color: #164e63;
}
.geo-map-legend .geo-legend-layer-soil {
border-color: #6f4c27;
background: rgba(154, 112, 64, 0.24);
@@ -6036,6 +6041,60 @@ section {
.geo-legend-ramp-population { background: linear-gradient(90deg, #eee7f6, #673a97); }
.geo-legend-ramp-accessibility { background: linear-gradient(90deg, #e9f1f4, #0f766e); }
.geo-legend-ramp-services { background: linear-gradient(90deg, #fff4bf, #b66d16); }
.geo-legend-ramp-bathymetry { background: linear-gradient(90deg, #cffafe, #0e7490); }
.bathymetry-profile-summary {
display: grid;
grid-template-columns: repeat(4, minmax(0, 1fr));
gap: 0.65rem;
margin-bottom: 0.9rem;
padding: 0.8rem;
border: 1px solid #a5d8e6;
border-radius: 6px;
background: #effbff;
}
.bathymetry-profile-summary > div {
display: grid;
gap: 0.2rem;
min-width: 0;
}
.bathymetry-profile-summary span,
.bathymetry-profile-summary small {
color: #46646f;
font-size: 0.75rem;
}
.bathymetry-profile-summary strong {
overflow-wrap: anywhere;
color: #113d4b;
}
.bathymetry-profile-summary a,
.bathymetry-profile-summary p,
.bathymetry-profile-summary small {
grid-column: 1 / -1;
}
.bathymetry-profile-summary a {
width: fit-content;
color: #075985;
font-weight: 700;
}
.bathymetry-profile-summary p {
margin: 0;
color: #315867;
font-size: 0.78rem;
line-height: 1.45;
}
@media (max-width: 760px) {
.bathymetry-profile-summary {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
}
.geo-map-legend .geo-legend-added {
border-color: #15803d;
+27
View File
@@ -442,6 +442,33 @@ export interface FloodHazardSelectionResponse {
generated_at: string
}
export interface BathymetryProfileAcquireRequest {
bbox: VectorSelectionBBox
area_id?: string | null
force_refresh?: boolean
}
export interface BathymetrySourceRead {
key: string
display_name: string
owner: string
authority_level: 'authoritative' | 'contextual'
geographic_coverage: string
data_kind: string
query_modes: string[]
vertical_reference: string
horizontal_crs: string
native_resolution?: string | null
integration_status: 'operational' | 'available_not_integrated' | 'catalog_only'
acquisition_supported: boolean
configured: boolean
service_url?: string | null
catalog_url: string
attribution: string
license_note: string
limitation_message: string
}
export interface ThematicRasterAcquireRequest {
bbox: VectorSelectionBBox
area_id?: string | null
+27
View File
@@ -1882,3 +1882,30 @@ boundary artifact. Defaults cap each municipality at 30,000 source features and
the assembled snapshot at 300,000 features. It never truncates silently, never
writes `vector_features` directly and never turns the single 2025 state into a
fabricated historical series.
## Mol VHA bathymetry profiles
Provision and verify the official profile points for the exact persisted Mol
Area:
```bash
docker exec geointel python /app/scripts/provision_mol_bathymetry_profiles.py
```
Use `--force` only for a deliberate fresh provider snapshot. The command finds
`Mol Municipality Workbench` and `Gemeente Mol`, calls the canonical
bathymetry acquisition endpoint and then verifies that vector selection count
and semantic metrics match the persisted Job result. It never writes directly
to PostGIS.
For another approved workspace or Area:
```bash
python scripts/provision_mol_bathymetry_profiles.py \
--base-url http://127.0.0.1:8000 \
--project-name "Project name" \
--area-name "Area name fragment"
```
The output is a point dataset with historical evidence. It is not a continuous
water-bottom raster and cannot calculate current water volume.
@@ -0,0 +1,154 @@
"""Provision official VHA cross-section profile points for the persisted Mol Area.
The operator uses only canonical GeoIntel API endpoints. The backend queries the
official VHA service, clips points against the exact persisted Area geometry,
stores the GeoJSON artifact and persists VectorFeature rows through DatasetService.
"""
from __future__ import annotations
import argparse
import json
import os
from typing import Any, Iterable
import requests
DEFAULT_API_URL = "http://127.0.0.1:8000"
DEFAULT_PROJECT_NAME = "Mol Municipality Workbench"
DEFAULT_AREA_FRAGMENT = "Gemeente Mol"
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Provision official VHA cross-section profiles for Mol.")
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME)
parser.add_argument("--area-name", default=DEFAULT_AREA_FRAGMENT)
parser.add_argument("--timeout", type=int, default=900)
parser.add_argument("--force", action="store_true")
return parser.parse_args()
def unwrap(response: requests.Response) -> Any:
response.raise_for_status()
payload = response.json()
if not isinstance(payload, dict) or "data" not in payload:
raise RuntimeError(f"Non-canonical API response from {response.url}")
return payload["data"]
def coordinates(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
def walk(value: Any):
if isinstance(value, list) and len(value) >= 2 and all(isinstance(item, (int, float)) for item in value[:2]):
yield float(value[0]), float(value[1])
return
if isinstance(value, list):
for child in value:
yield from walk(child)
yield from walk(geometry.get("coordinates", []))
def geometry_bbox(geometry: dict[str, Any]) -> dict[str, float | str]:
points = list(coordinates(geometry))
if not points:
raise RuntimeError("Persisted Area geometry contains no coordinates")
xs = [point[0] for point in points]
ys = [point[1] for point in points]
return {
"min_x": min(xs),
"min_y": min(ys),
"max_x": max(xs),
"max_y": max(ys),
"crs": "EPSG:4326",
}
def main() -> int:
args = parse_args()
base_url = args.base_url.rstrip("/")
session = requests.Session()
session.headers.update({"User-Agent": "GeoIntel-Bathymetry-Operator/1.0"})
projects = unwrap(session.get(f"{base_url}/api/v1/projects", params={"limit": 200, "offset": 0}, timeout=60))["items"]
project = next((item for item in projects if item["name"] == args.project_name), None)
if project is None:
raise RuntimeError(f"Project {args.project_name!r} was not found")
areas = unwrap(
session.get(
f"{base_url}/api/v1/projects/{project['id']}/areas",
params={"limit": 200, "offset": 0},
timeout=60,
)
)["items"]
fragment = args.area_name.casefold()
area = next((item for item in areas if fragment in item["name"].casefold()), None)
if area is None:
raise RuntimeError(f"Area containing {args.area_name!r} was not found")
bbox = geometry_bbox(area["geometry"])
job = unwrap(
session.post(
f"{base_url}/api/v1/projects/{project['id']}/datasets/bathymetry/profiles/acquire",
json={
"bbox": bbox,
"area_id": area["id"],
"force_refresh": args.force,
},
timeout=args.timeout,
)
)
if job.get("status") != "success" or not job.get("output_dataset_id"):
raise RuntimeError(f"Bathymetry profile acquisition failed: {job.get('error_message') or job}")
result = job.get("result_json") or {}
dataset_id = job["output_dataset_id"]
selection = unwrap(
session.post(
f"{base_url}/api/v1/projects/{project['id']}/datasets/{dataset_id}/vector/select",
json={"bbox": bbox, "area_id": area["id"], "limit": 1000},
timeout=args.timeout,
)
)
if selection.get("total_feature_count") != result.get("profile_count"):
raise RuntimeError(
"Persisted vector selection count does not match the exact acquired profile count "
f"({selection.get('total_feature_count')} != {result.get('profile_count')})"
)
metrics = {
item["metric_key"]: item
for item in (selection.get("summary") or {}).get("metrics", [])
if isinstance(item, dict) and item.get("metric_key")
}
if metrics.get("profile_count", {}).get("metric_value") != result.get("profile_count"):
raise RuntimeError("Bathymetry profile summary does not expose the persisted profile count")
print(
json.dumps(
{
"status": "ok",
"project_id": project["id"],
"area_id": area["id"],
"area_name": area["name"],
"bbox": bbox,
"dataset_id": dataset_id,
"reused": bool(result.get("reused")),
"profile_count": result.get("profile_count"),
"document_count": result.get("document_count"),
"structured_depth_count": result.get("structured_depth_count"),
"watercourse_count": result.get("watercourse_count"),
"measurement_date_min": result.get("measurement_date_min"),
"measurement_date_max": result.get("measurement_date_max"),
"metrics": list(metrics.values()),
"limitation_message": result.get("limitation_message"),
},
ensure_ascii=False,
indent=2,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+1
View File
@@ -59,6 +59,7 @@ ${PYTHON_BIN} -m py_compile scripts/orthophoto_release_preflight.py
${PYTHON_BIN} -m py_compile scripts/manage_orthophoto_release.py
${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_bathymetry_profiles.py
${PYTHON_BIN} -m py_compile scripts/provision_regional_dhmv.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_flood_hazards.py
${PYTHON_BIN} -m py_compile scripts/provision_regional_flood_hazards.py