From 5b4e18059a5ea0f1c1aaec96642a76f257fe2d57 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 17 Jul 2026 15:25:00 +0200 Subject: [PATCH] Add governed bathymetry profile workflow --- .env.example | 7 + CHANGELOG.md | 15 + backend/README.md | 20 + backend/app/api/routes/datasets.py | 26 + backend/app/core/config.py | 33 + backend/app/schemas/__init__.py | 8 + backend/app/schemas/bathymetry.py | 52 ++ .../bathymetry_profile_acquisition_service.py | 671 ++++++++++++++++++ backend/app/services/dataset_service.py | 120 ++++ .../app/services/vector_feature_service.py | 10 +- .../test_sprint235_bathymetry_profiles.py | 355 +++++++++ deploy/unraid/Dockerfile.all-in-one | 1 + docker-compose.yml | 7 + docs/API_CONTRACTS.md | 34 + docs/BATHYMETRY_EXPANSION_ROADMAP.md | 131 ++++ docs/CODEX_EXECUTION_LOG.md | 24 + docs/DATABASE_IMPLEMENTATION_PLAN.md | 15 + docs/DATA_SOURCES.md | 23 + docs/DATA_SPECIFICATION.md | 19 + docs/TODO.md | 15 + frontend/README.md | 13 + .../datasets/SourceCatalogPanel.tsx | 18 +- frontend/src/components/map/MapWorkspace.tsx | 60 +- frontend/src/lib/datasetDisplay.ts | 5 + frontend/src/lib/sourcePortfolio.ts | 36 + frontend/src/services/api/datasets.ts | 8 + frontend/src/styles/app.css | 59 ++ frontend/src/types.ts | 27 + scripts/README.md | 27 + scripts/provision_mol_bathymetry_profiles.py | 154 ++++ scripts/run_readiness_check.sh | 1 + 31 files changed, 1990 insertions(+), 4 deletions(-) create mode 100644 backend/app/schemas/bathymetry.py create mode 100644 backend/app/services/bathymetry_profile_acquisition_service.py create mode 100644 backend/tests/test_sprint235_bathymetry_profiles.py create mode 100644 docs/BATHYMETRY_EXPANSION_ROADMAP.md create mode 100644 scripts/provision_mol_bathymetry_profiles.py diff --git a/.env.example b/.env.example index 3f81a92a..00ac7afc 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,13 @@ FLOOD_HAZARD_MAX_SIDE_M=20000 FLOOD_HAZARD_MAX_PIXELS=12000000 FLOOD_HAZARD_TIMEOUT_SECONDS=300 FLOOD_HAZARD_MAX_RESPONSE_MB=160 +BATHYMETRY_PROFILES_ENABLED=true +BATHYMETRY_PROFILES_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0 +BATHYMETRY_WATERCOURSE_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1 +BATHYMETRY_PROFILES_PAGE_SIZE=1000 +BATHYMETRY_PROFILES_MAX_FEATURES=50000 +BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120 +BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32 THEMATIC_RASTER_ENABLED=true THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs THEMATIC_RASTER_MIN_SIDE_M=100 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dfbcc09..97abc00f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ # Changelog +## Sprint 235 Governed bathymetry profiles and Belgian scale architecture (2026-07-17) + +- Added bounded official VHA cross-section acquisition with exact Area + clipping, complete paging, checksums and ordinary Dataset/VectorFeature + persistence. +- Added nullable structured depth/width metrics, official profile-document + evidence and explicit unsupported volume semantics. +- Added a bathymetry source registry covering operational VHA profiles and + audited MDK North Sea, SPW Walloon and port candidates without pretending + those future connectors are configured. +- Added the `Waterbodem` Map theme and a concise profile inspector. +- Added the Mol operator command and a staged expansion roadmap for Flanders, + Belgium, the territorial sea, EEZ and continental shelf with strict + TAW/LAT/mDNG separation. + ## Sprint 234 Full audit closure and workspace lifecycle cleanup (2026-07-17) - Added reversible active/archived project lifecycle handling and made active diff --git a/backend/README.md b/backend/README.md index ef7eee0e..d2f9d72a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1708,3 +1708,23 @@ The command never deletes projects or related datasets, jobs, analyses, quality checks and exports. It always preserves `Kempen Regional Workbench` and `Mol Municipality Workbench`, defaults to dry-run and can print every matched name with `--show-names`. + +## Governed VHA bathymetry profiles + +`POST /api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire` +performs a bounded official VHA ArcGIS query, exact persisted-Area clipping, +watercourse-name normalization and ordinary Dataset/VectorFeature persistence. +`GET /api/v1/projects/{project_id}/datasets/bathymetry/sources` reports VHA as +operational and the audited MDK/SPW candidates as unavailable for acquisition. + +Runtime controls are `BATHYMETRY_PROFILES_ENABLED`, +`BATHYMETRY_PROFILES_LAYER_URL`, `BATHYMETRY_WATERCOURSE_LAYER_URL`, +`BATHYMETRY_PROFILES_PAGE_SIZE`, `BATHYMETRY_PROFILES_MAX_FEATURES`, +`BATHYMETRY_PROFILES_TIMEOUT_SECONDS` and +`BATHYMETRY_PROFILES_MAX_RESPONSE_MB`. The feature limit intentionally forces +large Flemish scopes into exact Area partitions. + +The Dataset exposes profile count and nullable structured depth/width metrics. +It does not claim a continuous bed model, current depth or volume. Use +`scripts/provision_mol_bathymetry_profiles.py` for the canonical Mol operator +flow. diff --git a/backend/app/api/routes/datasets.py b/backend/app/api/routes/datasets.py index 09db99ac..5c24f1cf 100644 --- a/backend/app/api/routes/datasets.py +++ b/backend/app/api/routes/datasets.py @@ -27,6 +27,7 @@ from app.schemas import ( FloodHazardAcquireRequest, FloodHazardPartitionSelectionRequest, FloodHazardSelectionRequest, + BathymetryProfileAcquireRequest, ThematicRasterAcquireRequest, ThematicRasterSelectionRequest, VectorBBoxResponse, @@ -52,6 +53,7 @@ from app.services.dhmv_acquisition_service import DhmvAcquisitionService from app.services.terrain_analysis_service import TerrainAnalysisService from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService +from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService from app.utils.response import envelope @@ -213,6 +215,30 @@ def list_flood_hazard_products(project_id: UUID, db: Session = Depends(get_db)): return envelope({"items": items, "total": len(items)}) +@router.get("/datasets/bathymetry/sources", response_model=dict) +def list_bathymetry_sources(project_id: UUID, db: Session = Depends(get_db)): + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + items = BathymetryProfileAcquisitionService.list_sources() + return envelope({"items": items, "total": len(items)}) + + +@router.post("/datasets/bathymetry/profiles/acquire", response_model=dict) +def acquire_bounded_bathymetry_profiles( + project_id: UUID, + payload: BathymetryProfileAcquireRequest, + db: Session = Depends(get_db), +): + job = JobService.run_sync_job( + db=db, + project_id=project_id, + job_type="vector.bathymetry_profiles.acquire", + parameters=payload.model_dump(mode="json"), + operation=lambda: BathymetryProfileAcquisitionService.acquire(db, project_id, payload), + ) + return envelope(job) + + @router.post("/datasets/thematic-raster/acquire", response_model=dict) def acquire_bounded_thematic_raster( project_id: UUID, diff --git a/backend/app/core/config.py b/backend/app/core/config.py index bb04c5c5..5ac0504f 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -90,6 +90,39 @@ class Settings(BaseSettings): flood_hazard_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="FLOOD_HAZARD_MAX_PIXELS") flood_hazard_timeout_seconds: int = Field(default=300, ge=1, validation_alias="FLOOD_HAZARD_TIMEOUT_SECONDS") flood_hazard_max_response_mb: int = Field(default=160, ge=1, validation_alias="FLOOD_HAZARD_MAX_RESPONSE_MB") + bathymetry_profiles_enabled: bool = Field(default=True, validation_alias="BATHYMETRY_PROFILES_ENABLED") + bathymetry_profiles_layer_url: str = Field( + default="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0", + validation_alias="BATHYMETRY_PROFILES_LAYER_URL", + ) + bathymetry_watercourse_layer_url: str = Field( + default="https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1", + validation_alias="BATHYMETRY_WATERCOURSE_LAYER_URL", + ) + bathymetry_profiles_page_size: int = Field( + default=1000, + ge=1, + le=2000, + validation_alias="BATHYMETRY_PROFILES_PAGE_SIZE", + ) + bathymetry_profiles_max_features: int = Field( + default=50_000, + ge=1, + le=250_000, + validation_alias="BATHYMETRY_PROFILES_MAX_FEATURES", + ) + bathymetry_profiles_timeout_seconds: int = Field( + default=120, + ge=1, + le=600, + validation_alias="BATHYMETRY_PROFILES_TIMEOUT_SECONDS", + ) + bathymetry_profiles_max_response_mb: int = Field( + default=32, + ge=1, + le=256, + validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB", + ) thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED") thematic_raster_wcs_url: str = Field( default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs", diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index e37cb822..fd78446b 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -65,6 +65,11 @@ from .flood_hazard import ( FloodHazardSelectionResponse, FloodHazardSelectionSummary, ) +from .bathymetry import ( + BathymetryProfileAcquireRequest, + BathymetryProfileAcquisitionResult, + BathymetrySourceRead, +) from .thematic_raster import ( ThematicRasterAcquireRequest, ThematicRasterAcquisitionResult, @@ -202,6 +207,9 @@ __all__ = [ "FloodHazardSelectionRequest", "FloodHazardSelectionResponse", "FloodHazardSelectionSummary", + "BathymetryProfileAcquireRequest", + "BathymetryProfileAcquisitionResult", + "BathymetrySourceRead", "ThematicRasterAcquireRequest", "ThematicRasterAcquisitionResult", "ThematicRasterMetric", diff --git a/backend/app/schemas/bathymetry.py b/backend/app/schemas/bathymetry.py new file mode 100644 index 00000000..77c3ccf5 --- /dev/null +++ b/backend/app/schemas/bathymetry.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, Field + +from .operations import VectorSelectionBBox + + +class BathymetryProfileAcquireRequest(BaseModel): + bbox: VectorSelectionBBox + area_id: UUID | None = None + force_refresh: bool = False + + +class BathymetrySourceRead(BaseModel): + key: str + display_name: str + owner: str + authority_level: Literal["authoritative", "contextual"] + geographic_coverage: str + data_kind: str + query_modes: list[str] + vertical_reference: str + horizontal_crs: str + native_resolution: str | None = None + integration_status: Literal["operational", "available_not_integrated", "catalog_only"] + acquisition_supported: bool + configured: bool + service_url: str | None = None + catalog_url: str + attribution: str + license_note: str + limitation_message: str + + +class BathymetryProfileAcquisitionResult(BaseModel): + output_dataset_id: UUID + reused: bool + provider: str + profile_count: int = Field(ge=0) + document_count: int = Field(ge=0) + structured_depth_count: int = Field(ge=0) + structured_width_count: int = Field(ge=0) + watercourse_count: int = Field(ge=0) + bbox_epsg4326: list[float] + clipped_to_area_id: UUID | None = None + measurement_date_min: str | None = None + measurement_date_max: str | None = None + attribution: str + limitation_message: str diff --git a/backend/app/services/bathymetry_profile_acquisition_service.py b/backend/app/services/bathymetry_profile_acquisition_service.py new file mode 100644 index 00000000..6db505a1 --- /dev/null +++ b/backend/app/services/bathymetry_profile_acquisition_service.py @@ -0,0 +1,671 @@ +from __future__ import annotations + +from datetime import UTC, datetime +import hashlib +import json +import math +from typing import Any, Callable +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen +from uuid import UUID + +from geoalchemy2.shape import to_shape +from shapely.geometry import Point, box, mapping + +from app.core.config import Settings, get_settings +from app.core.errors import AppError +from app.models import Area, Dataset, Project +from app.schemas.bathymetry import ( + BathymetryProfileAcquireRequest, + BathymetryProfileAcquisitionResult, + BathymetrySourceRead, +) +from app.services.dataset_service import DatasetService + + +class BathymetryProfileAcquisitionService: + PROVIDER = "vmm_vha_bathymetry_profiles" + SOURCE_VERSION = "VHA digitale atlas ArcGIS MapServer" + PROFILE_OUT_FIELDS = ( + "OBJECTID,vhag,atlaspunt,opg_kruinb,opg_vloerb,d_opmeti," + "hyperlink,bron,kunstwerkid,opg_diepte" + ) + ATTRIBUTION = "Vlaamse Milieumaatschappij (VMM), Vlaamse Hydrografische Atlas" + LICENSE_NOTE = "Hergebruik volgens de voorwaarden van de Vlaamse overheid en de bronmetadata." + LIMITATION = ( + "Dwarsprofielen zijn historische puntmetingen met bronafhankelijke meetdatum en verticale referentie. " + "Ze vormen geen continue actuele bodemkaart en ondersteunen zonder gelijktijdig waterpeil geen " + "gebiedsdekkend of actueel watervolume." + ) + _SOURCES = ( + { + "key": "vha_inland_profiles", + "display_name": "VHA dwarsprofielen binnenwater", + "owner": "Vlaamse Milieumaatschappij", + "authority_level": "authoritative", + "geographic_coverage": "Vlaanderen, puntlocaties op gekarteerde waterlopen", + "data_kind": "dwarsprofielpunten met meetvelden en brondocumenten", + "query_modes": ["bbox", "persisted_area"], + "vertical_reference": "document-specific; niet uniform als één peilreferentie te behandelen", + "horizontal_crs": "EPSG:4326", + "native_resolution": None, + "integration_status": "operational", + "acquisition_supported": True, + "configured": True, + "service_url": "https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0", + "catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/vlaamse-hydrografische-atlas-waterlopen", + "attribution": ATTRIBUTION, + "license_note": LICENSE_NOTE, + "limitation_message": LIMITATION, + }, + { + "key": "mdk_bcp_bathymetry", + "display_name": "Dieptemodel Belgisch Continentaal Plat", + "owner": "Agentschap Maritieme Dienstverlening en Kust", + "authority_level": "authoritative", + "geographic_coverage": "Belgisch Continentaal Plat en Noordzee", + "data_kind": "continu bathymetrisch raster", + "query_modes": ["wcs", "wmts", "bounded_raster"], + "vertical_reference": "LAT", + "horizontal_crs": "bronafhankelijk; expliciet per WCS-respons", + "native_resolution": "20 x 20 m", + "integration_status": "available_not_integrated", + "acquisition_supported": False, + "configured": False, + "service_url": "https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/WCS_Public", + "catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee", + "attribution": "Agentschap Maritieme Dienstverlening en Kust", + "license_note": "Zie de officiële datasetmetadata en gebruiksvoorwaarden.", + "limitation_message": ( + "Niet operationeel in GeoIntel totdat WCS, maritieme begrenzing, tegels, LAT-semantiek en " + "servercertificaten in een live integratietest zijn gevalideerd." + ), + }, + { + "key": "spw_walloon_waterway_bathymetry", + "display_name": "Bathymétrie des voies navigables et lacs-réservoirs", + "owner": "Service public de Wallonie", + "authority_level": "authoritative", + "geographic_coverage": "Waalse bevaarbare waterwegen en stuwmeren met uitgevoerde opmetingen", + "data_kind": "bodemhoogteraster en XYZ-puntenwolk", + "query_modes": ["download", "arcgis_map_service"], + "vertical_reference": "mDNG", + "horizontal_crs": "EPSG:3812; visualisatieservice kan EPSG:31370 aanbieden", + "native_resolution": "0,5 m", + "integration_status": "available_not_integrated", + "acquisition_supported": False, + "configured": False, + "service_url": "https://geoservices.wallonie.be/arcgis/rest/services/EAU/BATHY/MapServer", + "catalog_url": "https://geoportail.wallonie.be/catalogue/c450c28f-d357-48af-8423-62d524632cf9.html", + "attribution": "Service public de Wallonie", + "license_note": "CC BY 4.0 volgens de officiële Geoportail-metadata.", + "limitation_message": ( + "Dekking en meetjaar verschillen per vaarweg of reservoir. Integratie vereist een beheerde " + "download- en mosaïekstroom plus expliciete omzetting van mDNG." + ), + }, + { + "key": "port_antwerp_bathymetry", + "display_name": "Havenbathymetrie Antwerpen-Brugge", + "owner": "Port of Antwerp-Bruges", + "authority_level": "contextual", + "geographic_coverage": "Gepubliceerde havenzones en meetcampagnes", + "data_kind": "periodieke peilingen", + "query_modes": ["catalog"], + "vertical_reference": "product-specific", + "horizontal_crs": "product-specific", + "native_resolution": None, + "integration_status": "catalog_only", + "acquisition_supported": False, + "configured": False, + "service_url": None, + "catalog_url": "https://data.gov.be/nl/datasets", + "attribution": "Port of Antwerp-Bruges", + "license_note": "Per publicatie te verifiëren.", + "limitation_message": ( + "Alleen als cataloguskandidaat geregistreerd; er is nog geen stabiel, publiek en machineleesbaar " + "acquisitiecontract in GeoIntel gevalideerd." + ), + }, + ) + + @staticmethod + def list_sources() -> list[dict[str, Any]]: + return [BathymetrySourceRead(**item).model_dump() for item in BathymetryProfileAcquisitionService._SOURCES] + + @staticmethod + def _validate_bbox(payload: BathymetryProfileAcquireRequest) -> tuple[float, float, float, float]: + bbox = payload.bbox + if bbox.crs.upper() != "EPSG:4326": + raise AppError( + code="BATHYMETRY_INVALID_CRS", + message="Bathymetry profile acquisition requires EPSG:4326", + status_code=400, + ) + values = (bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y) + if not all(math.isfinite(value) for value in values): + raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box values must be finite", status_code=400) + if bbox.min_x >= bbox.max_x or bbox.min_y >= bbox.max_y: + raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box has no area", status_code=400) + if bbox.min_x < -180 or bbox.max_x > 180 or bbox.min_y < -90 or bbox.max_y > 90: + raise AppError(code="BATHYMETRY_INVALID_BBOX", message="Bounding box is outside EPSG:4326", status_code=400) + return values + + @staticmethod + def _scope_geometry(db, project_id: UUID, area_id: UUID | None, bbox_values: tuple[float, float, float, float]): + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + selection = box(*bbox_values) + if area_id is None: + return selection + area = db.get(Area, area_id) + if area is None: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + if area.project_id != project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400) + area_geometry = area.geometry if hasattr(area.geometry, "__geo_interface__") else to_shape(area.geometry) + intersection = area_geometry.intersection(selection) + if intersection.is_empty: + raise AppError( + code="BATHYMETRY_SCOPE_EMPTY", + message="The requested bounding box does not intersect the selected area", + status_code=400, + ) + return intersection + + @staticmethod + def _query_url(base_url: str, parameters: dict[str, Any]) -> str: + return f"{base_url}?{urlencode(parameters)}" + + @staticmethod + def _fetch_json( + url: str, + settings: Settings, + opener: Callable[..., Any] | None, + ) -> tuple[dict[str, Any], str]: + request = Request( + url, + headers={ + "Accept": "application/json", + "User-Agent": "GeoIntel/1.0 bathymetry-profile-acquisition", + }, + ) + try: + with (opener or urlopen)(request, timeout=settings.bathymetry_profiles_timeout_seconds) as response: + limit = settings.bathymetry_profiles_max_response_mb * 1024 * 1024 + content = response.read(limit + 1) + except HTTPError as exc: + raise AppError( + code="BATHYMETRY_PROVIDER_HTTP_ERROR", + message="VHA profile service returned an HTTP error", + details={"status_code": exc.code}, + status_code=502, + ) from exc + except (TimeoutError, URLError, OSError) as exc: + raise AppError( + code="BATHYMETRY_PROVIDER_UNAVAILABLE", + message="VHA profile service is unavailable", + status_code=502, + ) from exc + if len(content) > limit: + raise AppError( + code="BATHYMETRY_PROVIDER_RESPONSE_TOO_LARGE", + message="VHA profile response exceeded the configured size limit", + status_code=502, + ) + try: + payload = json.loads(content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise AppError( + code="BATHYMETRY_PROVIDER_INVALID_RESPONSE", + message="VHA profile service returned invalid JSON", + status_code=502, + ) from exc + if not isinstance(payload, dict) or payload.get("error"): + raise AppError( + code="BATHYMETRY_PROVIDER_INVALID_RESPONSE", + message="VHA profile service returned an ArcGIS error", + details={"provider_error": payload.get("error") if isinstance(payload, dict) else None}, + status_code=502, + ) + return payload, hashlib.sha256(content).hexdigest() + + @staticmethod + def _base_spatial_parameters(bbox_values: tuple[float, float, float, float]) -> dict[str, str]: + return { + "f": "json", + "where": "1=1", + "geometry": ",".join(f"{value:.12g}" for value in bbox_values), + "geometryType": "esriGeometryEnvelope", + "inSR": "4326", + "outSR": "4326", + "spatialRel": "esriSpatialRelIntersects", + } + + @staticmethod + def _fetch_profiles( + bbox_values: tuple[float, float, float, float], + settings: Settings, + opener: Callable[..., Any] | None, + ) -> tuple[list[dict[str, Any]], dict[str, Any]]: + base = settings.bathymetry_profiles_layer_url.rstrip("/") + "/query" + count_url = BathymetryProfileAcquisitionService._query_url( + base, + { + **BathymetryProfileAcquisitionService._base_spatial_parameters(bbox_values), + "returnCountOnly": "true", + "returnGeometry": "false", + }, + ) + count_payload, count_sha = BathymetryProfileAcquisitionService._fetch_json(count_url, settings, opener) + candidate_count = int(count_payload.get("count") or 0) + if candidate_count > settings.bathymetry_profiles_max_features: + raise AppError( + code="BATHYMETRY_SCOPE_TOO_LARGE", + message="The requested profile scope exceeds the configured feature limit; acquire smaller area partitions", + details={ + "candidate_count": candidate_count, + "max_features": settings.bathymetry_profiles_max_features, + }, + status_code=413, + ) + + features: list[dict[str, Any]] = [] + response_hashes: list[str] = [] + request_urls: list[str] = [count_url] + offset = 0 + while offset < candidate_count: + page_url = BathymetryProfileAcquisitionService._query_url( + base, + { + **BathymetryProfileAcquisitionService._base_spatial_parameters(bbox_values), + "outFields": BathymetryProfileAcquisitionService.PROFILE_OUT_FIELDS, + "returnGeometry": "true", + "orderByFields": "OBJECTID", + "resultOffset": str(offset), + "resultRecordCount": str(settings.bathymetry_profiles_page_size), + }, + ) + page, page_sha = BathymetryProfileAcquisitionService._fetch_json(page_url, settings, opener) + page_features = page.get("features") + if not isinstance(page_features, list): + raise AppError( + code="BATHYMETRY_PROVIDER_INVALID_RESPONSE", + message="VHA profile response does not contain a feature list", + status_code=502, + ) + features.extend(item for item in page_features if isinstance(item, dict)) + response_hashes.append(page_sha) + request_urls.append(page_url) + if not page_features: + break + offset += len(page_features) + if len(features) != candidate_count: + raise AppError( + code="BATHYMETRY_PROVIDER_INCOMPLETE_RESPONSE", + message="VHA profile pagination did not return the announced number of records", + details={"expected": candidate_count, "received": len(features)}, + status_code=502, + ) + return features, { + "candidate_count": candidate_count, + "request_urls": request_urls, + "response_sha256": [count_sha, *response_hashes], + } + + @staticmethod + def _fetch_watercourse_names( + vhag_codes: set[int], + settings: Settings, + opener: Callable[..., Any] | None, + ) -> tuple[dict[int, dict[str, str | None]], list[str], list[str]]: + if not vhag_codes: + return {}, [], [] + base = settings.bathymetry_watercourse_layer_url.rstrip("/") + "/query" + names: dict[int, dict[str, str | None]] = {} + urls: list[str] = [] + hashes: list[str] = [] + ordered_codes = sorted(vhag_codes) + for start in range(0, len(ordered_codes), 100): + chunk = ordered_codes[start : start + 100] + offset = 0 + while True: + url = BathymetryProfileAcquisitionService._query_url( + base, + { + "f": "json", + "where": f"\"wlasvl.vhag\" IN ({','.join(str(code) for code in chunk)})", + "outFields": "wlasvl.vhag,VHAG_TABEL.naam,VHAG_TABEL.namen", + "returnGeometry": "false", + "orderByFields": "wlasvl.vhag", + "resultOffset": str(offset), + "resultRecordCount": str(settings.bathymetry_profiles_page_size), + }, + ) + payload, response_sha = BathymetryProfileAcquisitionService._fetch_json(url, settings, opener) + urls.append(url) + hashes.append(response_sha) + page_features = payload.get("features") + if not isinstance(page_features, list): + raise AppError( + code="BATHYMETRY_PROVIDER_INVALID_RESPONSE", + message="VHA watercourse response does not contain a feature list", + status_code=502, + ) + for feature in page_features: + attributes = feature.get("attributes") if isinstance(feature, dict) else None + if not isinstance(attributes, dict): + continue + raw_code = attributes.get("wlasvl.vhag") + if raw_code is None: + continue + code = int(raw_code) + if code not in names: + names[code] = { + "name": attributes.get("VHAG_TABEL.naam"), + "alternative_names": attributes.get("VHAG_TABEL.namen"), + } + if not payload.get("exceededTransferLimit") or not page_features: + break + offset += len(page_features) + return names, urls, hashes + + @staticmethod + def _date_from_arcgis(value: Any) -> str | None: + if not isinstance(value, (int, float)) or not math.isfinite(float(value)): + return None + try: + return datetime.fromtimestamp(float(value) / 1000.0, tz=UTC).date().isoformat() + except (OverflowError, OSError, ValueError): + return None + + @staticmethod + def _numeric_or_none(value: Any) -> float | None: + if value is None: + return None + try: + normalized = float(value) + except (TypeError, ValueError): + return None + return normalized if math.isfinite(normalized) else None + + @staticmethod + def _document_url(value: Any) -> str | None: + if not isinstance(value, str) or not value.strip(): + return None + normalized = value.strip() + if normalized.startswith("http://vha.waterinfo.be/"): + normalized = "https://" + normalized[len("http://") :] + return normalized if normalized.startswith("https://vha.waterinfo.be/") else None + + @staticmethod + def _normalize_features( + raw_features: list[dict[str, Any]], + scope_geometry, + watercourse_names: dict[int, dict[str, str | None]], + ) -> tuple[dict[str, Any], dict[str, Any]]: + normalized: list[dict[str, Any]] = [] + dates: list[str] = [] + watercourse_codes: set[int] = set() + document_count = 0 + depth_count = 0 + width_count = 0 + for raw_feature in raw_features: + attributes = raw_feature.get("attributes") + geometry = raw_feature.get("geometry") + if not isinstance(attributes, dict) or not isinstance(geometry, dict): + continue + x = BathymetryProfileAcquisitionService._numeric_or_none(geometry.get("x")) + y = BathymetryProfileAcquisitionService._numeric_or_none(geometry.get("y")) + if x is None or y is None: + continue + point = Point(x, y) + if not scope_geometry.covers(point): + continue + raw_vhag = attributes.get("vhag") + vhag = int(raw_vhag) if isinstance(raw_vhag, (int, float)) else None + if vhag is not None: + watercourse_codes.add(vhag) + names = watercourse_names.get(vhag or -1, {}) + measurement_date = BathymetryProfileAcquisitionService._date_from_arcgis(attributes.get("d_opmeti")) + if measurement_date: + dates.append(measurement_date) + document_url = BathymetryProfileAcquisitionService._document_url(attributes.get("hyperlink")) + depth = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_diepte")) + crown_width = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_kruinb")) + floor_width = BathymetryProfileAcquisitionService._numeric_or_none(attributes.get("opg_vloerb")) + if document_url: + document_count += 1 + if depth is not None: + depth_count += 1 + if crown_width is not None or floor_width is not None: + width_count += 1 + object_id = str(attributes.get("OBJECTID")) + normalized.append( + { + "type": "Feature", + "id": object_id, + "properties": { + "source_feature_id": object_id, + "provider_record_id": object_id, + "watercourse_vhag": vhag, + "watercourse_name": names.get("name") or (f"VHA-waterloop {vhag}" if vhag else "Onbekende waterloop"), + "watercourse_alternative_names": names.get("alternative_names"), + "profile_number": attributes.get("atlaspunt"), + "measurement_date": measurement_date, + "recorded_depth_m": depth, + "recorded_crown_width_m": crown_width, + "recorded_floor_width_m": floor_width, + "source_document_url": document_url, + "document_available": document_url is not None, + "structured_depth_available": depth is not None, + "source_code": attributes.get("bron"), + "structure_id": attributes.get("kunstwerkid"), + "provider": BathymetryProfileAcquisitionService.PROVIDER, + "measurement_semantics": "historical_cross_section_profile_point", + "vertical_reference": "document-specific", + }, + "geometry": mapping(point), + } + ) + return ( + { + "type": "FeatureCollection", + "name": "vha_bathymetry_profiles", + "crs": {"type": "name", "properties": {"name": "EPSG:4326"}}, + "features": normalized, + }, + { + "profile_count": len(normalized), + "document_count": document_count, + "structured_depth_count": depth_count, + "structured_width_count": width_count, + "watercourse_count": len(watercourse_codes), + "measurement_date_min": min(dates) if dates else None, + "measurement_date_max": max(dates) if dates else None, + }, + ) + + @staticmethod + def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None: + return ( + db.query(Dataset) + .filter( + Dataset.project_id == project_id, + Dataset.name == filename, + Dataset.source_name == BathymetryProfileAcquisitionService.PROVIDER, + Dataset.status == "ready", + ) + .order_by(Dataset.created_at.desc()) + .first() + ) + + @staticmethod + def _result(dataset: Dataset, *, reused: bool) -> dict[str, Any]: + metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {} + return BathymetryProfileAcquisitionResult( + output_dataset_id=dataset.id, + reused=reused, + provider=BathymetryProfileAcquisitionService.PROVIDER, + profile_count=int(metadata.get("profile_count") or 0), + document_count=int(metadata.get("document_count") or 0), + structured_depth_count=int(metadata.get("structured_depth_count") or 0), + structured_width_count=int(metadata.get("structured_width_count") or 0), + watercourse_count=int(metadata.get("watercourse_count") or 0), + bbox_epsg4326=list(metadata.get("bbox_epsg4326") or []), + clipped_to_area_id=dataset.area_id, + measurement_date_min=metadata.get("measurement_date_min"), + measurement_date_max=metadata.get("measurement_date_max"), + attribution=BathymetryProfileAcquisitionService.ATTRIBUTION, + limitation_message=BathymetryProfileAcquisitionService.LIMITATION, + ).model_dump(mode="json") + + @staticmethod + def acquire( + db, + project_id: UUID, + payload: BathymetryProfileAcquireRequest, + *, + settings: Settings | None = None, + opener: Callable[..., Any] | None = None, + ) -> dict[str, Any]: + resolved_settings = settings or get_settings() + if not resolved_settings.bathymetry_profiles_enabled: + raise AppError( + code="BATHYMETRY_NOT_CONFIGURED", + message="VHA bathymetry profile acquisition is disabled", + status_code=503, + ) + bbox_values = BathymetryProfileAcquisitionService._validate_bbox(payload) + scope_geometry = BathymetryProfileAcquisitionService._scope_geometry( + db, project_id, payload.area_id, bbox_values + ) + exact_bbox = tuple(float(value) for value in scope_geometry.bounds) + request_identity = { + "provider": BathymetryProfileAcquisitionService.PROVIDER, + "bbox_epsg4326": list(exact_bbox), + "area_id": str(payload.area_id) if payload.area_id else None, + "source_version": BathymetryProfileAcquisitionService.SOURCE_VERSION, + } + request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode()).hexdigest() + filename = f"vha_bathymetry_profiles_{request_hash[:12]}.geojson" + if not payload.force_refresh: + cached = BathymetryProfileAcquisitionService._cached_dataset(db, project_id, filename) + if cached is not None: + return BathymetryProfileAcquisitionService._result(cached, reused=True) + + raw_features, fetch_provenance = BathymetryProfileAcquisitionService._fetch_profiles( + exact_bbox, resolved_settings, opener + ) + vhag_codes = { + int(feature["attributes"]["vhag"]) + for feature in raw_features + if isinstance(feature.get("attributes"), dict) + and isinstance(feature["attributes"].get("vhag"), (int, float)) + } + names, name_urls, name_hashes = BathymetryProfileAcquisitionService._fetch_watercourse_names( + vhag_codes, resolved_settings, opener + ) + feature_collection, summary = BathymetryProfileAcquisitionService._normalize_features( + raw_features, scope_geometry, names + ) + if summary["profile_count"] == 0: + raise AppError( + code="BATHYMETRY_NO_PROFILES", + message="No VHA bathymetry profiles intersect the requested area", + status_code=404, + ) + artifact = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + acquired_at = datetime.now(UTC) + source_metadata = { + "provider": BathymetryProfileAcquisitionService.PROVIDER, + "service": "ArcGIS MapServer", + "source_version": BathymetryProfileAcquisitionService.SOURCE_VERSION, + "theme": "bathymetry", + "layer_name": "bathymetry_profiles", + "coverage_scope": "municipality" if payload.area_id else "bounded_selection", + "bbox_epsg4326": list(exact_bbox), + **summary, + "selection_aggregation": { + "metric_key": "profile_count", + "method": "feature_count", + "label": "Dwarsprofielen", + "unit": "profielen", + }, + "selection_metrics": [ + { + "metric_key": "recorded_depth_mean_m", + "method": "mean", + "property": "recorded_depth_m", + "label": "Gemiddelde geregistreerde diepte", + "unit": "m", + "warning": "Alleen profielen met een gestructureerde dieptewaarde; meetdata kunnen verschillen.", + }, + { + "metric_key": "recorded_depth_min_m", + "method": "min", + "property": "recorded_depth_m", + "label": "Kleinste geregistreerde diepte", + "unit": "m", + }, + { + "metric_key": "recorded_depth_max_m", + "method": "max", + "property": "recorded_depth_m", + "label": "Grootste geregistreerde diepte", + "unit": "m", + }, + { + "metric_key": "recorded_crown_width_mean_m", + "method": "mean", + "property": "recorded_crown_width_m", + "label": "Gemiddelde geregistreerde kruinbreedte", + "unit": "m", + }, + { + "metric_key": "recorded_floor_width_mean_m", + "method": "mean", + "property": "recorded_floor_width_m", + "label": "Gemiddelde geregistreerde vloerbreedte", + "unit": "m", + }, + ], + "attribution": BathymetryProfileAcquisitionService.ATTRIBUTION, + "license_note": BathymetryProfileAcquisitionService.LICENSE_NOTE, + "limitation_message": BathymetryProfileAcquisitionService.LIMITATION, + "volume_supported": False, + } + dataset = DatasetService.import_vector_bytes( + db, + project_id=project_id, + area_id=payload.area_id, + filename=filename, + content=artifact, + source="VHA digitale atlas dwarsprofielen", + source_name=BathymetryProfileAcquisitionService.PROVIDER, + dataset_role="reference", + reference_layer_name="bathymetry_profiles", + source_version=BathymetryProfileAcquisitionService.SOURCE_VERSION, + source_metadata=source_metadata, + provenance_metadata={ + "acquisition": "explicit_bounded_arcgis_feature_query", + "acquired_at": acquired_at.isoformat(), + "request_hash": request_hash, + "profile_query_urls": fetch_provenance["request_urls"], + "watercourse_name_query_urls": name_urls, + "response_sha256": [*fetch_provenance["response_sha256"], *name_hashes], + "artifact_sha256": hashlib.sha256(artifact).hexdigest(), + "candidate_count": fetch_provenance["candidate_count"], + "exact_profile_count": summary["profile_count"], + "clipped_to_area_id": str(payload.area_id) if payload.area_id else None, + "scope_geometry_type": scope_geometry.geom_type, + "vertical_reference": "document-specific", + "bathymetric_surface_available": False, + "water_surface_level_available": False, + "water_volume_available": False, + "limitation_message": BathymetryProfileAcquisitionService.LIMITATION, + }, + ) + persisted = db.get(Dataset, dataset.id) + return BathymetryProfileAcquisitionService._result(persisted, reused=False) diff --git a/backend/app/services/dataset_service.py b/backend/app/services/dataset_service.py index 153040f7..ca245183 100644 --- a/backend/app/services/dataset_service.py +++ b/backend/app/services/dataset_service.py @@ -410,6 +410,126 @@ class DatasetService: return DatasetService._to_response(dataset) + @staticmethod + def import_vector_bytes( + db: Session, + *, + project_id: UUID, + filename: str, + content: bytes, + source: str, + source_name: str, + dataset_role: str, + reference_layer_name: str | None, + source_metadata: dict[str, Any], + provenance_metadata: dict[str, Any], + area_id: UUID | None = None, + temporal_series_key: str | None = None, + observed_at: datetime | None = None, + valid_from: datetime | None = None, + valid_to: datetime | None = None, + temporal_granularity: str | None = None, + source_version: str | None = None, + content_type: str = "application/geo+json", + ) -> DatasetCreateResponse: + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + if area_id is not None: + area = db.get(Area, area_id) + if not area: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + if area.project_id != project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400) + if not content: + raise AppError(code="INVALID_UPLOAD", message="Vector artifact is empty", status_code=400) + safe_filename = DatasetService._validate_upload_filename(filename) + if DatasetService._extension_for_path(safe_filename) not in DatasetService.VECTOR_EXTENSIONS: + raise AppError(code="INVALID_UPLOAD", message="Vector artifacts require .geojson or .json files", status_code=415) + normalized_role = DatasetService._normalize_dataset_role(dataset_role) + normalized_source_name = (source_name or "").strip() or ("manual" if normalized_role == "reference" else None) + temporal = DatasetService._validate_temporal_metadata( + temporal_series_key=temporal_series_key, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_granularity=temporal_granularity, + source_version=source_version, + ) + try: + text = content.decode("utf-8") + except UnicodeDecodeError as exc: + raise AppError(code="INVALID_UPLOAD", message="Vector artifact must be UTF-8 encoded", status_code=400) from exc + try: + metadata = parse_geojson_payload(text) + vector_payload = json.loads(text) + except (ValueError, json.JSONDecodeError) as exc: + raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc + + dataset_id = uuid.uuid4() + storage_info = StorageService.persist_dataset_file( + project_id=str(project_id), + dataset_id=str(dataset_id), + dataset_type="vector", + original_filename=safe_filename, + content=content, + content_type=content_type, + ) + dataset = Dataset( + id=dataset_id, + project_id=project_id, + area_id=area_id, + name=safe_filename, + dataset_type="vector", + source=source, + dataset_role=normalized_role, + source_name=normalized_source_name, + reference_layer_name=reference_layer_name if normalized_role == "reference" else None, + source_metadata=source_metadata, + provenance_metadata=provenance_metadata, + imported_at=datetime.now(timezone.utc), + **temporal, + storage_path=storage_info["storage_path"], + original_filename=storage_info["original_filename"], + stored_filename=storage_info["stored_filename"], + content_type=storage_info["content_type"], + size_bytes=storage_info["size_bytes"], + checksum_sha256=storage_info["checksum_sha256"], + crs=metadata.get("crs"), + bounds_json=metadata.get("bounds_json"), + metadata_json=metadata, + status="ready", + ) + try: + db.add(dataset) + db.add( + DatasetVersion( + dataset_id=dataset.id, + version=1, + storage_path=dataset.storage_path, + source_version=dataset.source_version, + observed_at=dataset.observed_at, + valid_from=dataset.valid_from, + valid_to=dataset.valid_to, + checksum_sha256=dataset.checksum_sha256, + source_metadata=dataset.source_metadata, + provenance_metadata=dataset.provenance_metadata, + ) + ) + VectorFeatureService.persist_geojson_features( + db=db, + dataset_id=dataset.id, + payload=vector_payload, + feature_class=reference_layer_name if normalized_role == "reference" else None, + commit=False, + ) + db.commit() + db.refresh(dataset) + except Exception: + db.rollback() + StorageService.remove_dataset_file(storage_info["storage_path"]) + raise + return DatasetService._to_response(dataset) + @staticmethod def import_raster_bytes( db: Session, diff --git a/backend/app/services/vector_feature_service.py b/backend/app/services/vector_feature_service.py index ef140908..dedf85ed 100644 --- a/backend/app/services/vector_feature_service.py +++ b/backend/app/services/vector_feature_service.py @@ -36,6 +36,8 @@ SEMANTIC_METRICS_DISABLED_OPERATOR_TOOLS = { # line metrics (road/watercourse length) would therefore be meaningless. "provision_regional_historical_landuse.py", } +PROPERTY_AGGREGATION_METHODS = {"sum", "mean", "area_weighted_sum"} +PROPERTY_EXTREMA_METHODS = {"min", "max"} PRECLIPPED_MUNICIPALITY_PARTITION_OPERATOR_TOOLS = { "provision_regional_bwk_natura2000.py", @@ -581,7 +583,7 @@ class VectorFeatureService: length_m = db.query(func.coalesce(func.sum(length_expression), 0.0)).filter(*metric_filter).scalar() divisor = 1_000.0 if unit == "km" else 1.0 metric_value = float(length_m or 0.0) / divisor - elif method in {"sum", "mean", "area_weighted_sum"}: + elif method in PROPERTY_AGGREGATION_METHODS | PROPERTY_EXTREMA_METHODS: property_name = str(config.get("property") or "").strip() if not property_name: raise AppError( @@ -599,7 +601,11 @@ class VectorFeatureService: ) coverage_ratio = intersection_area / func.nullif(source_area, 0.0) value_expression = numeric_value * coverage_ratio - aggregate_function = func.avg if method == "mean" else func.sum + aggregate_function = { + "mean": func.avg, + "min": func.min, + "max": func.max, + }.get(method, func.sum) aggregate_value = ( db.query(func.coalesce(aggregate_function(value_expression), 0.0)) .filter(*metric_filter) diff --git a/backend/tests/test_sprint235_bathymetry_profiles.py b/backend/tests/test_sprint235_bathymetry_profiles.py new file mode 100644 index 00000000..941cd053 --- /dev/null +++ b/backend/tests/test_sprint235_bathymetry_profiles.py @@ -0,0 +1,355 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from urllib.parse import parse_qs, urlparse +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from geoalchemy2.shape import from_shape +from shapely.geometry import MultiPolygon, Polygon + +from app.core.config import Settings +from app.core.errors import AppError +from app.db.session import get_db +from app.main import app +from app.models import Area, Dataset, Job, Project +from app.schemas.bathymetry import BathymetryProfileAcquireRequest +from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService +from app.services.dataset_service import DatasetService + + +ROOT = Path(__file__).resolve().parents[2] + + +class FakeQuery: + def __init__(self, result=None): + self.result = result + + def filter(self, *_args): + return self + + def order_by(self, *_args): + return self + + def first(self): + return self.result + + def all(self): + return self.result if isinstance(self.result, list) else [] + + +class FakeSession: + def __init__(self, rows=None, query_result=None): + self.rows = rows or {} + self.query_result = query_result + self.added = [] + + def get(self, model, row_id): + row = self.rows.get((model, row_id)) + if row is not None: + return row + return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None) + + def add(self, row): + self.added.append(row) + + def commit(self): + return None + + def rollback(self): + return None + + def refresh(self, row): + return row + + def query(self, _model): + return FakeQuery(self.query_result) + + +class JsonResponse: + def __init__(self, payload): + self.content = json.dumps(payload).encode("utf-8") + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def read(self, size=-1): + return self.content if size < 0 else self.content[:size] + + +def request(*, area_id=None, force_refresh=True) -> BathymetryProfileAcquireRequest: + return BathymetryProfileAcquireRequest( + bbox={ + "min_x": 5.0, + "min_y": 51.0, + "max_x": 6.0, + "max_y": 52.0, + "crs": "EPSG:4326", + }, + area_id=area_id, + force_refresh=force_refresh, + ) + + +def profile(object_id, vhag, x, y, *, depth=None, document=None, measured_at=951868800000): + return { + "attributes": { + "OBJECTID": object_id, + "vhag": vhag, + "atlaspunt": str(object_id), + "opg_kruinb": 4.5 if depth is not None else None, + "opg_vloerb": 1.2 if depth is not None else None, + "d_opmeti": measured_at, + "hyperlink": document, + "bron": 4, + "kunstwerkid": f"structure-{object_id}", + "opg_diepte": depth, + }, + "geometry": {"x": x, "y": y}, + } + + +def provider_opener(*, count=3): + profiles = [ + profile( + 1, + 8506, + 5.2, + 51.2, + depth=1.8, + document="http://vha.waterinfo.be/download/dwarsprofielen/Molse_Nete/8506_DP_1.pdf", + ), + profile(2, 8634, 5.8, 51.8, depth=2.4), + profile(3, 8506, 5.2, 51.8), + ] + + def opener(raw_request, timeout): + assert timeout == 120 + url = raw_request.full_url + query = parse_qs(urlparse(url).query) + if query.get("returnCountOnly") == ["true"]: + return JsonResponse({"count": count}) + if "MapServer/1/query" in url: + return JsonResponse( + { + "features": [ + { + "attributes": { + "wlasvl.vhag": 8506, + "VHAG_TABEL.naam": "Molse Nete", + "VHAG_TABEL.namen": "Molse Nete - Mol Neet", + } + }, + { + "attributes": { + "wlasvl.vhag": 8634, + "VHAG_TABEL.naam": "Scheppelijke Nete", + "VHAG_TABEL.namen": "Scheppelijke Nete - Stevensloop", + } + }, + ] + } + ) + return JsonResponse({"features": profiles[:count]}) + + return opener + + +def test_bathymetry_source_registry_is_honest_and_nationally_extensible() -> None: + sources = BathymetryProfileAcquisitionService.list_sources() + by_key = {item["key"]: item for item in sources} + + assert set(by_key) == { + "vha_inland_profiles", + "mdk_bcp_bathymetry", + "spw_walloon_waterway_bathymetry", + "port_antwerp_bathymetry", + } + assert by_key["vha_inland_profiles"]["integration_status"] == "operational" + assert by_key["vha_inland_profiles"]["acquisition_supported"] is True + assert by_key["mdk_bcp_bathymetry"]["vertical_reference"] == "LAT" + assert by_key["mdk_bcp_bathymetry"]["acquisition_supported"] is False + assert by_key["spw_walloon_waterway_bathymetry"]["vertical_reference"] == "mDNG" + assert by_key["spw_walloon_waterway_bathymetry"]["license_note"].startswith("CC BY 4.0") + + +def test_bathymetry_normalization_exactly_clips_area_and_preserves_evidence() -> None: + l_shape = Polygon( + [ + (5.0, 51.0), + (6.0, 51.0), + (6.0, 51.4), + (5.4, 51.4), + (5.4, 52.0), + (5.0, 52.0), + (5.0, 51.0), + ] + ) + raw, _provenance = BathymetryProfileAcquisitionService._fetch_profiles( + (5.0, 51.0, 6.0, 52.0), + Settings(_env_file=None), + provider_opener(), + ) + names, _urls, _hashes = BathymetryProfileAcquisitionService._fetch_watercourse_names( + {8506, 8634}, + Settings(_env_file=None), + provider_opener(), + ) + collection, summary = BathymetryProfileAcquisitionService._normalize_features(raw, l_shape, names) + + assert summary == { + "profile_count": 2, + "document_count": 1, + "structured_depth_count": 1, + "structured_width_count": 1, + "watercourse_count": 1, + "measurement_date_min": "2000-03-01", + "measurement_date_max": "2000-03-01", + } + assert {feature["id"] for feature in collection["features"]} == {"1", "3"} + first = collection["features"][0]["properties"] + assert first["watercourse_name"] == "Molse Nete" + assert first["recorded_depth_m"] == 1.8 + assert first["source_document_url"].startswith("https://vha.waterinfo.be/") + assert first["vertical_reference"] == "document-specific" + + +def test_bathymetry_acquisition_persists_reference_dataset_through_dataset_service(monkeypatch) -> None: + project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4() + l_shape = MultiPolygon( + [ + Polygon( + [ + (5.0, 51.0), + (6.0, 51.0), + (6.0, 51.4), + (5.4, 51.4), + (5.4, 52.0), + (5.0, 52.0), + (5.0, 51.0), + ] + ) + ] + ) + project = Project(id=project_id, name="Mol") + area = Area( + id=area_id, + project_id=project_id, + name="Gemeente Mol - officieel", + geometry=from_shape(l_shape, srid=4326), + ) + db = FakeSession({(Project, project_id): project, (Area, area_id): area}) + captured = {} + + def persist(_db, **kwargs): + captured.update(kwargs) + dataset = Dataset( + id=dataset_id, + project_id=project_id, + area_id=area_id, + name=kwargs["filename"], + dataset_type="vector", + source=kwargs["source"], + dataset_role=kwargs["dataset_role"], + source_name=kwargs["source_name"], + reference_layer_name=kwargs["reference_layer_name"], + source_metadata=kwargs["source_metadata"], + provenance_metadata=kwargs["provenance_metadata"], + status="ready", + ) + db.rows[(Dataset, dataset_id)] = dataset + return SimpleNamespace(id=dataset_id) + + monkeypatch.setattr(DatasetService, "import_vector_bytes", persist) + result = BathymetryProfileAcquisitionService.acquire( + db, + project_id, + request(area_id=area_id), + settings=Settings(_env_file=None), + opener=provider_opener(), + ) + + assert result["output_dataset_id"] == str(dataset_id) + assert result["profile_count"] == 2 + assert captured["dataset_role"] == "reference" + assert captured["source_name"] == BathymetryProfileAcquisitionService.PROVIDER + assert captured["reference_layer_name"] == "bathymetry_profiles" + assert captured["source_metadata"]["theme"] == "bathymetry" + assert captured["source_metadata"]["volume_supported"] is False + assert captured["provenance_metadata"]["water_volume_available"] is False + payload = json.loads(captured["content"]) + assert len(payload["features"]) == 2 + + +def test_bathymetry_acquisition_rejects_unbounded_feature_volume() -> None: + settings = Settings(_env_file=None, BATHYMETRY_PROFILES_MAX_FEATURES=2) + with pytest.raises(AppError) as exc_info: + BathymetryProfileAcquisitionService._fetch_profiles( + (5.0, 51.0, 6.0, 52.0), + settings, + provider_opener(count=3), + ) + assert exc_info.value.code == "BATHYMETRY_SCOPE_TOO_LARGE" + assert exc_info.value.details["candidate_count"] == 3 + + +def test_bathymetry_source_api_uses_canonical_envelope() -> None: + project_id = uuid4() + db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}) + app.dependency_overrides[get_db] = lambda: db + try: + response = TestClient(app).get(f"/api/v1/projects/{project_id}/datasets/bathymetry/sources") + finally: + app.dependency_overrides.clear() + + assert response.status_code == 200 + assert set(response.json()) == {"data"} + assert response.json()["data"]["total"] == 4 + assert response.json()["data"]["items"][0]["integration_status"] == "operational" + + +def test_bathymetry_acquisition_route_stays_inside_existing_job_envelope(monkeypatch) -> None: + project_id, dataset_id = uuid4(), uuid4() + db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}) + monkeypatch.setattr( + BathymetryProfileAcquisitionService, + "acquire", + lambda *_args, **_kwargs: { + "output_dataset_id": str(dataset_id), + "provider": BathymetryProfileAcquisitionService.PROVIDER, + "profile_count": 2, + }, + ) + app.dependency_overrides[get_db] = lambda: db + try: + response = TestClient(app).post( + f"/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire", + json=request().model_dump(mode="json"), + ) + finally: + app.dependency_overrides.clear() + + assert response.status_code == 200 + assert set(response.json()) == {"data"} + assert response.json()["data"]["job_type"] == "vector.bathymetry_profiles.acquire" + assert response.json()["data"]["output_dataset_id"] == str(dataset_id) + assert any(isinstance(item, Job) for item in db.added) + + +def test_bathymetry_contract_and_expansion_roadmap_are_documented() -> None: + api_contracts = (ROOT / "docs" / "API_CONTRACTS.md").read_text(encoding="utf-8") + roadmap = ROOT / "docs" / "BATHYMETRY_EXPANSION_ROADMAP.md" + assert "bathymetry/profiles/acquire" in api_contracts + assert roadmap.exists() + contents = roadmap.read_text(encoding="utf-8") + assert "LAT" in contents and "mDNG" in contents and "territoriale zee" in contents + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") + assert "py_compile scripts/provision_mol_bathymetry_profiles.py" in readiness + assert "COPY scripts/provision_mol_bathymetry_profiles.py" in dockerfile diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index ade4667f..27fd463d 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -88,6 +88,7 @@ COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_hist COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_regional_historical_landuse.py COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py +COPY scripts/provision_mol_bathymetry_profiles.py /app/scripts/provision_mol_bathymetry_profiles.py COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py diff --git a/docker-compose.yml b/docker-compose.yml index 279dc927..86674aa4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,6 +54,13 @@ services: FLOOD_HAZARD_MAX_PIXELS: ${FLOOD_HAZARD_MAX_PIXELS:-12000000} FLOOD_HAZARD_TIMEOUT_SECONDS: ${FLOOD_HAZARD_TIMEOUT_SECONDS:-300} FLOOD_HAZARD_MAX_RESPONSE_MB: ${FLOOD_HAZARD_MAX_RESPONSE_MB:-160} + BATHYMETRY_PROFILES_ENABLED: ${BATHYMETRY_PROFILES_ENABLED:-true} + BATHYMETRY_PROFILES_LAYER_URL: ${BATHYMETRY_PROFILES_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0} + BATHYMETRY_WATERCOURSE_LAYER_URL: ${BATHYMETRY_WATERCOURSE_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1} + BATHYMETRY_PROFILES_PAGE_SIZE: ${BATHYMETRY_PROFILES_PAGE_SIZE:-1000} + BATHYMETRY_PROFILES_MAX_FEATURES: ${BATHYMETRY_PROFILES_MAX_FEATURES:-50000} + BATHYMETRY_PROFILES_TIMEOUT_SECONDS: ${BATHYMETRY_PROFILES_TIMEOUT_SECONDS:-120} + BATHYMETRY_PROFILES_MAX_RESPONSE_MB: ${BATHYMETRY_PROFILES_MAX_RESPONSE_MB:-32} THEMATIC_RASTER_ENABLED: ${THEMATIC_RASTER_ENABLED:-true} THEMATIC_RASTER_WCS_URL: ${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs} THEMATIC_RASTER_MIN_SIDE_M: ${THEMATIC_RASTER_MIN_SIDE_M:-100} diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index c03f7f3d..6c7564f6 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -2055,3 +2055,37 @@ specifically, no water volume is inferred from 2D water geometry. The backend sets an explicit Ollama context window and returns `OLLAMA_RESPONSE_TRUNCATED` instead of accepting a response with `done_reason=length` as a complete answer. + +## Bathymetry and VHA profile acquisition + +### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources` + +Returns the governed bathymetry source registry in the canonical envelope. +VHA inland profiles are `operational`. MDK Belgian Continental Shelf and SPW +Walloon bathymetry remain `available_not_integrated`; they cannot be acquired +through this contract until their raster/download and vertical-datum flows +pass live validation. + +### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire` + +```json +{ + "bbox": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.3, "max_y": 51.3, "crs": "EPSG:4326"}, + "area_id": "optional persisted Area uuid", + "force_refresh": false +} +``` + +The endpoint creates a synchronous `vector.bathymetry_profiles.acquire` Job. +It queries the official VHA Digital Atlas in bounded pages, intersects every +point with the persisted Area when supplied and persists one ordinary +reference Dataset plus VectorFeature rows. The result reports exact profile, +document, structured-depth/width and watercourse counts and the source +measurement-date range. + +The acquired Dataset uses +`source_name=vmm_vha_bathymetry_profiles`, +`dataset_role=reference` and +`reference_layer_name=bathymetry_profiles`. Existing vector content and +selection endpoints provide GeoJSON and metrics. The contract does not expose +a continuous bathymetric surface, current water depth or water volume. diff --git a/docs/BATHYMETRY_EXPANSION_ROADMAP.md b/docs/BATHYMETRY_EXPANSION_ROADMAP.md new file mode 100644 index 00000000..68fe6651 --- /dev/null +++ b/docs/BATHYMETRY_EXPANSION_ROADMAP.md @@ -0,0 +1,131 @@ +# Bathymetry expansion roadmap + +## Purpose + +GeoIntel must distinguish three different questions: + +1. Where were cross-sections measured and what does the source document say? +2. What is the continuous elevation of the bed at a specific survey epoch? +3. What is the water depth or volume at a specific moment? + +Only the first question is operational for Mol through the VHA cross-section +profile layer. A bed model does not provide water depth without a compatible +water-surface elevation. Flood-hazard maximum depth is a scenario result and +must not be reused as current water level. + +## Governed source matrix + +| Source | Coverage | Data | Vertical reference | GeoIntel status | +| --- | --- | --- | --- | --- | +| VMM VHA Digital Atlas | Flanders | Point locations, structured profile fields, PDF evidence | Document-specific | Operational, bounded vector acquisition | +| MDK Belgian Continental Shelf model | Belgian North Sea | Continuous 20 x 20 m bathymetric raster | LAT | Available, not integrated | +| SPW navigable waterways and reservoir lakes | Wallonia | 0.5 m bed-elevation raster and XYZ cloud | mDNG | Available, not integrated | +| Port of Antwerp-Bruges publications | Port survey areas | Periodic soundings | Product-specific | Catalog candidate | + +VHA contains approximately 129,643 profile points across Flanders at the +observed catalog state. This is a scale indication, not a fixed contractual +count. Mol contained 828 exact in-boundary points during source validation, +715 with document links and 112 with a structured depth field. Production +provisioning always records the live counts and checksums. + +## Operational tier 1: Mol + +- Query only an explicit EPSG:4326 bbox. +- Intersect the bbox with the exact persisted Mol Area. +- Page the official ArcGIS FeatureServer response without truncation. +- Resolve VHA watercourse names from the official atlas layer. +- Normalize profile points to EPSG:4326. +- Persist the artifact through `DatasetService.import_vector_bytes`. +- Persist every point through `VectorFeatureService`; the provider never + writes directly to `vector_features`. +- Expose document, measurement date, depth and width fields without parsing or + inventing values from scanned PDFs. +- Keep volume unsupported. + +## Tier 2: all of Flanders + +Flanders must be provisioned as exact municipality or other approved Area +partitions, not as one monolithic request. The backend feature limit protects +the provider and the application. A regional logical layer may group complete +partitions, but each Dataset retains its Area id, query URLs, checksums, exact +count and measurement-date range. + +Before regional activation: + +- add a governed Flanders boundary manifest and partition coordinator; +- prove idempotent resume and no duplicate VHA `OBJECTID` within a partition; +- benchmark PostGIS point selection and viewport delivery; +- add freshness/version probing for the VHA MapServer; +- keep individual profile dates instead of fabricating one Dataset + `observed_at`. + +## Tier 3: Belgium + +Belgian coverage is a federation of source adapters with one normalized +contract, not one assumed national dataset: + +- Flanders: VHA profiles and future validated bed rasters; +- Wallonia: SPW bathymetry for measured navigable waterways/reservoirs; +- Brussels: hydrological context until an authoritative public bathymetric + product is identified; +- federal/maritime: MDK and legally appropriate maritime boundaries. + +Every adapter must emit: + +- authority and owner; +- exact geographic and temporal coverage; +- horizontal and vertical CRS/datum; +- survey/acquisition time; +- resolution or sample density; +- source URL, request identity, checksum, attribution and license; +- explicit supported and unsupported metrics. + +LAT, TAW and mDNG values must never be merged or compared without a documented, +tested vertical transformation and uncertainty statement. + +## Tier 4: the Belgian North Sea + +The map must distinguish: + +- the Belgian land boundary and baseline; +- the territoriale zee (up to 12 nautical miles); +- the Belgian EEZ and continental shelf, which are jurisdictional maritime + zones and should not be labelled ordinary municipal or provincial + "grondgebied". + +The MDK bathymetry WCS is the preferred continuous source candidate. Activation +requires a live Docker validation of TLS/certificates, GetCapabilities, +coverage identifiers, bounded GeoTIFF retrieval, CRS, LAT, nodata, pixel size, +response limits and maritime clipping. WMTS can support visual context but is +not the analytical source. + +## Depth and volume rules + +For a compatible bed raster and water-surface raster at the same time and +vertical datum: + +`volume_m3 = sum(max(0, water_surface_z - bed_z) * cell_area_m2)` + +For surveyed cross-sections along a connected reach: + +`volume_m3 = sum(((section_area_i + section_area_i+1) / 2) * reach_length_i)` + +The second method requires complete profile geometry, ordered chainage, +contemporaneous water level and defensible interpolation. VHA profile points +alone do not satisfy those prerequisites. + +Historical evolution compares only survey epochs with documented compatible +coverage, datum and method. A changed raster footprint is not automatically +bed evolution. + +## Implementation order + +1. Operate and validate the Mol VHA profile Dataset and map flow. +2. Add VHA municipal partition orchestration for Flanders. +3. Implement a bounded MDK WCS probe, then acquisition behind live evidence. +4. Implement SPW download staging and vertical-datum metadata validation. +5. Add maritime boundaries as separate authoritative scope layers. +6. Add cross-source vertical-datum transformation only with authoritative + grids/parameters and uncertainty tests. +7. Add volume only after a compatible measured or modeled water-surface source + is part of the same analysis contract. diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index e30abd08..c949510c 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -10010,3 +10010,27 @@ Validation: stays within its 375 px client width; the theme inventory is bounded to a compact scrollable selector so it no longer pushes the map behind all 15 theme cards. + +## Sprint 235 - Governed bathymetry profiles and Belgian expansion model (2026-07-17) + +Implemented: +- Audited official VHA, MDK Belgian Continental Shelf and SPW Walloon + bathymetry services and separated profile evidence, continuous bed models + and time-specific water depth/volume. +- Added bounded, paged VHA profile acquisition with exact persisted-Area + clipping, official watercourse names, document links, checksums and standard + Dataset/VectorFeature persistence. +- Added reusable vector-byte import and governed `min`/`max` property metrics + to the existing selection aggregation path. +- Added the bathymetry source and acquisition API, environment controls, Mol + operator command, map theme, profile inspector and source inventory. +- Documented partitioned Flanders scaling and federated Belgium/maritime + scaling with explicit territorial sea, EEZ/continental shelf and + TAW/LAT/mDNG semantics. + +Validation: +- Backend compilation, 914 backend tests, documentation/contract audits, + Alembic head `202607160001`, frontend TypeScript typecheck and the production + Vite build passed in the complete readiness gate. +- Full readiness and live Mol acceptance are recorded after final validation + and deployment below. diff --git a/docs/DATABASE_IMPLEMENTATION_PLAN.md b/docs/DATABASE_IMPLEMENTATION_PLAN.md index 2e944ebc..326b2961 100644 --- a/docs/DATABASE_IMPLEMENTATION_PLAN.md +++ b/docs/DATABASE_IMPLEMENTATION_PLAN.md @@ -360,3 +360,18 @@ the composite expression index It supports exact preclipped municipality selection without changing the canonical `vector_features` schema or introducing operator-specific tables. Free rectangle queries continue to use the geometry GiST index. + +## Bathymetry profile persistence + +VHA profile points require no new table or migration. The immutable GeoJSON +artifact is stored as one normal reference `datasets` row and +`dataset_versions` row; each exact point is a normal `vector_features` row with +EPSG:4326 geometry and source properties. Existing dataset, source-feature and +GiST indexes support selection. + +Profile measurement dates remain feature properties because a bounded Dataset +can contain many historical campaigns. No artificial Dataset `observed_at` or +temporal series is assigned. A future bed raster remains file/object storage +plus Dataset metadata, not raster-in-database storage. Any national/maritime +extension keeps source vertical datum, survey epoch and Area partition +explicit and does not create a provider-specific shadow schema. diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 569b8ad5..2830f3f7 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -709,3 +709,26 @@ thematic rasters for space occupation, open space, population density, node value and service level. The digital soil map should follow the existing canonical vector persistence path. Watercourse/runoff additions remain in the roadmap but no longer precede these cross-domain gaps. + +## Bathymetry, inland profiles and maritime scope + +The official VHA Digital Atlas profile-point layer is the first operational +bathymetry-adjacent source. GeoIntel requests an explicit bbox, clips against +the exact persisted Area and retains VHA point identifiers, watercourse names, +profile numbers, measurement dates, available structured depth/width values +and official document URLs. Scanned documents remain evidence; missing fields +are not filled by fabricated OCR output. + +The following sources are audited but not yet operational: + +- MDK Dieptemodel Belgisch Continentaal Plat/Noordzee: 20 x 20 m continuous + raster in LAT, exposed through WCS/WMTS. +- SPW bathymetry of navigable waterways and reservoir lakes: 0.5 m bed + elevation and XYZ data in mDNG. +- Port of Antwerp-Bruges periodic soundings: catalog candidate pending a + stable public machine contract. + +See `docs/BATHYMETRY_EXPANSION_ROADMAP.md`. TAW, LAT and mDNG remain separate +until an authoritative vertical transformation is implemented and tested. +The territorial sea, EEZ and continental shelf are separate scope layers and +must be labelled according to their legal meaning. diff --git a/docs/DATA_SPECIFICATION.md b/docs/DATA_SPECIFICATION.md index c7f923b3..5e686cef 100644 --- a/docs/DATA_SPECIFICATION.md +++ b/docs/DATA_SPECIFICATION.md @@ -438,3 +438,22 @@ Sprint 7B makes provider architecture operationally visible without performing l - status: `configured` - dataset mapping: `dataset_role=reference`, `source_name=fixture` - write path: checked-in demo/test fixture flow + +## Bathymetry profile vector contract + +The operational VHA layer is an EPSG:4326 point FeatureCollection. Required +normalized properties are: + +- `provider_record_id` and `source_feature_id`; +- `watercourse_vhag`, `watercourse_name` and alternative names; +- `profile_number` and `measurement_date`; +- nullable `recorded_depth_m`, `recorded_crown_width_m` and + `recorded_floor_width_m`; +- nullable allowlisted `source_document_url`; +- `document_available`, `structured_depth_available`; +- `measurement_semantics=historical_cross_section_profile_point`; +- `vertical_reference=document-specific`. + +Null means the provider did not expose a structured value. It is never +converted to zero. Dataset metadata records exact counts, measurement range, +scope, attribution and `volume_supported=false`. diff --git a/docs/TODO.md b/docs/TODO.md index 25c10f86..e5449fd2 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -749,3 +749,18 @@ This file now starts with the current implementation status. Older preparation/b - [x] Keep orthophoto pixel refresh manual through a separate plan-stage-review-apply flow that retains the passed preflight identity and creates a new immutable Dataset with official `YYYY.NN` source version. +# Bathymetry follow-up + +- [ ] Add governed municipality partition orchestration for all of Flanders + after the Mol VHA operator passes live acceptance. +- [ ] Add read-only MDK WCS capability/TLS probe and maritime scope metadata. +- [ ] Add bounded MDK GeoTIFF acquisition only after live CRS, LAT, nodata and + response-limit validation. +- [ ] Add SPW staged-download adapter with mDNG metadata and survey-epoch + coverage validation. +- [ ] Add authoritative territorial-sea, EEZ and continental-shelf boundary + layers with legally accurate labels. +- [ ] Add vertical-datum conversion only when authoritative transforms and + uncertainty tests exist; never merge TAW, LAT and mDNG implicitly. +- [ ] Add water volume only when bed and water-surface inputs share a governed + time, datum and coverage contract. diff --git a/frontend/README.md b/frontend/README.md index 29e21e13..3df831b7 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -635,3 +635,16 @@ validation remain inside the collapsed follow-up list with an explicit priority and examples of the measurements they should eventually support. Theme/time-series detail and active-source limitations are also collapsed by default so the Data workspace stays readable. + +## Bathymetry profile map theme + +Ready `vmm_vha_bathymetry_profiles` Datasets activate the `Waterbodem` map +theme. The layer renders persisted profile points through the existing +MapLibre/GeoJSON path. Clicking a point shows the watercourse, profile number, +measurement date, nullable registered depth and an allowlisted official VHA +profile-document link. + +The UI always labels the points as historical cross-sections. It does not +present them as a continuous bed raster and does not calculate water volume. +The Sources workspace lists the future MDK North Sea and SPW Walloon +bathymetry integrations as planned until backend acquisition is operational. diff --git a/frontend/src/components/datasets/SourceCatalogPanel.tsx b/frontend/src/components/datasets/SourceCatalogPanel.tsx index 953c590b..b70872c6 100644 --- a/frontend/src/components/datasets/SourceCatalogPanel.tsx +++ b/frontend/src/components/datasets/SourceCatalogPanel.tsx @@ -19,6 +19,7 @@ const THEME_LABELS: Record = { nature_value: 'Natuurwaarde', agriculture: 'Landbouw', water: 'Water', + bathymetry: 'Waterbodem en dwarsprofielen', flood_hazard: 'Overstromingsgevaar', elevation: 'Hoogte en reliëf', roads: 'Wegen en transport', @@ -81,6 +82,9 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E ) const dhmvDatasets = ready.filter((dataset) => dataset.source_name === 'digitaal_vlaanderen_dhmv') const floodHazardDatasets = ready.filter((dataset) => dataset.source_name === 'vmm_flood_hazard') + const bathymetryProfileDatasets = ready.filter( + (dataset) => dataset.source_name === 'vmm_vha_bathymetry_profiles', + ) const latestBuildingsRegister = [...buildingsRegisterDatasets].sort( (left, right) => new Date(right.observed_at ?? 0).getTime() - new Date(left.observed_at ?? 0).getTime(), )[0] @@ -184,7 +188,7 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E - {waterinfoDatasets.length > 0 || historicalOrthophotos.length > 0 || bwkDatasets.length > 0 || agricultureDatasets.length > 0 || buildingsRegisterDatasets.length > 0 || dhmvDatasets.length > 0 || floodHazardDatasets.length > 0 ? ( + {waterinfoDatasets.length > 0 || historicalOrthophotos.length > 0 || bwkDatasets.length > 0 || agricultureDatasets.length > 0 || buildingsRegisterDatasets.length > 0 || dhmvDatasets.length > 0 || floodHazardDatasets.length > 0 || bathymetryProfileDatasets.length > 0 ? (
Actieve broncollecties en hun beperkingen
@@ -244,6 +248,18 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E

Gemodelleerde maximumdiepte per kansscenario. Geen actuele waterstand, bathymetrie of permanent watervolume.

) : null} + {bathymetryProfileDatasets.length > 0 ? ( +
+ VHA-dwarsprofielen waterbodem + + {bathymetryProfileDatasets.reduce( + (total, dataset) => total + Number(dataset.feature_count ?? 0), + 0, + ).toLocaleString('nl-BE')} profielpunten · historische meetcampagnes + +

Meetvelden en officiële profielbladen per punt. Geen continue actuele bathymetrie of volume zonder een gelijktijdig waterpeil.

+
+ ) : null}
) : null} diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index c4c646a5..709d9da7 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -36,7 +36,7 @@ const DEFAULT_SELECTED_FEATURE_FILENAME = 'selected-feature.geojson' const DEFAULT_AREA_SELECTION_FILENAME = 'area-selection.geojson' const EMPTY_TEMPORAL_SERIES: DatasetCreateResponse[] = [] -type DataThemeId = 'buildings' | 'space_occupation' | 'open_space' | 'population' | 'forest' | 'nature_value' | 'agriculture' | 'soil' | 'water' | 'flood_hazard' | 'elevation' | 'accessibility' | 'services' | 'roads' | 'parcels' +type DataThemeId = 'buildings' | 'space_occupation' | 'open_space' | 'population' | 'forest' | 'nature_value' | 'agriculture' | 'soil' | 'water' | 'bathymetry' | 'flood_hazard' | 'elevation' | 'accessibility' | 'services' | 'roads' | 'parcels' interface DataTheme { id: DataThemeId @@ -116,6 +116,13 @@ const DATA_THEMES: DataTheme[] = [ description: 'Waterlopen, grachten, kanalen en wateroppervlakken.', tokens: ['waterways', 'waterway', 'water', 'hydro', 'river', 'stream', 'canal', 'waterloop'], }, + { + id: 'bathymetry', + label: 'Waterbodem', + shortLabel: 'Dwarsprofielen', + description: 'Officiële historische VHA-dwarsprofielen met meetvelden en brondocumenten.', + tokens: ['bathymetry', 'bathymetry_profiles', 'dwarsprofielen', 'waterbodem'], + }, { id: 'flood_hazard', label: 'Overstroming', @@ -170,6 +177,7 @@ const DATA_THEME_MAP_STYLES: Record agriculture: { fill: '#7b8f32', line: '#53671d' }, soil: { fill: '#9a7040', line: '#6f4c27' }, water: { fill: '#2676a8', line: '#155b85' }, + bathymetry: { fill: '#0e7490', line: '#164e63' }, flood_hazard: { fill: '#1597c2', line: '#075985' }, elevation: { fill: '#a57a4b', line: '#315f59' }, accessibility: { fill: '#0f766e', line: '#115e59' }, @@ -188,6 +196,11 @@ function datasetAvailabilityLabel(dataset: DatasetCreateResponse, partitionCount const resolution = Number(dataset.source_metadata?.['analysis_resolution_m']) return `${Number.isFinite(resolution) ? `${resolution.toLocaleString('nl-BE')} m` : 'Raster'} overstromingsscenario${regionalSuffix}` } + if (dataset.source_name === 'vmm_vha_bathymetry_profiles') { + const profiles = dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0 + const documents = Number(dataset.source_metadata?.['document_count'] ?? 0) + return `${profiles.toLocaleString('nl-BE')} profielen · ${documents.toLocaleString('nl-BE')} bronbladen` + } if (dataset.dataset_type === 'raster' && dataset.source_name === 'department_omgeving_thematic_raster') { const resolution = Number(dataset.source_metadata?.['analysis_resolution_m']) const year = Number(dataset.source_metadata?.['observation_year']) @@ -221,6 +234,9 @@ function datasetMatchesTheme(dataset: DatasetCreateResponse, theme: DataTheme): if (dataset.source_name === 'vmm_flood_hazard') { return theme.id === 'flood_hazard' } + if (dataset.source_name === 'vmm_vha_bathymetry_profiles') { + return theme.id === 'bathymetry' + } if (dataset.source_name === 'digitaal_vlaanderen_dhmv') { return theme.id === 'elevation' } @@ -310,6 +326,7 @@ function pickThemeDataset( (dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register' ? 5_000_000 : 0) + (dataset.source_name === 'digitaal_vlaanderen_dhmv' ? 5_000_000 : 0) + (dataset.source_name === 'vmm_flood_hazard' ? 5_000_000 : 0) + + (dataset.source_name === 'vmm_vha_bathymetry_profiles' ? 5_000_000 : 0) + (dataset.source_metadata?.['product_key'] === 'dtm_1m' ? 1_000_000 : 0) + (dataset.source_metadata?.['product_key'] === 'pluviaal_current_t100' ? 1_000_000 : 0) + (dataset.dataset_role === 'reference' ? 10_000 : 0) @@ -618,6 +635,12 @@ export function MapWorkspace({ const selectedMapArea = areas.find((area) => area.id === selectedMapAreaId) const regionalScopeSelected = Boolean(selectedMapArea && !isMunicipalityAreaName(selectedMapArea.name)) const featureProperties = selectedMapFeature?.properties ?? null + const isBathymetryProfile = featureProperties?.['provider'] === 'vmm_vha_bathymetry_profiles' + || featureProperties?.['measurement_semantics'] === 'historical_cross_section_profile_point' + const bathymetryDocumentUrl = typeof featureProperties?.['source_document_url'] === 'string' + && featureProperties['source_document_url'].startsWith('https://vha.waterinfo.be/') + ? featureProperties['source_document_url'] + : null const featureSummaryEntries = featureProperties ? Object.entries(featureProperties) .filter(([, value]) => value !== null && value !== undefined && typeof value !== 'object') @@ -2555,6 +2578,41 @@ export function MapWorkspace({ {selectedMapFeature ? ( <> + {isBathymetryProfile ? ( +
+
+ Waterloop + {String(featureProperties?.['watercourse_name'] ?? 'Onbekende waterloop')} +
+
+ Profiel + {String(featureProperties?.['profile_number'] ?? 'n.v.t.')} +
+
+ Meetdatum + {String(featureProperties?.['measurement_date'] ?? 'Niet geregistreerd')} +
+
+ Geregistreerde diepte + + {typeof featureProperties?.['recorded_depth_m'] === 'number' + ? `${featureProperties['recorded_depth_m'].toLocaleString('nl-BE')} m` + : 'Niet als veld beschikbaar'} + +
+ {bathymetryDocumentUrl ? ( + + Officieel profielblad openen + + ) : ( + Voor dit meetpunt is geen digitaal profielblad gekoppeld. + )} +

+ Historisch dwarsprofiel. Dit punt is geen continue actuele bodemkaart en levert zonder + gelijktijdig waterpeil geen actueel watervolume. +

+
+ ) : null}
Geometrie diff --git a/frontend/src/lib/datasetDisplay.ts b/frontend/src/lib/datasetDisplay.ts index 04a610cc..4662006a 100644 --- a/frontend/src/lib/datasetDisplay.ts +++ b/frontend/src/lib/datasetDisplay.ts @@ -4,6 +4,7 @@ const DATASET_LABEL_BY_LAYER: Record = { buildings: 'Bebouwing', roads: 'Wegen', water: 'Water', + bathymetry_profiles: 'Dwarsprofielen waterbodem', parcels: 'Percelen', population: 'Bevolking', forest: 'Bos en groen', @@ -34,6 +35,7 @@ const DATASET_SOURCE_LABELS: Record = { vrbg: 'Digitaal Vlaanderen', waterinfo: 'Waterinfo Vlaanderen', vmm_flood_hazard: 'Vlaamse Milieumaatschappij', + vmm_vha_bathymetry_profiles: 'VMM / Vlaamse Hydrografische Atlas', department_omgeving_thematic_raster: 'Departement Omgeving', dov_soil_map: 'Databank Ondergrond Vlaanderen', } @@ -52,6 +54,9 @@ export function getDatasetDisplayName(dataset: DatasetCreateResponse): string { const productName = dataset.source_metadata?.['product_display_name'] return typeof productName === 'string' && productName.trim() ? productName : 'VMM-overstromingsscenario' } + if (dataset.source_name === 'vmm_vha_bathymetry_profiles') { + return 'VHA-dwarsprofielen waterbodem' + } if (dataset.source_name === 'department_omgeving_thematic_raster') { const productName = dataset.source_metadata?.['product_display_name'] return typeof productName === 'string' && productName.trim() ? productName : 'Officieel Vlaams themaraster' diff --git a/frontend/src/lib/sourcePortfolio.ts b/frontend/src/lib/sourcePortfolio.ts index 3569442c..400a336d 100644 --- a/frontend/src/lib/sourcePortfolio.ts +++ b/frontend/src/lib/sourcePortfolio.ts @@ -384,6 +384,42 @@ export const OFFICIAL_SOURCE_PORTFOLIO: OfficialSourceDefinition[] = [ url: 'https://www.vlaanderen.be/datavindplaats/catalogus/ogc-api-features-vlaamse-hydrografische-atlas-waterlopen', matches: (dataset) => sourceNameIs(dataset, 'vmm_vha'), }, + { + key: 'vha_bathymetry_profiles', + domain: 'climate', + name: 'VHA-dwarsprofielen waterbodem', + owner: 'Vlaamse Milieumaatschappij', + coverage: 'Historische profielpunten op Vlaamse waterlopen', + value: 'Gemeten profielinformatie en officiële bronbladen op exacte kaartlocaties.', + metricExamples: 'aantal profielen, geregistreerde diepte en breedte per meetpunt', + priority: 'next', + url: 'https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer', + matches: (dataset) => sourceNameIs(dataset, 'vmm_vha_bathymetry_profiles'), + }, + { + key: 'mdk_bcp_bathymetry', + domain: 'climate', + name: 'Dieptemodel Belgisch Continentaal Plat', + owner: 'Agentschap Maritieme Dienstverlening en Kust', + coverage: 'Belgische Noordzee, 20 x 20 m raster in LAT', + value: 'Continue zeebodemhoogte voor mariene selectie en evolutie.', + metricExamples: 'minimum/maximum diepte, diepteklassen en bodemvolume per peilcampagne', + priority: 'planned', + url: 'https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee', + matches: (dataset) => sourceNameIs(dataset, 'mdk_bcp_bathymetry'), + }, + { + key: 'spw_waterway_bathymetry', + domain: 'climate', + name: 'Waalse vaarweg- en stuwmeerbathymetrie', + owner: 'Service public de Wallonie', + coverage: 'Gemeten Waalse vaarwegen en stuwmeren, 0,5 m in mDNG', + value: 'Hoogwaardige bodemrasters voor federale uitbreiding buiten Vlaanderen.', + metricExamples: 'bodemhoogte, diepteprofiel en vergelijkbare meetcampagnes', + priority: 'planned', + url: 'https://geoportail.wallonie.be/catalogue/c450c28f-d357-48af-8423-62d524632cf9.html', + matches: (dataset) => sourceNameIs(dataset, 'spw_walloon_waterway_bathymetry'), + }, ] export function operationalSources( diff --git a/frontend/src/services/api/datasets.ts b/frontend/src/services/api/datasets.ts index 06f7e968..a7b8cb66 100644 --- a/frontend/src/services/api/datasets.ts +++ b/frontend/src/services/api/datasets.ts @@ -22,6 +22,8 @@ import type { FloodHazardAcquireRequest, FloodHazardProductRead, FloodHazardSelectionResponse, + BathymetryProfileAcquireRequest, + BathymetrySourceRead, DhmvProductRead, TerrainSelectionResponse, ThematicRasterAcquireRequest, @@ -167,6 +169,12 @@ export const datasetsApi = { payload: { bbox: VectorSelectionRequest['bbox']; area_id?: string; product_key: string }, ): Promise => apiPost(`/api/v1/projects/${projectId}/datasets/raster/flood-hazard/select`, payload), + listBathymetrySources: (projectId: string): Promise<{ items: BathymetrySourceRead[]; total: number }> => + apiGet<{ items: BathymetrySourceRead[]; total: number }>( + `/api/v1/projects/${projectId}/datasets/bathymetry/sources`, + ), + acquireBathymetryProfiles: (projectId: string, payload: BathymetryProfileAcquireRequest): Promise => + apiPost(`/api/v1/projects/${projectId}/datasets/bathymetry/profiles/acquire`, payload), acquireThematicRaster: (projectId: string, payload: ThematicRasterAcquireRequest): Promise => apiPost(`/api/v1/projects/${projectId}/datasets/thematic-raster/acquire`, payload), listThematicRasterProducts: (projectId: string): Promise<{ items: ThematicRasterProductRead[]; total: number }> => diff --git a/frontend/src/styles/app.css b/frontend/src/styles/app.css index 1f509211..14b5eab8 100644 --- a/frontend/src/styles/app.css +++ b/frontend/src/styles/app.css @@ -5966,6 +5966,11 @@ section { background: rgba(38, 118, 168, 0.24); } +.geo-map-legend .geo-legend-layer-bathymetry { + background: #0e7490; + border-color: #164e63; +} + .geo-map-legend .geo-legend-layer-soil { border-color: #6f4c27; background: rgba(154, 112, 64, 0.24); @@ -6036,6 +6041,60 @@ section { .geo-legend-ramp-population { background: linear-gradient(90deg, #eee7f6, #673a97); } .geo-legend-ramp-accessibility { background: linear-gradient(90deg, #e9f1f4, #0f766e); } .geo-legend-ramp-services { background: linear-gradient(90deg, #fff4bf, #b66d16); } +.geo-legend-ramp-bathymetry { background: linear-gradient(90deg, #cffafe, #0e7490); } + +.bathymetry-profile-summary { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 0.65rem; + margin-bottom: 0.9rem; + padding: 0.8rem; + border: 1px solid #a5d8e6; + border-radius: 6px; + background: #effbff; +} + +.bathymetry-profile-summary > div { + display: grid; + gap: 0.2rem; + min-width: 0; +} + +.bathymetry-profile-summary span, +.bathymetry-profile-summary small { + color: #46646f; + font-size: 0.75rem; +} + +.bathymetry-profile-summary strong { + overflow-wrap: anywhere; + color: #113d4b; +} + +.bathymetry-profile-summary a, +.bathymetry-profile-summary p, +.bathymetry-profile-summary small { + grid-column: 1 / -1; +} + +.bathymetry-profile-summary a { + width: fit-content; + color: #075985; + font-weight: 700; +} + +.bathymetry-profile-summary p { + margin: 0; + color: #315867; + font-size: 0.78rem; + line-height: 1.45; +} + +@media (max-width: 760px) { + .bathymetry-profile-summary { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} .geo-map-legend .geo-legend-added { border-color: #15803d; diff --git a/frontend/src/types.ts b/frontend/src/types.ts index d1c27345..40ced7c3 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -442,6 +442,33 @@ export interface FloodHazardSelectionResponse { generated_at: string } +export interface BathymetryProfileAcquireRequest { + bbox: VectorSelectionBBox + area_id?: string | null + force_refresh?: boolean +} + +export interface BathymetrySourceRead { + key: string + display_name: string + owner: string + authority_level: 'authoritative' | 'contextual' + geographic_coverage: string + data_kind: string + query_modes: string[] + vertical_reference: string + horizontal_crs: string + native_resolution?: string | null + integration_status: 'operational' | 'available_not_integrated' | 'catalog_only' + acquisition_supported: boolean + configured: boolean + service_url?: string | null + catalog_url: string + attribution: string + license_note: string + limitation_message: string +} + export interface ThematicRasterAcquireRequest { bbox: VectorSelectionBBox area_id?: string | null diff --git a/scripts/README.md b/scripts/README.md index 9a5129fc..8eb63663 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1882,3 +1882,30 @@ boundary artifact. Defaults cap each municipality at 30,000 source features and the assembled snapshot at 300,000 features. It never truncates silently, never writes `vector_features` directly and never turns the single 2025 state into a fabricated historical series. + +## Mol VHA bathymetry profiles + +Provision and verify the official profile points for the exact persisted Mol +Area: + +```bash +docker exec geointel python /app/scripts/provision_mol_bathymetry_profiles.py +``` + +Use `--force` only for a deliberate fresh provider snapshot. The command finds +`Mol Municipality Workbench` and `Gemeente Mol`, calls the canonical +bathymetry acquisition endpoint and then verifies that vector selection count +and semantic metrics match the persisted Job result. It never writes directly +to PostGIS. + +For another approved workspace or Area: + +```bash +python scripts/provision_mol_bathymetry_profiles.py \ + --base-url http://127.0.0.1:8000 \ + --project-name "Project name" \ + --area-name "Area name fragment" +``` + +The output is a point dataset with historical evidence. It is not a continuous +water-bottom raster and cannot calculate current water volume. diff --git a/scripts/provision_mol_bathymetry_profiles.py b/scripts/provision_mol_bathymetry_profiles.py new file mode 100644 index 00000000..ce8a23e4 --- /dev/null +++ b/scripts/provision_mol_bathymetry_profiles.py @@ -0,0 +1,154 @@ +"""Provision official VHA cross-section profile points for the persisted Mol Area. + +The operator uses only canonical GeoIntel API endpoints. The backend queries the +official VHA service, clips points against the exact persisted Area geometry, +stores the GeoJSON artifact and persists VectorFeature rows through DatasetService. +""" + +from __future__ import annotations + +import argparse +import json +import os +from typing import Any, Iterable + +import requests + + +DEFAULT_API_URL = "http://127.0.0.1:8000" +DEFAULT_PROJECT_NAME = "Mol Municipality Workbench" +DEFAULT_AREA_FRAGMENT = "Gemeente Mol" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Provision official VHA cross-section profiles for Mol.") + parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL)) + parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME) + parser.add_argument("--area-name", default=DEFAULT_AREA_FRAGMENT) + parser.add_argument("--timeout", type=int, default=900) + parser.add_argument("--force", action="store_true") + return parser.parse_args() + + +def unwrap(response: requests.Response) -> Any: + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict) or "data" not in payload: + raise RuntimeError(f"Non-canonical API response from {response.url}") + return payload["data"] + + +def coordinates(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]: + def walk(value: Any): + if isinstance(value, list) and len(value) >= 2 and all(isinstance(item, (int, float)) for item in value[:2]): + yield float(value[0]), float(value[1]) + return + if isinstance(value, list): + for child in value: + yield from walk(child) + + yield from walk(geometry.get("coordinates", [])) + + +def geometry_bbox(geometry: dict[str, Any]) -> dict[str, float | str]: + points = list(coordinates(geometry)) + if not points: + raise RuntimeError("Persisted Area geometry contains no coordinates") + xs = [point[0] for point in points] + ys = [point[1] for point in points] + return { + "min_x": min(xs), + "min_y": min(ys), + "max_x": max(xs), + "max_y": max(ys), + "crs": "EPSG:4326", + } + + +def main() -> int: + args = parse_args() + base_url = args.base_url.rstrip("/") + session = requests.Session() + session.headers.update({"User-Agent": "GeoIntel-Bathymetry-Operator/1.0"}) + + projects = unwrap(session.get(f"{base_url}/api/v1/projects", params={"limit": 200, "offset": 0}, timeout=60))["items"] + project = next((item for item in projects if item["name"] == args.project_name), None) + if project is None: + raise RuntimeError(f"Project {args.project_name!r} was not found") + + areas = unwrap( + session.get( + f"{base_url}/api/v1/projects/{project['id']}/areas", + params={"limit": 200, "offset": 0}, + timeout=60, + ) + )["items"] + fragment = args.area_name.casefold() + area = next((item for item in areas if fragment in item["name"].casefold()), None) + if area is None: + raise RuntimeError(f"Area containing {args.area_name!r} was not found") + bbox = geometry_bbox(area["geometry"]) + + job = unwrap( + session.post( + f"{base_url}/api/v1/projects/{project['id']}/datasets/bathymetry/profiles/acquire", + json={ + "bbox": bbox, + "area_id": area["id"], + "force_refresh": args.force, + }, + timeout=args.timeout, + ) + ) + if job.get("status") != "success" or not job.get("output_dataset_id"): + raise RuntimeError(f"Bathymetry profile acquisition failed: {job.get('error_message') or job}") + result = job.get("result_json") or {} + dataset_id = job["output_dataset_id"] + selection = unwrap( + session.post( + f"{base_url}/api/v1/projects/{project['id']}/datasets/{dataset_id}/vector/select", + json={"bbox": bbox, "area_id": area["id"], "limit": 1000}, + timeout=args.timeout, + ) + ) + if selection.get("total_feature_count") != result.get("profile_count"): + raise RuntimeError( + "Persisted vector selection count does not match the exact acquired profile count " + f"({selection.get('total_feature_count')} != {result.get('profile_count')})" + ) + metrics = { + item["metric_key"]: item + for item in (selection.get("summary") or {}).get("metrics", []) + if isinstance(item, dict) and item.get("metric_key") + } + if metrics.get("profile_count", {}).get("metric_value") != result.get("profile_count"): + raise RuntimeError("Bathymetry profile summary does not expose the persisted profile count") + + print( + json.dumps( + { + "status": "ok", + "project_id": project["id"], + "area_id": area["id"], + "area_name": area["name"], + "bbox": bbox, + "dataset_id": dataset_id, + "reused": bool(result.get("reused")), + "profile_count": result.get("profile_count"), + "document_count": result.get("document_count"), + "structured_depth_count": result.get("structured_depth_count"), + "watercourse_count": result.get("watercourse_count"), + "measurement_date_min": result.get("measurement_date_min"), + "measurement_date_max": result.get("measurement_date_max"), + "metrics": list(metrics.values()), + "limitation_message": result.get("limitation_message"), + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 5e0d7f82..8f6b38c1 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -59,6 +59,7 @@ ${PYTHON_BIN} -m py_compile scripts/orthophoto_release_preflight.py ${PYTHON_BIN} -m py_compile scripts/manage_orthophoto_release.py ${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py +${PYTHON_BIN} -m py_compile scripts/provision_mol_bathymetry_profiles.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_dhmv.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_flood_hazards.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_flood_hazards.py