diff --git a/CHANGELOG.md b/CHANGELOG.md index 26f77f47..bd4de775 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,29 @@ # Changelog +## National history and governed SPW bathymetry (2026-07-19) + +- Accepted the two documented official Statbel geometry-archive variants: + internal member names with or without the repeated `31370` token and + situation dates in ISO or `YYYY/MM/DD` notation. CRS, release-year, schema, + join, topology, total and checksum validation remain fail-closed. +- Prepared the official Statbel 2021-2024 national snapshots so the existing + 2025 Belgium population layer can become one consistent five-edition + evolution series. +- Added a pinned SPW bathymetry operator for the official 2023-05-23 + 0.5-m GeoTIFF release. It validates SHA-256, archive safety, EPSG:3812, + Float32, nodata and value bounds, reads through `/vsizip/`, creates only a + bounded COG and persists through the canonical Dataset API. +- Added persisted SPW bathymetry selection and PNG routes, with waterbed-height + percentiles in mDNG, surveyed hectares and coverage. Current water depth, + volume and vertical-datum conversion remain explicitly unsupported. +- Integrated ready SPW rasters into the Waterbodem map theme, MapLibre image + overlay, rectangle analysis and source portfolio while excluding analytical + bathymetry rasters from detection imagery. +- Kept MDK North Sea acquisition blocked behind strict TLS and official + low-resolution data-request prerequisites; no certificate bypass or + synthesized North Sea depth was introduced. + ## Post-RC Belgium data federation (2026-07-19) - Made persisted NGI administrative, RBINS marine reporting and Belgian diff --git a/backend/README.md b/backend/README.md index 368e5536..38d63d81 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1806,7 +1806,8 @@ matched name with `--show-names`. performs a bounded official VHA ArcGIS query, exact persisted-Area clipping, watercourse-name normalization and ordinary Dataset/VectorFeature persistence. `GET /api/v1/projects/{project_id}/datasets/bathymetry/sources` reports VHA as -operational and the audited MDK/SPW candidates as unavailable for acquisition. +operational, MDK as probe-only and the pinned SPW raster operator as +operational. Runtime controls are `BATHYMETRY_PROFILES_ENABLED`, `BATHYMETRY_PROFILES_LAYER_URL`, `BATHYMETRY_WATERCOURSE_LAYER_URL`, @@ -1854,6 +1855,29 @@ readiness state such as TLS or endpoint failure. Runtime controls are `MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS` and `MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB`. TLS verification cannot be disabled. +### SPW waterbed raster + +The official 2023-05-23 SPW bathymetry ZIP is integrated only through the +bounded operator. Stage the immutable ZIP under persistent storage and run: + +```bash +docker exec geointel python /app/scripts/import_spw_bathymetry.py \ + --base-url http://127.0.0.1:8000 \ + --project-name "Belgium and North Sea Workbench" \ + --area "RC Golden - Wallonia urban-rural" \ + --bbox 4.85,50.45,4.87,50.47 \ + --raw-zip /app/storage/operator-evidence/spw-bathymetry/2023-05-23/raw/BATHY_50CM_ALTITUDE_DNG_GEOTIFF_3812.zip \ + --output-dir /app/storage/operator-evidence/spw-bathymetry/2023-05-23/derived +``` + +The script validates the pinned official checksum, safe archive members, +EPSG:3812, one Float32 band, approximately 0.5 m cells and nodata `-9999`. +It then creates a bounded COG and uploads it through `/datasets/upload`. +`POST .../raster/bathymetry/select` returns waterbed elevation in mDNG, +surveyed surface and coverage. Current depth, volume and datum conversion stay +unavailable without a compatible water-surface source. Selection analysis is +bounded by `BATHYMETRY_RASTER_MAX_PIXELS` (30 million by default). + ## Governed regional official-vector acquisition The thematic raster registry includes forest and agricultural land-use masks diff --git a/backend/app/api/routes/datasets.py b/backend/app/api/routes/datasets.py index b5ce4613..f2717e82 100644 --- a/backend/app/api/routes/datasets.py +++ b/backend/app/api/routes/datasets.py @@ -48,6 +48,8 @@ from app.schemas import ( FloodHazardSelectionRequest, BathymetryPartitionFinalizeRequest, BathymetryProfileAcquireRequest, + BathymetryRasterSelectionRequest, + BathymetryRasterSelectionResponse, ThematicRasterAcquireRequest, ThematicRasterProductRead, ThematicRasterSelectionResponse, @@ -89,6 +91,7 @@ from app.services.terrain_analysis_service import TerrainAnalysisService from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService +from app.services.bathymetry_raster_analysis_service import BathymetryRasterAnalysisService from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService @@ -888,6 +891,33 @@ def raster_terrain_image( ) +@router.post( + "/datasets/{dataset_id}/raster/bathymetry/select", + response_model=Envelope[BathymetryRasterSelectionResponse], +) +def raster_bathymetry_selection( + project_id: UUID, + dataset_id: UUID, + payload: BathymetryRasterSelectionRequest, + db: Session = Depends(get_db), +): + return envelope(BathymetryRasterAnalysisService.analyze(db, project_id, dataset_id, payload)) + + +@router.get("/datasets/{dataset_id}/raster/bathymetry/image") +def raster_bathymetry_image( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + content = BathymetryRasterAnalysisService.render_png(db, project_id, dataset_id) + return Response( + content=content, + media_type="image/png", + headers={"Cache-Control": "private, max-age=86400"}, + ) + + @router.post( "/datasets/{dataset_id}/raster/flood-hazard/select", response_model=Envelope[FloodHazardSelectionResponse], diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 33ab2959..507aaf91 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -220,6 +220,11 @@ class Settings(BaseSettings): le=256, validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB", ) + bathymetry_raster_max_pixels: int = Field( + default=30_000_000, + ge=1, + validation_alias="BATHYMETRY_RASTER_MAX_PIXELS", + ) mdk_bathymetry_probe_enabled: bool = Field(default=True, validation_alias="MDK_BATHYMETRY_PROBE_ENABLED") mdk_bathymetry_wcs_url: str = Field( default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs", diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index ea8c12a8..7f8ee75f 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -93,6 +93,10 @@ from .bathymetry import ( BathymetryPartitionFinalizationResult, BathymetryProfileAcquireRequest, BathymetryProfileAcquisitionResult, + BathymetryRasterMetric, + BathymetryRasterSelectionRequest, + BathymetryRasterSelectionResponse, + BathymetryRasterSelectionSummary, BathymetrySourceProbeRead, BathymetrySourceRead, ) diff --git a/backend/app/schemas/bathymetry.py b/backend/app/schemas/bathymetry.py index 6b849ea8..b1b51ac2 100644 --- a/backend/app/schemas/bathymetry.py +++ b/backend/app/schemas/bathymetry.py @@ -109,3 +109,43 @@ class BathymetrySourceProbeRead(BaseModel): checked_at: datetime message: str limitation_message: str + + +class BathymetryRasterSelectionRequest(BaseModel): + bbox: VectorSelectionBBox + area_id: UUID | None = None + + +class BathymetryRasterMetric(BaseModel): + metric_key: str + metric_label: str + metric_value: float + metric_unit: str + aggregation_method: str + is_estimate: bool = False + + +class BathymetryRasterSelectionSummary(BaseModel): + metric_label: str + metric_value: float + metric_unit: str + aggregation_method: str + primary_metric_key: str + metrics: list[BathymetryRasterMetric] + + +class BathymetryRasterSelectionResponse(BaseModel): + dataset_id: UUID + product_key: str + selection_bbox: VectorSelectionBBox + selection_area_id: UUID | None = None + selected_cell_count: int = Field(ge=1) + valid_cell_count: int = Field(ge=1) + coverage_ratio: float = Field(ge=0, le=1) + resolution_m: float = Field(gt=0) + vertical_reference: str + survey_period: str + summary: BathymetryRasterSelectionSummary + unsupported_metrics: list[str] + limitation_message: str + generated_at: str diff --git a/backend/app/services/bathymetry_profile_acquisition_service.py b/backend/app/services/bathymetry_profile_acquisition_service.py index 17155969..d4ba7b79 100644 --- a/backend/app/services/bathymetry_profile_acquisition_service.py +++ b/backend/app/services/bathymetry_profile_acquisition_service.py @@ -91,20 +91,21 @@ class BathymetryProfileAcquisitionService: "authority_level": "authoritative", "geographic_coverage": "Waalse bevaarbare waterwegen en stuwmeren met uitgevoerde opmetingen", "data_kind": "bodemhoogteraster en XYZ-puntenwolk", - "query_modes": ["download", "arcgis_map_service"], + "query_modes": ["operator_archive", "bounded_raster", "arcgis_map_service"], "vertical_reference": "mDNG", "horizontal_crs": "EPSG:3812; visualisatieservice kan EPSG:31370 aanbieden", "native_resolution": "0,5 m", - "integration_status": "available_not_integrated", - "acquisition_supported": False, - "configured": False, + "integration_status": "operational", + "acquisition_supported": True, + "configured": True, "service_url": "https://geoservices.wallonie.be/arcgis/rest/services/EAU/BATHY/MapServer", - "catalog_url": "https://geoportail.wallonie.be/catalogue/c450c28f-d357-48af-8423-62d524632cf9.html", + "catalog_url": "https://geoportail.wallonie.be/catalogue/0a544b42-0b30-4c8e-85e7-38149b99eae0.html", "attribution": "Service public de Wallonie", "license_note": "CC BY 4.0 volgens de officiële Geoportail-metadata.", "limitation_message": ( - "Dekking en meetjaar verschillen per vaarweg of reservoir. Integratie vereist een beheerde " - "download- en mosaïekstroom plus expliciete omzetting van mDNG." + "De gepinde officiële release kan begrensd als raster worden geïmporteerd via de operator. " + "Dekking verschilt per vaarweg; de waarden zijn bodemhoogtes in mDNG uit 2019-2022, " + "zonder stilzwijgende datumconversie of afleiding van actuele waterdiepte." ), }, { diff --git a/backend/app/services/bathymetry_raster_analysis_service.py b/backend/app/services/bathymetry_raster_analysis_service.py new file mode 100644 index 00000000..28d7b0ca --- /dev/null +++ b/backend/app/services/bathymetry_raster_analysis_service.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +import io +import math +from datetime import UTC, datetime +from pathlib import Path +from uuid import UUID + +from geoalchemy2.shape import to_shape +from pyproj import Transformer +from shapely.geometry import box, mapping +from shapely.ops import transform as shapely_transform + +from app.core.config import Settings, get_settings +from app.core.errors import AppError +from app.models import Area, Dataset +from app.schemas.bathymetry import ( + BathymetryRasterMetric, + BathymetryRasterSelectionRequest, + BathymetryRasterSelectionResponse, + BathymetryRasterSelectionSummary, +) + + +class BathymetryRasterAnalysisService: + SOURCE_NAME = "spw_bathymetry" + PRODUCT_KEY = "spw_bathymetry_50cm_mdng" + UNSUPPORTED_METRICS = [ + "current_water_depth_m", + "water_volume_m3", + "vertical_datum_conversion", + ] + LIMITATION = ( + "De rasterwaarden zijn waterbodemhoogtes in mDNG uit een samengestelde SPW-opmeting " + "(2019-2022). Zonder een gelijktijdig waterpeil zijn actuele waterdiepte en watervolume " + "niet berekenbaar. mDNG wordt niet stilzwijgend naar TAW, LAT of een ander verticaal datum omgezet." + ) + + @staticmethod + def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> Dataset: + dataset = db.get(Dataset, dataset_id) + if not dataset or dataset.project_id != project_id: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + if dataset.dataset_type != "raster" or dataset.source_name != BathymetryRasterAnalysisService.SOURCE_NAME: + raise AppError( + code="INVALID_BATHYMETRY_RASTER_DATASET", + message="Bathymetry analysis requires a governed SPW bathymetry raster dataset", + status_code=400, + ) + if dataset.status != "ready" or not dataset.storage_path or not Path(dataset.storage_path).is_file(): + raise AppError( + code="DATASET_FILE_MISSING", + message="Persisted bathymetry raster file is unavailable", + status_code=404, + ) + return dataset + + @staticmethod + def _metadata(dataset: Dataset) -> dict: + metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {} + if ( + metadata.get("product_key") != BathymetryRasterAnalysisService.PRODUCT_KEY + or metadata.get("theme") != "bathymetry" + or metadata.get("value_semantics") != "bed_elevation" + or metadata.get("vertical_reference") != "mDNG" + or metadata.get("source_crs") != "EPSG:3812" + ): + raise AppError( + code="INVALID_BATHYMETRY_RASTER_METADATA", + message="Bathymetry raster provenance or value semantics are incomplete", + status_code=409, + ) + return metadata + + @staticmethod + def _selection_geometry(db, project_id: UUID, payload: BathymetryRasterSelectionRequest): + selection = box(payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y) + if payload.area_id is None: + return selection + area = db.get(Area, payload.area_id) + if not area: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + if area.project_id != project_id: + raise AppError( + code="INVALID_DATASET_SCOPE", + message="Area does not belong to this project", + status_code=400, + ) + selection = selection.intersection(to_shape(area.geometry)) + if selection.is_empty or selection.area <= 0: + raise AppError( + code="BATHYMETRY_SELECTION_OUTSIDE_AREA", + message="Selection does not overlap the selected work area", + status_code=422, + ) + return selection + + @staticmethod + def analyze( + db, + project_id: UUID, + dataset_id: UUID, + payload: BathymetryRasterSelectionRequest, + *, + settings: Settings | None = None, + ) -> dict: + resolved_settings = settings or get_settings() + dataset = BathymetryRasterAnalysisService._load_dataset(db, project_id, dataset_id) + source_metadata = BathymetryRasterAnalysisService._metadata(dataset) + selection_4326 = BathymetryRasterAnalysisService._selection_geometry(db, project_id, payload) + try: + import numpy as np + import rasterio + from rasterio.features import geometry_mask + from rasterio.mask import mask + except ImportError as exc: + raise AppError( + code="RASTER_PROCESSING_UNAVAILABLE", + message="Rasterio and numpy are required for bathymetry analysis", + status_code=503, + ) from exc + + try: + with rasterio.open(dataset.storage_path) as source: + if source.crs is None or source.crs.to_epsg() != 3812: + raise AppError( + code="INVALID_DATASET_CRS", + message="SPW bathymetry raster CRS must be EPSG:3812", + status_code=409, + ) + if source.count != 1: + raise AppError( + code="INVALID_BATHYMETRY_RASTER_BANDS", + message="SPW bathymetry requires one bed-elevation band", + status_code=409, + ) + transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True) + selection_metric = shapely_transform(transformer.transform, selection_4326) + analysis_geometry = selection_metric.intersection(box(*source.bounds)) + if analysis_geometry.is_empty or analysis_geometry.area <= 0: + raise AppError( + code="BATHYMETRY_SELECTION_OUTSIDE_DATASET", + message="Selection does not overlap the persisted bathymetry raster", + status_code=422, + ) + min_x, min_y, max_x, max_y = analysis_geometry.bounds + expected_cells = math.ceil((max_x - min_x) / abs(source.res[0])) * math.ceil( + (max_y - min_y) / abs(source.res[1]) + ) + if expected_cells > resolved_settings.bathymetry_raster_max_pixels: + raise AppError( + code="BATHYMETRY_SELECTION_TOO_LARGE", + message="Bathymetry analysis exceeds the configured raster cell limit", + details={ + "pixel_count": expected_cells, + "max_pixels": resolved_settings.bathymetry_raster_max_pixels, + }, + status_code=422, + ) + clipped, clipped_transform = mask( + source, + [mapping(analysis_geometry)], + crop=True, + filled=False, + indexes=[1], + ) + band = np.ma.asarray(clipped[0], dtype="float64") + raw = band.filled(np.nan) + selected_cells = geometry_mask( + [mapping(analysis_geometry)], + out_shape=band.shape, + transform=clipped_transform, + invert=True, + ) + valid_cells = selected_cells & ~np.ma.getmaskarray(band) & np.isfinite(raw) + if source.nodata is not None: + valid_cells &= ~np.isclose(raw, float(source.nodata)) + values = raw[valid_cells] + if values.size == 0: + raise AppError( + code="BATHYMETRY_NO_VALID_DATA", + message="No surveyed waterbed cells occur in this selection", + status_code=422, + ) + resolution_x = abs(float(source.res[0])) + resolution_y = abs(float(source.res[1])) + cell_area_m2 = resolution_x * resolution_y + except AppError: + raise + except Exception as exc: + raise AppError( + code="BATHYMETRY_ANALYSIS_FAILED", + message="The persisted bathymetry raster could not be analysed", + details={"reason": str(exc)}, + status_code=500, + ) from exc + + def metric(key: str, label: str, value: float, unit: str, method: str) -> BathymetryRasterMetric: + return BathymetryRasterMetric( + metric_key=key, + metric_label=label, + metric_value=round(float(value), 4), + metric_unit=unit, + aggregation_method=method, + ) + + selected_cell_count = int(selected_cells.sum()) + valid_cell_count = int(values.size) + vertical_unit = str(source_metadata["vertical_reference"]) + coverage_ratio = valid_cell_count / max(1, selected_cell_count) + metrics = [ + metric( + "bed_elevation_mean_m", + "Gemiddelde waterbodemhoogte", + values.mean(), + f"m {vertical_unit}", + "mean_valid_source_cells", + ), + metric( + "bed_elevation_min_m", + "Laagste waterbodemhoogte", + values.min(), + f"m {vertical_unit}", + "minimum_valid_source_cells", + ), + metric( + "bed_elevation_max_m", + "Hoogste waterbodemhoogte", + values.max(), + f"m {vertical_unit}", + "maximum_valid_source_cells", + ), + metric( + "bed_elevation_p10_m", + "10e percentiel waterbodemhoogte", + np.percentile(values, 10), + f"m {vertical_unit}", + "percentile_10_valid_source_cells", + ), + metric( + "bed_elevation_p90_m", + "90e percentiel waterbodemhoogte", + np.percentile(values, 90), + f"m {vertical_unit}", + "percentile_90_valid_source_cells", + ), + metric( + "surveyed_bed_surface_ha", + "Oppervlakte met gemeten waterbodem", + valid_cell_count * cell_area_m2 / 10_000.0, + "ha", + "valid_source_cells_times_cell_area", + ), + metric( + "bathymetry_coverage_pct", + "Dekking waterbodemmeting", + coverage_ratio * 100.0, + "%", + "valid_source_cells_divided_by_selected_cells", + ), + ] + primary = metrics[0] + response = BathymetryRasterSelectionResponse( + dataset_id=dataset.id, + product_key=BathymetryRasterAnalysisService.PRODUCT_KEY, + selection_bbox=payload.bbox, + selection_area_id=payload.area_id, + selected_cell_count=selected_cell_count, + valid_cell_count=valid_cell_count, + coverage_ratio=round(coverage_ratio, 6), + resolution_m=round(max(resolution_x, resolution_y), 4), + vertical_reference=vertical_unit, + survey_period=str(source_metadata.get("survey_period") or "2019-2022"), + summary=BathymetryRasterSelectionSummary( + metric_label=primary.metric_label, + metric_value=primary.metric_value, + metric_unit=primary.metric_unit, + aggregation_method=primary.aggregation_method, + primary_metric_key=primary.metric_key, + metrics=metrics, + ), + unsupported_metrics=BathymetryRasterAnalysisService.UNSUPPORTED_METRICS, + limitation_message=BathymetryRasterAnalysisService.LIMITATION, + generated_at=datetime.now(UTC).isoformat(), + ) + return response.model_dump(mode="json") + + @staticmethod + def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes: + dataset = BathymetryRasterAnalysisService._load_dataset(db, project_id, dataset_id) + BathymetryRasterAnalysisService._metadata(dataset) + try: + import numpy as np + import rasterio + from PIL import Image + from rasterio.enums import Resampling + except ImportError as exc: + raise AppError( + code="RASTER_PROCESSING_UNAVAILABLE", + message="Rasterio, numpy and Pillow are required for bathymetry rendering", + status_code=503, + ) from exc + + try: + with rasterio.open(dataset.storage_path) as source: + scale = min(1.0, max_dimension / max(source.width, source.height)) + width = max(1, round(source.width * scale)) + height = max(1, round(source.height * scale)) + data = source.read( + 1, + out_shape=(height, width), + masked=True, + resampling=Resampling.bilinear, + ) + values = np.asarray(data.filled(np.nan), dtype="float64") + valid = np.isfinite(values) & ~np.ma.getmaskarray(data) + if source.nodata is not None: + valid &= ~np.isclose(values, float(source.nodata)) + if not valid.any(): + raise AppError( + code="BATHYMETRY_NO_VALID_DATA", + message="Bathymetry raster contains no renderable cells", + status_code=422, + ) + low, high = np.percentile(values[valid], [2, 98]) + if high <= low: + high = low + 1.0 + normalized = np.clip((values - low) / (high - low), 0.0, 1.0) + normalized = np.where(valid, normalized, 0.0) + stops = np.asarray([0.0, 0.35, 0.7, 1.0]) + colors = np.asarray( + [ + [8, 47, 73], + [15, 118, 140], + [103, 190, 170], + [236, 224, 163], + ], + dtype="float64", + ) + rgba = np.zeros((height, width, 4), dtype="uint8") + for channel in range(3): + rgba[:, :, channel] = np.interp( + normalized, + stops, + colors[:, channel], + ).astype("uint8") + rgba[:, :, 3] = np.where(valid, 220, 0).astype("uint8") + output = io.BytesIO() + Image.fromarray(rgba).save(output, format="PNG", optimize=True) + return output.getvalue() + except AppError: + raise + except Exception as exc: + raise AppError( + code="BATHYMETRY_PREVIEW_FAILED", + message="The persisted bathymetry raster could not be rendered", + details={"reason": str(exc)}, + status_code=500, + ) from exc diff --git a/backend/app/services/coverage_registry_service.py b/backend/app/services/coverage_registry_service.py index a63cc5d1..cd4d8214 100644 --- a/backend/app/services/coverage_registry_service.py +++ b/backend/app/services/coverage_registry_service.py @@ -233,11 +233,11 @@ SOURCE_DEFINITIONS = ( attribution="Service public de Wallonie", license_note="Consult the license of each Geoportail Wallonie product.", limitation_message=( - "Bounded PICC buildings, road axes and hydrography are operational; " + "Bounded PICC buildings, road axes, hydrography and operator-imported SPW bathymetry are operational; " "other Walloon themes remain unavailable until separately governed." ), - materialized_source_names=("spw_picc",), - operational_themes=("buildings", "roads", "surface_water"), + materialized_source_names=("spw_picc", "spw_bathymetry"), + operational_themes=("buildings", "roads", "surface_water", "bathymetry"), ), _contract( source_name="urbis", @@ -348,15 +348,16 @@ FLANDERS_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = { "flood_climate": {"vmm_flood_hazard": ()}, } -REGIONAL_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = { +REGIONAL_THEME_DATASETS: dict[str, dict[str, dict[str, tuple[str, ...]]]] = { "spw_geoportail": { - "buildings": ("buildings",), - "roads": ("roads",), - "surface_water": ("water",), + "buildings": {"spw_picc": ("buildings",)}, + "roads": {"spw_picc": ("roads",)}, + "surface_water": {"spw_picc": ("water",)}, + "bathymetry": {"spw_bathymetry": ()}, }, "urbis": { - "buildings": ("buildings",), - "parcels": ("parcels",), + "buildings": {"urbis": ("buildings",)}, + "parcels": {"urbis": ("parcels",)}, }, } @@ -437,10 +438,10 @@ class CoverageRegistryService: continue layer_names = theme_sources[dataset.source_name] elif definition.contract.source_name in REGIONAL_THEME_DATASETS: - theme_layers = REGIONAL_THEME_DATASETS[definition.contract.source_name].get(theme) - if theme_layers is None: + theme_sources = REGIONAL_THEME_DATASETS[definition.contract.source_name].get(theme, {}) + if dataset.source_name not in theme_sources: continue - layer_names = theme_layers + layer_names = theme_sources[dataset.source_name] metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {} coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or [] if isinstance(coverage_zones, str): diff --git a/backend/tests/test_rc4_national_coverage.py b/backend/tests/test_rc4_national_coverage.py index e6342882..4c8dbeee 100644 --- a/backend/tests/test_rc4_national_coverage.py +++ b/backend/tests/test_rc4_national_coverage.py @@ -221,6 +221,53 @@ def test_bounded_api_materialization_only_covers_its_persisted_bbox() -> None: assert outside.items[0].materialized_dataset_ids == [] +def test_spw_bathymetry_materialization_is_source_specific() -> None: + project_id = uuid4() + scope = [ + scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)), + scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8)), + ] + selection = CoverageBBox(minx=4.851, miny=50.451, maxx=4.869, maxy=50.469) + spw_picc = SimpleNamespace( + id=uuid4(), + status="ready", + source_name="spw_picc", + reference_layer_name="buildings", + source_metadata={ + "coverage_zones": ["wallonia"], + "bbox_epsg4326": [4.85, 50.45, 4.87, 50.47], + }, + ) + without_bathymetry = CoverageRegistryService.resolve( + FakeSession(project=SimpleNamespace(id=project_id), areas=scope, datasets=[spw_picc]), + project_id, + selection, + ["bathymetry"], + ) + assert without_bathymetry.items[0].status == "partial" + assert without_bathymetry.items[0].materialized_dataset_ids == [] + + bathymetry_id = uuid4() + bathymetry = SimpleNamespace( + id=bathymetry_id, + status="ready", + source_name="spw_bathymetry", + reference_layer_name=None, + source_metadata={ + "coverage_zones": ["wallonia"], + "bbox_epsg4326": [4.85, 50.45, 4.87, 50.47], + }, + ) + with_bathymetry = CoverageRegistryService.resolve( + FakeSession(project=SimpleNamespace(id=project_id), areas=scope, datasets=[spw_picc, bathymetry]), + project_id, + selection, + ["bathymetry"], + ) + assert with_bathymetry.items[0].status == "operational" + assert with_bathymetry.items[0].materialized_dataset_ids == [bathymetry_id] + + def test_mixed_land_and_north_sea_selection_remains_split() -> None: project_id = uuid4() project = SimpleNamespace(id=project_id) diff --git a/backend/tests/test_sprint241_spw_bathymetry_raster.py b/backend/tests/test_sprint241_spw_bathymetry_raster.py new file mode 100644 index 00000000..a1fb8a57 --- /dev/null +++ b/backend/tests/test_sprint241_spw_bathymetry_raster.py @@ -0,0 +1,270 @@ +from __future__ import annotations + +import importlib.util +import io +from pathlib import Path +import sys +import zipfile +from uuid import uuid4 + +import numpy as np +import pytest +import rasterio +from fastapi.testclient import TestClient +from pyproj import Transformer +from rasterio.io import MemoryFile +from rasterio.transform import from_origin +from shapely.geometry import shape + +from app.core.config import Settings +from app.core.errors import AppError +from app.db.session import get_db +from app.main import app +from app.models import Dataset +from app.schemas.bathymetry import BathymetryRasterSelectionRequest +from app.services.bathymetry_raster_analysis_service import BathymetryRasterAnalysisService + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT_PATH = ROOT / "scripts" / "import_spw_bathymetry.py" + + +def load_operator(): + name = "test_import_spw_bathymetry_sprint241" + spec = importlib.util.spec_from_file_location(name, SCRIPT_PATH) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +OPERATOR = load_operator() + + +class FakeSession: + def __init__(self, rows): + self.rows = rows + + def get(self, model, row_id): + return self.rows.get((model, row_id)) + + +def bathymetry_tiff(*, nodata_only: bool = False) -> bytes: + values = np.linspace(72.0, 80.0, 400, dtype="float32").reshape(20, 20) + values[:, :5] = -9999.0 + if nodata_only: + values[:] = -9999.0 + with MemoryFile() as memory: + with memory.open( + driver="GTiff", + width=20, + height=20, + count=1, + dtype="float32", + crs="EPSG:3812", + transform=from_origin(684_000, 629_000, 0.5, 0.5), + nodata=-9999.0, + ) as output: + output.write(values, 1) + return memory.read() + + +def selection_payload() -> BathymetryRasterSelectionRequest: + transformer = Transformer.from_crs("EPSG:3812", "EPSG:4326", always_xy=True) + min_x, min_y = transformer.transform(684_000, 628_990) + max_x, max_y = transformer.transform(684_010, 629_000) + return BathymetryRasterSelectionRequest( + bbox={ + "min_x": min(min_x, max_x), + "min_y": min(min_y, max_y), + "max_x": max(min_x, max_x), + "max_y": max(min_y, max_y), + "crs": "EPSG:4326", + } + ) + + +def persisted_dataset(path: Path, *, metadata: dict | None = None) -> Dataset: + path.write_bytes(bathymetry_tiff()) + return Dataset( + id=uuid4(), + project_id=uuid4(), + name="spw_bathymetry_test_3812.tif", + dataset_type="raster", + source="SPW official operator archive", + source_name="spw_bathymetry", + source_metadata=metadata + or { + "product_key": "spw_bathymetry_50cm_mdng", + "theme": "bathymetry", + "value_semantics": "bed_elevation", + "vertical_reference": "mDNG", + "source_crs": "EPSG:3812", + "survey_period": "2019-2022", + }, + storage_path=str(path), + status="ready", + ) + + +def test_bathymetry_analysis_returns_real_bed_elevation_and_surface_metrics(tmp_path: Path) -> None: + dataset = persisted_dataset(tmp_path / "bathymetry.tif") + db = FakeSession({(Dataset, dataset.id): dataset}) + + result = BathymetryRasterAnalysisService.analyze( + db, + dataset.project_id, + dataset.id, + selection_payload(), + settings=Settings(_env_file=None, bathymetry_raster_max_pixels=10_000), + ) + + metrics = {item["metric_key"]: item for item in result["summary"]["metrics"]} + assert result["product_key"] == "spw_bathymetry_50cm_mdng" + assert result["vertical_reference"] == "mDNG" + assert result["survey_period"] == "2019-2022" + assert result["selected_cell_count"] == 400 + assert result["valid_cell_count"] == 300 + assert result["coverage_ratio"] == pytest.approx(0.75) + assert metrics["bed_elevation_mean_m"]["metric_unit"] == "m mDNG" + assert metrics["surveyed_bed_surface_ha"]["metric_value"] == pytest.approx(0.0075) + assert metrics["bathymetry_coverage_pct"]["metric_value"] == pytest.approx(75.0) + assert result["unsupported_metrics"] == [ + "current_water_depth_m", + "water_volume_m3", + "vertical_datum_conversion", + ] + + +def test_bathymetry_analysis_fails_closed_for_metadata_size_and_empty_cells(tmp_path: Path) -> None: + dataset = persisted_dataset(tmp_path / "bathymetry.tif") + db = FakeSession({(Dataset, dataset.id): dataset}) + + with pytest.raises(AppError) as size_error: + BathymetryRasterAnalysisService.analyze( + db, + dataset.project_id, + dataset.id, + selection_payload(), + settings=Settings(_env_file=None, bathymetry_raster_max_pixels=100), + ) + assert size_error.value.code == "BATHYMETRY_SELECTION_TOO_LARGE" + + dataset.source_metadata = {"theme": "bathymetry"} + with pytest.raises(AppError) as metadata_error: + BathymetryRasterAnalysisService.analyze( + db, + dataset.project_id, + dataset.id, + selection_payload(), + ) + assert metadata_error.value.code == "INVALID_BATHYMETRY_RASTER_METADATA" + + dataset.source_metadata = { + "product_key": "spw_bathymetry_50cm_mdng", + "theme": "bathymetry", + "value_semantics": "bed_elevation", + "vertical_reference": "mDNG", + "source_crs": "EPSG:3812", + } + Path(dataset.storage_path).write_bytes(bathymetry_tiff(nodata_only=True)) + with pytest.raises(AppError) as empty_error: + BathymetryRasterAnalysisService.analyze( + db, + dataset.project_id, + dataset.id, + selection_payload(), + ) + assert empty_error.value.code == "BATHYMETRY_NO_VALID_DATA" + + +def test_bathymetry_image_and_route_use_persisted_raster_and_canonical_envelope(tmp_path: Path) -> None: + dataset = persisted_dataset(tmp_path / "bathymetry.tif") + db = FakeSession({(Dataset, dataset.id): dataset}) + image = BathymetryRasterAnalysisService.render_png(db, dataset.project_id, dataset.id) + + assert image.startswith(b"\x89PNG\r\n\x1a\n") + + app.dependency_overrides[get_db] = lambda: db + try: + response = TestClient(app).post( + f"/api/v1/projects/{dataset.project_id}/datasets/{dataset.id}/raster/bathymetry/select", + json=selection_payload().model_dump(mode="json"), + ) + finally: + app.dependency_overrides.clear() + assert response.status_code == 200 + payload = response.json() + assert set(payload) == {"data"} + assert payload["data"]["dataset_id"] == str(dataset.id) + assert payload["data"]["summary"]["primary_metric_key"] == "bed_elevation_mean_m" + + +def test_operator_validates_pinned_archive_and_rejects_unsafe_members( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + safe_path = tmp_path / "safe.zip" + with zipfile.ZipFile(safe_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(OPERATOR.SOURCE_MEMBER, bathymetry_tiff()) + monkeypatch.setattr(OPERATOR, "SOURCE_SHA256", OPERATOR.sha256_file(safe_path)) + + member = OPERATOR.validate_archive(safe_path) + + assert member.filename == OPERATOR.SOURCE_MEMBER + + unsafe_path = tmp_path / "unsafe.zip" + with zipfile.ZipFile(unsafe_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr("../escape.txt", "unsafe") + archive.writestr(OPERATOR.SOURCE_MEMBER, bathymetry_tiff()) + monkeypatch.setattr(OPERATOR, "SOURCE_SHA256", OPERATOR.sha256_file(unsafe_path)) + with pytest.raises(OPERATOR.SpwBathymetryImportError, match="unsafe member"): + OPERATOR.validate_archive(unsafe_path) + + +def test_operator_crops_zip_member_to_valid_cog(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + archive_path = tmp_path / "source.zip" + with zipfile.ZipFile(archive_path, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr(OPERATOR.SOURCE_MEMBER, bathymetry_tiff()) + monkeypatch.setattr(OPERATOR, "SOURCE_SHA256", OPERATOR.sha256_file(archive_path)) + member = OPERATOR.validate_archive(archive_path) + output_path = tmp_path / "bounded.tif" + + diagnostics = OPERATOR.crop_source( + archive_path, + member, + shape( + { + "type": "Polygon", + "coordinates": [[ + [selection_payload().bbox.min_x, selection_payload().bbox.min_y], + [selection_payload().bbox.max_x, selection_payload().bbox.min_y], + [selection_payload().bbox.max_x, selection_payload().bbox.max_y], + [selection_payload().bbox.min_x, selection_payload().bbox.max_y], + [selection_payload().bbox.min_x, selection_payload().bbox.min_y], + ]], + } + ), + output_path, + max_pixels=10_000, + ) + + with rasterio.open(output_path) as output: + assert output.crs.to_epsg() == 3812 + assert output.driver == "GTiff" + assert output.nodata == -9999.0 + assert output.profile["tiled"] is True + assert diagnostics["valid_cell_count"] == 300 + assert len(diagnostics["output_sha256"]) == 64 + + +def test_operator_is_api_only_and_does_not_claim_depth_or_volume() -> None: + source = SCRIPT_PATH.read_text(encoding="utf-8") + + assert "/datasets/upload" in source + assert "water_depth_available" in source + assert '"water_volume_available": False' in source + assert "SessionLocal" not in source + assert "db.add(" not in source diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index fbaa2f86..79f71769 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -100,6 +100,7 @@ COPY scripts/provision_mol_bathymetry_profiles.py /app/scripts/provision_mol_bat COPY scripts/provision_flanders_geographic_scope.py /app/scripts/provision_flanders_geographic_scope.py COPY scripts/provision_flanders_bathymetry_profiles.py /app/scripts/provision_flanders_bathymetry_profiles.py COPY scripts/probe_mdk_bathymetry.py /app/scripts/probe_mdk_bathymetry.py +COPY scripts/import_spw_bathymetry.py /app/scripts/import_spw_bathymetry.py COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index b254ec3a..e40f5ea5 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -2192,9 +2192,9 @@ sets an explicit Ollama context window and returns Returns the governed bathymetry source registry in the canonical envelope. VHA inland profiles are `operational`. MDK Belgian Continental Shelf is -`probe_only`; SPW Walloon bathymetry remains `available_not_integrated`. -Neither source can be acquired until its raster/download and vertical-datum -flow passes live validation. +`probe_only`. The pinned SPW Walloon bathymetry archive is `operational` +through a bounded, explicit operator import. There is no browser-side source +fetch and no arbitrary source URL. ### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness` @@ -2261,6 +2261,31 @@ request limit, while `total_feature_count` and all configured depth/width metrics are calculated across the complete spatial result. Empty municipalities return an empty, honest result. No provider request occurs during analysis. +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/bathymetry/select` + +Runs a bounded selection against a ready persisted +`source_name=spw_bathymetry` raster. The request contains an EPSG:4326 bbox +and optional persisted `area_id`, matching the other governed raster-selection +contracts. + +The response reports: + +- mean, minimum, maximum, p10 and p90 waterbed elevation in `m mDNG`; +- exact raster-cell surface with surveyed bed values in hectares; +- source coverage percentage inside the selected geometry; +- source resolution, survey period `2019-2022`, vertical reference and + explicit unsupported metrics. + +`current_water_depth_m`, `water_volume_m3` and vertical-datum conversion remain +unsupported. The endpoint reads the persisted EPSG:3812 geometry-aligned +raster; it does not query SPW and does not synthesize missing cells. + +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/bathymetry/image` + +Returns a transparent PNG rendering of the persisted bathymetry COG for the +MapLibre image-overlay path. Rendering changes presentation only. Analytical +values always come from the stored Float32 source cells. + Future provider output continues to use DatasetService and, for vectors, VectorFeatureService. Arbitrary service URLs, browser-side fetches, insecure TLS bypasses and startup downloads remain forbidden. diff --git a/docs/BATHYMETRY_EXPANSION_ROADMAP.md b/docs/BATHYMETRY_EXPANSION_ROADMAP.md index c77f4bce..0af28504 100644 --- a/docs/BATHYMETRY_EXPANSION_ROADMAP.md +++ b/docs/BATHYMETRY_EXPANSION_ROADMAP.md @@ -8,10 +8,11 @@ GeoIntel must distinguish three different questions: 2. What is the continuous elevation of the bed at a specific survey epoch? 3. What is the water depth or volume at a specific moment? -Only the first question is operational for Mol through the VHA cross-section -profile layer. A bed model does not provide water depth without a compatible -water-surface elevation. Flood-hazard maximum depth is a scenario result and -must not be reused as current water level. +The first question is operational for Flanders through VHA cross-section +profiles. The second is now operational for bounded, surveyed Walloon +waterways through the official SPW raster. A bed model does not provide water +depth without a compatible water-surface elevation. Flood-hazard maximum depth +is a scenario result and must not be reused as current water level. ## Governed source matrix @@ -19,7 +20,7 @@ must not be reused as current water level. | --- | --- | --- | --- | --- | | VMM VHA Digital Atlas | Flanders | Point locations, structured profile fields, PDF evidence | Document-specific | Operational, bounded vector acquisition | | MDK Belgian Continental Shelf model | Belgian North Sea | Continuous 20 x 20 m bathymetric raster | LAT | Available, not integrated | -| SPW navigable waterways and reservoir lakes | Wallonia | 0.5 m bed-elevation raster and XYZ cloud | mDNG | Available, not integrated | +| SPW navigable waterways and reservoir lakes | Wallonia | 0.5 m bed-elevation raster and XYZ cloud | mDNG | Operational, pinned operator archive and bounded COG | | Port of Antwerp-Bruges publications | Port survey areas | Periodic soundings | Product-specific | Catalog candidate | VHA contains approximately 129,643 profile points across Flanders at the @@ -135,7 +136,10 @@ bed evolution. **Probe implemented. Acquisition blocked because the live endpoint fails strict hostname validation and does not expose usable capabilities.** 4. Implement SPW download staging and vertical-datum metadata validation. + **Done: pinned official ZIP, safe `/vsizip/` access, bounded COG, + DatasetService persistence, map overlay and mDNG selection metrics.** 5. Add maritime boundaries as separate authoritative scope layers. + **Done for the Belgian territorial sea, EEZ and continental shelf.** 6. Add cross-source vertical-datum transformation only with authoritative grids/parameters and uncertainty tests. 7. Add volume only after a compatible measured or modeled water-surface source diff --git a/docs/BUILD_STATUS.md b/docs/BUILD_STATUS.md index f134e4e1..f27bf461 100644 --- a/docs/BUILD_STATUS.md +++ b/docs/BUILD_STATUS.md @@ -1,53 +1,69 @@ # GeoIntel Build Status -Updated: 2026-07-17 +Updated: 2026-07-19 ## Current state -GeoIntel is an implemented map-first GeoAI workbench running as an all-in-one -Unraid container with embedded PostGIS, FastAPI, React/MapLibre, local -Ollama integration and optional local YOLO/PyTorch inference. +GeoIntel `v1.0.0-rc.1` is an accepted map-first GeoAI workbench for Belgium +and the legally distinct Belgian maritime scopes. It runs as an immutable +all-in-one Unraid image with PostGIS, FastAPI, React/MapLibre, local Ollama +integration and optional local YOLO/PyTorch inference. -The active release program is -`docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md`. +Mol and the Kempen are deep regression areas, not the product boundary. The +national workbench also has deterministic golden journeys for Wallonia, +Brussels, a language-boundary selection, the coast and the Belgian North Sea. -## Product scope +## Release status -- Target: all Belgian land and the separately labelled Belgian territorial - sea, EEZ and continental shelf. -- Existing deep regression references: Mol and the Kempen transport region. -- Required new golden areas: Wallonia, Brussels, a cross-region area, coast - and Belgian North Sea. -- Architecture: federated authoritative providers with one coverage matrix; - no assumption that GRB, PICC, UrbIS and NGI are semantically identical. +There are no open release blockers for the signed `v1.0.0-rc.1` evidence set. +RC-0 through RC-11 are complete. Backup/restore, fresh install, upgrade, +rollback, fail-closed readiness, one Alembic head, API contracts, supply-chain +policy, responsive browser journeys and live PostGIS acceptance are proven. -## Implemented foundation +The active post-RC candidate extends real source coverage. Its repository gate +passes 1,052 backend tests, 22 frontend tests, frontend typecheck/build, +one Alembic head and the complete readiness script. Its remaining acceptance +step is live SPW bathymetry persistence and browser verification before a new +immutable image is called releasable. -- Projects, Areas, Datasets, versions, PostGIS vector features and raster - artifacts. -- Map selection, semantic metrics, historical comparison and exports. -- Governed Flemish sources for buildings, context, orthophotos, terrain, - flooding, soil, nature, agriculture, population and policy rasters. -- Detection persistence, local configured YOLO/PyTorch path, QA/QC and review - evidence. -- Segmentation persistence foundation without fake production inference. -- Local Ollama assistant grounded in persisted evidence. -- Single-container Unraid deployment on port 1202. +## Operational product loop -## Release blockers +- Open one national map without selecting a technical project. +- Select an understandable theme and draw or reuse a bounded Area. +- Resolve governed coverage per Belgian jurisdiction. +- Analyse persisted PostGIS vectors or bounded raster artifacts. +- Show source-appropriate metrics, provenance, time and limitations. +- Compare compatible Statbel 2021-2025 and other governed historical series. +- Export persisted evidence or question it through the local Ollama assistant. +- Run configured local YOLO detection with persisted QA and explicit human + review limitations. -- Current production backup and isolated restore proof. -- Fail-closed readiness and truthful runtime capability reporting. -- Stale job/run reconciliation and correlated exception logging. -- Temporal compatibility guard for detection QA. -- Production secret/configuration parity, immutable images and rollback. -- Full CI, reproducible dependencies and supply-chain evidence. -- Critical response-model typing and real frontend/browser E2E coverage. -- National and maritime scope/providers/golden areas. +## Explicit non-blocking boundaries -## Latest evidence +- MDK analytical North Sea bathymetry remains `not_configured`: the public + endpoint fails strict hostname validation and the official low-resolution + product is currently request-based. TLS and source-integrity checks are not + bypassed. +- SPW supplies governed Walloon waterbed elevation in mDNG, not current water + depth. Water volume remains unsupported until a compatible water-surface, + time, vertical-datum and uncertainty contract exists. +- Building detection is an assisted review workflow. The retained benchmark + still requires independent false-negative review before any claim of + autonomous production accuracy or another training pass. +- Real SAM/YOLO-seg inference, a training studio, LiDAR, production + multi-user authentication and real-time monitoring remain post-V1 scope. +- Additional official datasets may improve coverage, but they are not V1 + completion blockers when the coverage matrix reports absence honestly. -Run: +## Reproduce current repository evidence + +```bash +bash scripts/run_readiness_check.sh +cd backend && python -m alembic upgrade head --sql +bash -n scripts/live_migration_smoke.sh +``` + +Live release evidence: ```bash python scripts/capture_release_evidence.py \ @@ -55,8 +71,3 @@ python scripts/capture_release_evidence.py \ --release-id rc-belgium-north-sea \ --live-base-url http://192.168.10.150:1202 ``` - -The evidence manifest is runtime output and is deliberately ignored by Git. -The first captured baseline reported one Alembic head (`202607160001`) and -reachable live health/capability routes. Health truthfulness is an open RC-2 -blocker until readiness becomes fail-closed. diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 2eef677e..469e783c 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -2,6 +2,28 @@ ### Post-RC national data federation (2026-07-19) +- Continued the national federation pass with official history and + bathymetry. Statbel's real 2021 geometry release exposed two documented + packaging differences: the ZIP member omits the repeated `31370` token and + `dt_situation` uses `YYYY/MM/DD`. The preflight now normalizes only those + forms and retains every existing fail-closed content check. +- The corrected Tower fetch-only pass validated national 2021-2024 snapshots, + each with 19,795 retained sector geometries, before any API persistence. +- Downloaded the fixed official SPW Geoportail bathymetry release + (`0a544b42-0b30-4c8e-85e7-38149b99eae0`) to persistent operator evidence. + SHA-256 is + `04122a1c5cecc7b77be025b580995d024545e82d8310e158b4c112f7fa5f8a6e`. + GDAL verified a 456,893 by 179,437 Float32 EPSG:3812 source at 0.5 m, + nodata `-9999`, with mDNG waterbed elevations. +- Added `import_spw_bathymetry.py`, bounded COG persistence, source-specific + coverage resolution, raster selection/image APIs and frontend Waterbodem + integration. The contract exposes bed elevation, measured-cell hectares and + coverage only; current depth, volume and datum conversion remain blocked. +- Added the bathymetry PNG route to the explicit non-envelope API audit set + after the first full gate correctly rejected its untracked binary response. +- The complete repository gate passed 1,052 backend tests, 22 frontend tests, + frontend typecheck/build, one Alembic head and all readiness checks before + live deployment. - Added explicit administrative and maritime themes for persisted NGI/RBINS reference data and made readiness depend on analyzable themes rather than a raw Dataset count. diff --git a/docs/DATABASE_IMPLEMENTATION_PLAN.md b/docs/DATABASE_IMPLEMENTATION_PLAN.md index f3b5cab1..c4d4a819 100644 --- a/docs/DATABASE_IMPLEMENTATION_PLAN.md +++ b/docs/DATABASE_IMPLEMENTATION_PLAN.md @@ -291,6 +291,15 @@ which creates the ordinary annual `datasets`, `dataset_versions` and year remains idempotent and previous annual snapshots are never updated or deleted. +SPW bathymetry also requires no migration. The immutable official ZIP and +bounded operator COG are filesystem artifacts. A successful API upload creates +an ordinary raster `Dataset` and `DatasetVersion`; raster values stay in the +versioned GeoTIFF rather than PostGIS. `source_metadata` retains EPSG:3812, +mDNG, survey period, source checksum and bounded extent, while +`provenance_metadata` retains the official URL, archive member, derived +checksum and selection geometry/hash. Analysis never writes metric rows and +never converts the vertical datum implicitly. + Definitive ALZ release management uses the same persistence boundary and adds no migration or release table. Catalog planning, staged archive/GeoJSON, crop-code evidence and named review live on the filesystem. Only an approved diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 79cccfbb..17b66a50 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -764,6 +764,13 @@ population queries therefore derive from one reviewed national edition rather than mutually incompatible regional imports. No edition is downloaded or promoted during application startup. +The live national workspace contains one consistent Statbel population series +for 2021-2025. The 2021-2024 geometry archives use the official +`YYYY/MM/DD` situation-date notation and archive member names without a +repeated CRS token; the fail-closed preflight normalizes only those documented +format variants and still validates EPSG:31370, release year, schema, joins, +totals, topology repairs and checksums. + ## Wallonia PICC and Brussels UrbIS The official-vector registry exposes bounded, source-specific regional @@ -792,7 +799,7 @@ profile numbers, measurement dates, available structured depth/width values and official document URLs. Scanned documents remain evidence; missing fields are not filled by fabricated OCR output. -The following sources are audited but not yet operational: +The following sources are audited: - MDK Dieptemodel Belgisch Continentaal Plat/Noordzee: 20 x 20 m continuous raster in LAT, exposed through WCS/WMTS. GeoIntel now has a strict-TLS, @@ -800,8 +807,13 @@ The following sources are audited but not yet operational: `bathy.agentschapmdk.be` presents a certificate for `*.l27powered.eu`. Hostname validation therefore fails with `tls_error`; raster acquisition remains disabled and TLS verification cannot be bypassed. -- SPW bathymetry of navigable waterways and reservoir lakes: 0.5 m bed - elevation and XYZ data in mDNG. +- SPW bathymetry of navigable waterways and reservoir lakes: the official + 2023-05-23 GeoTIFF release is operational through + `scripts/import_spw_bathymetry.py`. The operator pins SHA-256 + `04122a1c5cecc7b77be025b580995d024545e82d8310e158b4c112f7fa5f8a6e`, + validates the EPSG:3812/Float32/0.5 m/-9999 contract, creates only a bounded + COG and persists it through the canonical dataset upload API. Values are + bed elevations in mDNG from surveys composed over 2019-2022. - Port of Antwerp-Bruges periodic soundings: catalog candidate pending a stable public machine contract. diff --git a/docs/POST_RC_DATA_COVERAGE_ROADMAP_BELGIUM_NORTH_SEA.md b/docs/POST_RC_DATA_COVERAGE_ROADMAP_BELGIUM_NORTH_SEA.md index 7e3c9ea8..5321e0ed 100644 --- a/docs/POST_RC_DATA_COVERAGE_ROADMAP_BELGIUM_NORTH_SEA.md +++ b/docs/POST_RC_DATA_COVERAGE_ROADMAP_BELGIUM_NORTH_SEA.md @@ -97,7 +97,7 @@ applicable live bounded journey fails. ## P1 - National Statbel population -**State: complete; reviewed 2025 national edition materialized live.** +**State: complete; reviewed 2021-2025 national series materialized live.** ### Work @@ -122,10 +122,17 @@ applicable live bounded journey fails. ### Live evidence -- Reviewed dataset: `ee0a46e5-6139-4b0a-93b5-9dabb3b3dfc7`. -- 20,781 persisted sectors and all 565 municipalities are represented. -- National reconciliation: 11,825,551 inhabitants, including 7,654 explicitly - retained as unlocated rather than silently assigned to geometry. +- Reviewed immutable datasets: + - 2021: `27b2aaba-6ec4-42d8-91c3-9085456b5e7a`; + - 2022: `312e8278-5552-4dd0-abf7-7e1f17d2562a`; + - 2023: `72bfb8a7-9ecb-462d-a622-f8cf693a3cfb`; + - 2024: `e7846fc6-85d1-4120-9442-e62a5b1cd346`; + - 2025: `ee0a46e5-6139-4b0a-93b5-9dabb3b3dfc7`. +- The 2021-2024 editions each retain 19,795 sector geometries. The 2025 + REDEGEO edition retains 20,781 sectors and represents all 565 municipalities. +- The 2025 national reconciliation reports 11,825,551 inhabitants, including + 7,654 explicitly retained as unlocated rather than silently assigned to + geometry. - A bounded Mol selection returns an explicitly labelled area-weighted estimate. ## P2 - Wallonia bounded topographic baseline @@ -207,7 +214,8 @@ capabilities probe before implementation is marked operational. ## P4 - Maritime and bathymetry hardening -**State: complete with MDK acquisition visibly blocked by strict-TLS evidence.** +**State: repository-complete; SPW live acceptance is the final candidate gate +and MDK acquisition remains visibly blocked by strict-TLS evidence.** ### Work @@ -218,6 +226,9 @@ capabilities probe before implementation is marked operational. semantics, edition, licence and LAT evidence pass. - Keep depth relative to LAT separate from water-surface elevation and volume. - SPW mDNG bathymetry remains a separate Walloon inland/navigation source. +- The bounded SPW operator stages the fixed official GeoTIFF archive, validates + its checksum and raster identity, writes a clipped COG through the canonical + Dataset flow and reports only waterbed elevation and surveyed coverage. ### Exit diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 50238f57..b17cf513 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -142,6 +142,23 @@ Dataset/DatasetVersion plus PostGIS `vector_features` through the canonical persistence services. Temporary standalone preflight output does not create a database record and may be removed explicitly after operator review. +SPW bathymetry follows the same immutable-source/derived-dataset separation: + +```text +storage/operator-evidence/spw-bathymetry/2023-05-23/ + raw/BATHY_50CM_ALTITUDE_DNG_GEOTIFF_3812.zip + derived/spw_bathymetry_{selection_hash}_3812.tif +``` + +The raw ZIP is pinned by official URL and SHA-256 and is never extracted as an +8+ GiB working tree. Rasterio/GDAL reads the official GeoTIFF through +`/vsizip/`; the operator writes a bounded compressed COG and uploads that COG +through DatasetService. The queryable file therefore remains an ordinary +versioned raster Dataset. Source URL, archive/member checksum, selection +geometry/hash, survey period, EPSG:3812 and mDNG semantics are retained in +source/provenance metadata. PNG map overlays are derived responses and are not +authoritative storage. + Governed release-decision evidence is separate from the source artifacts: ```text diff --git a/docs/TODO.md b/docs/TODO.md index fb914761..242f9beb 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -100,9 +100,11 @@ geen open productroadmap meer. Bewuste, niet-blokkerende grenzen: -- Watervolume blijft niet beschikbaar zolang geen gekoppelde bathymetrie, - bodemhoogte en onzekerheidsmodel bestaan. Dit is geospatiale correctheid, - geen onafgewerkte UI. +- De begrensde SPW-rasterflow maakt Waalse waterbodemhoogte in mDNG + analyseerbaar. Watervolume blijft niet beschikbaar zolang geen compatibele + wateroppervlaktehoogte, tijdskoppeling, datumtransformatie en + onzekerheidsmodel bestaan. Dit is geospatiale correctheid, geen onafgewerkte + UI. - Gebouwdetectie is operationeel maar blijft een controlekandidaat; verdere training gebeurt pas na nieuwe, onafhankelijke reviewdata. - Een echt SAM/YOLO-seg-model en een trainingsstudio blijven post-V1. De @@ -129,7 +131,10 @@ Bewuste, niet-blokkerende grenzen: - [x] Expand the modern 2013-2025 land-use operator with water, built-function and transport surfaces from the retained official raster. - [x] Add a source inventory that separates loaded data from audited official follow-up sources. - [x] Add a local Ollama question window grounded in persisted GeoIntel metrics and installed server models. -- [ ] Add a governed depth/bathymetry source before exposing water volume; never infer volume from 2D GRB water geometry. +- [x] Add a governed bounded SPW bed-elevation source with explicit mDNG and + survey-period semantics; never infer depth or volume from 2D water geometry. +- [ ] Add a compatible water-surface, time, vertical-datum and uncertainty + contract before exposing water volume. - [x] Integrate governed Waterinfo/VMM annual station series without presenting point measurements as area-wide water volume. - [x] Add bounded historical orthophoto acquisition for official 1971-2025 products with a map overlay and no current-GRB QA on old imagery. - [x] Add BWK/Natura 2000 through an explicit provider/operator contract. @@ -811,9 +816,9 @@ This file now starts with the current implementation status. Older preparation/b - [ ] Add bounded MDK GeoTIFF acquisition only after live CRS, LAT, nodata and response-limit validation. Current official endpoint is blocked by TLS hostname mismatch and unavailable capabilities; never bypass verification. -- [ ] Add SPW staged-download adapter with mDNG metadata and survey-epoch +- [x] Add SPW staged-download adapter with mDNG metadata and survey-epoch coverage validation. -- [ ] Add authoritative territorial-sea, EEZ and continental-shelf boundary +- [x] Add authoritative territorial-sea, EEZ and continental-shelf boundary layers with legally accurate labels. - [ ] Add vertical-datum conversion only when authoritative transforms and uncertainty tests exist; never merge TAW, LAT and mDNG implicitly. diff --git a/frontend/README.md b/frontend/README.md index f0607417..fdb752f2 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -695,8 +695,12 @@ The UI always labels the points as historical cross-sections. It does not present them as a continuous bed raster and does not calculate water volume. Municipality partitions remain hidden for a regional Area until their backend manifest reports complete coverage. The Sources workspace lists MDK North Sea -as read-only probe-only and SPW Walloon bathymetry as planned until bounded -raster/download acquisition is operational. +as read-only probe-only. Ready `spw_bathymetry` rasters activate the same +`Waterbodem` theme for Wallonia with a bounded MapLibre image overlay and +selection metrics for mDNG waterbed height, measured surface and source +coverage. SPW import remains an explicit backend operator workflow; the +browser never downloads the source archive. Missing water level, depth, +volume and vertical-datum conversion remain clearly unsupported. For a complete regional manifest, the theme card totals all data-bearing municipality partitions instead of displaying one representative partition. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 98d10746..27afc7fd 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -212,7 +212,10 @@ function App(): JSX.Element { const thematicRasters = datasets.filter( (dataset) => dataset.dataset_type === 'raster' && dataset.source_name === 'department_omgeving_thematic_raster' && dataset.status === 'ready', ) - return [...vectors, ...terrain, ...floodHazards, ...thematicRasters] + const bathymetryRasters = datasets.filter( + (dataset) => dataset.dataset_type === 'raster' && dataset.source_name === 'spw_bathymetry' && dataset.status === 'ready', + ) + return [...vectors, ...terrain, ...floodHazards, ...thematicRasters, ...bathymetryRasters] }, [datasets], ) @@ -694,12 +697,17 @@ function App(): JSX.Element { ? FLANDERS_WORKSPACE_LABEL : selectedProject?.name ?? 'Geen werkruimte' const areaContextLabel = selectedArea?.name ?? (areas.length > 0 ? 'Kies een gebied' : 'Geen gebied') - const bathymetryContextActive = Boolean( + const bathymetryProfileContextActive = Boolean( activeWorkspace === 'map' && selectedDataset?.source_name === 'vmm_vha_bathymetry_profiles', ) + const bathymetryRasterContextActive = Boolean( + activeWorkspace === 'map' + && selectedDataset?.source_name === 'spw_bathymetry', + ) + const bathymetryContextActive = bathymetryProfileContextActive || bathymetryRasterContextActive const regionalBathymetryContextActive = Boolean( - bathymetryContextActive + bathymetryProfileContextActive && selectedDataset?.source_metadata?.regional_partitions_complete === true && selectedArea && !/^Gemeente\s/i.test(selectedArea.name), @@ -744,9 +752,11 @@ function App(): JSX.Element { const layerContextLabel = activeWorkspace === 'map' && mapContextLayerLabel ? mapContextLayerLabel : bathymetryContextActive - ? regionalBathymetryContextActive - ? `${bathymetryProfileCount.toLocaleString('nl-BE')} profielen · ${regionalBathymetryPartitions.length} gemeenten` - : `${bathymetryProfileCount.toLocaleString('nl-BE')} profielen` + ? bathymetryRasterContextActive + ? `${Number(selectedDataset?.source_metadata?.analysis_resolution_m ?? 0.5).toLocaleString('nl-BE')} m raster · mDNG · ${String(selectedDataset?.source_metadata?.survey_period ?? '2019-2022')}` + : regionalBathymetryContextActive + ? `${bathymetryProfileCount.toLocaleString('nl-BE')} profielen · ${regionalBathymetryPartitions.length} gemeenten` + : `${bathymetryProfileCount.toLocaleString('nl-BE')} profielen` : mapFeatureCollection ? `${mapFeatureCount.toLocaleString('nl-BE')} objecten` : workspaceDataLoading diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index 56c8465e..c9f0a241 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -9,6 +9,7 @@ import { TemporalTrendChart } from './TemporalTrendChart' import { terrainImageUrl } from '../../lib/terrainImage' import { floodHazardImageUrl } from '../../lib/floodHazardImage' import { thematicRasterImageUrl } from '../../lib/thematicRaster' +import { bathymetryRasterImageUrl } from '../../lib/bathymetryRaster' import { FLANDERS_WORKSPACE_PROJECT_NAME } from '../../config/primaryFocus' import { MAP_ANALYSIS_BUDGET_MS, @@ -306,6 +307,11 @@ function datasetAvailabilityLabel( const resolution = Number(dataset.source_metadata?.['analysis_resolution_m']) return `${Number.isFinite(resolution) ? `${resolution.toLocaleString('nl-BE')} m` : 'Raster'} overstromingsscenario${regionalSuffix}` } + if (dataset.dataset_type === 'raster' && dataset.source_name === 'spw_bathymetry') { + const resolution = Number(dataset.source_metadata?.['analysis_resolution_m']) + const period = String(dataset.source_metadata?.['survey_period'] ?? '2019-2022') + return `${Number.isFinite(resolution) ? `${resolution.toLocaleString('nl-BE')} m` : 'Raster'} waterbodemhoogte · ${period}` + } if (dataset.source_name === 'vmm_vha_bathymetry_profiles') { const profiles = partitions.reduce( (total, item) => total + (item.feature_count ?? item.vector_summary?.feature_count ?? 0), @@ -362,6 +368,9 @@ function datasetMatchesTheme(dataset: DatasetCreateResponse, theme: DataTheme): if (dataset.source_name === 'vmm_vha_bathymetry_profiles') { return theme.id === 'bathymetry' } + if (dataset.source_name === 'spw_bathymetry') { + return theme.id === 'bathymetry' + } if (dataset.source_name === 'digitaal_vlaanderen_dhmv') { return theme.id === 'elevation' } @@ -492,6 +501,7 @@ function pickThemeDataset( (dataset.source_name === 'digitaal_vlaanderen_dhmv' ? 5_000_000 : 0) + (dataset.source_name === 'vmm_flood_hazard' ? 5_000_000 : 0) + (dataset.source_name === 'vmm_vha_bathymetry_profiles' ? 5_000_000 : 0) + + (dataset.source_name === 'spw_bathymetry' ? 5_100_000 : 0) + (dataset.source_metadata?.['product_key'] === 'dtm_1m' ? 1_000_000 : 0) + (dataset.source_metadata?.['product_key'] === 'pluviaal_current_t100' ? 1_000_000 : 0) + (dataset.dataset_role === 'reference' ? 10_000 : 0) @@ -594,6 +604,9 @@ function formatDatasetObservation(dataset: DatasetCreateResponse): string { ? `historische profielen ${firstMeasurement}-${lastMeasurement}` : 'historische profielmetingen' } + if (dataset.source_name === 'spw_bathymetry') { + return `samengestelde waterbodemmeting ${String(dataset.source_metadata?.['survey_period'] ?? '2019-2022')} · mDNG` + } if (dataset.source_name === 'department_omgeving_thematic_raster') { const observationYear = Number(dataset.source_metadata?.['observation_year']) if (Number.isFinite(observationYear)) { @@ -1206,17 +1219,33 @@ export function MapWorkspace({ : [], [activeThemeDataset, selectedProjectId, thematicRasterBounds], ) + const bathymetryRasterBounds = activeThemeDataset?.source_name === 'spw_bathymetry' + ? activeThemeDataset.source_metadata?.['bbox_epsg4326'] + : null + const bathymetryRasterImageOverlays = useMemo( + () => activeThemeDataset?.source_name === 'spw_bathymetry' && selectedProjectId && Array.isArray(bathymetryRasterBounds) && bathymetryRasterBounds.length === 4 + ? [{ + url: bathymetryRasterImageUrl(selectedProjectId, activeThemeDataset.id), + bbox: bathymetryRasterBounds.map(Number) as [number, number, number, number], + label: 'Waterbodemhoogte in mDNG', + opacity: 0.86, + }] + : [], + [activeThemeDataset, bathymetryRasterBounds, selectedProjectId], + ) const thematicLegendMin = String(activeThemeDataset?.source_metadata?.['legend_min_label'] ?? 'Lagere waarde') const thematicLegendMax = String(activeThemeDataset?.source_metadata?.['legend_max_label'] ?? 'Hogere waarde') const activeImageOverlays = useMemo( - () => thematicRasterImageOverlays.length > 0 - ? thematicRasterImageOverlays - : floodHazardImageOverlays.length > 0 + () => bathymetryRasterImageOverlays.length > 0 + ? bathymetryRasterImageOverlays + : thematicRasterImageOverlays.length > 0 + ? thematicRasterImageOverlays + : floodHazardImageOverlays.length > 0 ? floodHazardImageOverlays : terrainImageOverlays.length > 0 ? terrainImageOverlays : orthophotoImageOverlay ? [orthophotoImageOverlay] : [], - [floodHazardImageOverlays, orthophotoImageOverlay, terrainImageOverlays, thematicRasterImageOverlays], + [bathymetryRasterImageOverlays, floodHazardImageOverlays, orthophotoImageOverlay, terrainImageOverlays, thematicRasterImageOverlays], ) const municipalityAreaCount = areas.filter((area) => /^Gemeente\s/i.test(area.name)).length const themeTemporalSeriesMap = useMemo( diff --git a/frontend/src/hooks/useMapSelectionExtract.ts b/frontend/src/hooks/useMapSelectionExtract.ts index 1afbb3f4..41f3aec4 100644 --- a/frontend/src/hooks/useMapSelectionExtract.ts +++ b/frontend/src/hooks/useMapSelectionExtract.ts @@ -5,6 +5,7 @@ import type { DatasetCreateResponse, VectorSelectionBBox, VectorSelectionRespons import { terrainSelectionToMapSelection } from '../lib/terrainSelection' import { floodHazardSelectionToMapSelection } from '../lib/floodHazardSelection' import { thematicRasterSelectionToMapSelection } from '../lib/thematicRaster' +import { bathymetryRasterSelectionToMapSelection } from '../lib/bathymetryRaster' interface MapSelectionExtractOptions { selectedProjectId: string | null @@ -44,7 +45,8 @@ export function useMapSelectionExtract({ const terrainDataset = selectedDataset.dataset_type === 'raster' && selectedDataset.source_name === 'digitaal_vlaanderen_dhmv' const floodHazardDataset = selectedDataset.dataset_type === 'raster' && selectedDataset.source_name === 'vmm_flood_hazard' const thematicRasterDataset = selectedDataset.dataset_type === 'raster' && selectedDataset.source_name === 'department_omgeving_thematic_raster' - if (!isVectorDatasetType(selectedDataset.dataset_type) && !terrainDataset && !floodHazardDataset && !thematicRasterDataset) { + const bathymetryRasterDataset = selectedDataset.dataset_type === 'raster' && selectedDataset.source_name === 'spw_bathymetry' + if (!isVectorDatasetType(selectedDataset.dataset_type) && !terrainDataset && !floodHazardDataset && !thematicRasterDataset && !bathymetryRasterDataset) { setMapSelectionError('Gebiedsanalyse ondersteunt een vectorlaag of een beheerd thematisch raster.') return null } @@ -70,6 +72,11 @@ export function useMapSelectionExtract({ bbox: { ...bbox, crs: 'EPSG:4326' }, area_id: areaId, })) + : bathymetryRasterDataset + ? bathymetryRasterSelectionToMapSelection(await datasetsApi.selectBathymetryRaster(selectedProjectId, selectedDataset.id, { + bbox: { ...bbox, crs: 'EPSG:4326' }, + area_id: areaId, + })) : await datasetsApi.selectVectorFeatures(selectedProjectId, selectedDataset.id, { bbox: { ...bbox, crs: 'EPSG:4326' }, area_id: areaId, diff --git a/frontend/src/hooks/useMapThemeSelectionInsights.ts b/frontend/src/hooks/useMapThemeSelectionInsights.ts index 98619f69..eab33a20 100644 --- a/frontend/src/hooks/useMapThemeSelectionInsights.ts +++ b/frontend/src/hooks/useMapThemeSelectionInsights.ts @@ -5,6 +5,7 @@ import type { DatasetCreateResponse, VectorSelectionBBox, VectorSelectionRespons import { terrainSelectionToMapSelection } from '../lib/terrainSelection' import { floodHazardSelectionToMapSelection } from '../lib/floodHazardSelection' import { thematicRasterSelectionToMapSelection } from '../lib/thematicRaster' +import { bathymetryRasterSelectionToMapSelection } from '../lib/bathymetryRaster' export type MapThemeAcquisitionKind = 'thematic_raster' | 'dhmv' | 'flood_hazard' | 'grb' | 'official_vector' @@ -148,6 +149,11 @@ export function useMapThemeSelectionInsights( bbox, area_id: areaId, })) + : dataset.dataset_type === 'raster' && dataset.source_name === 'spw_bathymetry' + ? bathymetryRasterSelectionToMapSelection(await datasetsApi.selectBathymetryRaster(selectedProjectId, dataset.id, { + bbox, + area_id: areaId, + })) : dataset.source_name === 'vmm_vha_bathymetry_profiles' && partitioned ? await datasetsApi.selectBathymetryProfilePartitions(selectedProjectId, { bbox, diff --git a/frontend/src/lib/bathymetryRaster.test.ts b/frontend/src/lib/bathymetryRaster.test.ts new file mode 100644 index 00000000..d2d63b1b --- /dev/null +++ b/frontend/src/lib/bathymetryRaster.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest' +import { + bathymetryRasterImageUrl, + bathymetryRasterSelectionToMapSelection, +} from './bathymetryRaster' + +describe('bathymetry raster mapping', () => { + it('uses the governed image endpoint', () => { + expect(bathymetryRasterImageUrl('project-1', 'dataset-1')).toBe( + '/api/v1/projects/project-1/datasets/dataset-1/raster/bathymetry/image', + ) + }) + + it('maps persisted bed-elevation metrics without turning them into object counts', () => { + const mapped = bathymetryRasterSelectionToMapSelection({ + dataset_id: 'dataset-1', + product_key: 'spw_bathymetry_50cm_mdng', + selection_bbox: { min_x: 4.85, min_y: 50.45, max_x: 4.87, max_y: 50.47, crs: 'EPSG:4326' }, + selected_cell_count: 100, + valid_cell_count: 25, + coverage_ratio: 0.25, + resolution_m: 0.5, + vertical_reference: 'mDNG', + survey_period: '2019-2022', + summary: { + metric_label: 'Gemiddelde waterbodemhoogte', + metric_value: 74.5, + metric_unit: 'm mDNG', + aggregation_method: 'mean_valid_source_cells', + primary_metric_key: 'bed_elevation_mean_m', + metrics: [{ + metric_key: 'bed_elevation_mean_m', + metric_label: 'Gemiddelde waterbodemhoogte', + metric_value: 74.5, + metric_unit: 'm mDNG', + aggregation_method: 'mean_valid_source_cells', + is_estimate: false, + }], + }, + unsupported_metrics: ['current_water_depth_m', 'water_volume_m3'], + limitation_message: 'Geen gelijktijdig waterpeil.', + generated_at: '2026-07-19T00:00:00Z', + }) + + expect(mapped.feature_count).toBe(25) + expect(mapped.geojson.features).toEqual([]) + expect(mapped.summary?.primary_metric_key).toBe('bed_elevation_mean_m') + expect(mapped.summary?.metric_unit).toBe('m mDNG') + expect(mapped.summary?.is_estimate).toBe(false) + expect(mapped.summary?.warning).toContain('waterpeil') + }) +}) diff --git a/frontend/src/lib/bathymetryRaster.ts b/frontend/src/lib/bathymetryRaster.ts new file mode 100644 index 00000000..7beca57c --- /dev/null +++ b/frontend/src/lib/bathymetryRaster.ts @@ -0,0 +1,29 @@ +import type { BathymetryRasterSelectionResponse, VectorSelectionResponse } from '../types' + +export function bathymetryRasterImageUrl(projectId: string, datasetId: string): string { + return `/api/v1/projects/${projectId}/datasets/${datasetId}/raster/bathymetry/image` +} + +export function bathymetryRasterSelectionToMapSelection( + result: BathymetryRasterSelectionResponse, +): VectorSelectionResponse { + return { + selection_bbox: result.selection_bbox, + selection_area_id: result.selection_area_id, + feature_count: result.valid_cell_count, + total_feature_count: result.valid_cell_count, + limit: 0, + truncated: false, + geojson: { type: 'FeatureCollection', features: [] }, + summary: { + ...result.summary, + feature_count: result.valid_cell_count, + is_estimate: false, + warning: result.limitation_message, + metrics: result.summary.metrics.map((metric) => ({ + ...metric, + is_estimate: false, + })), + }, + } +} diff --git a/frontend/src/lib/datasetCapabilities.ts b/frontend/src/lib/datasetCapabilities.ts index 0a13d4ff..97097248 100644 --- a/frontend/src/lib/datasetCapabilities.ts +++ b/frontend/src/lib/datasetCapabilities.ts @@ -4,6 +4,7 @@ const NON_IMAGERY_RASTER_SOURCES = new Set([ 'department_omgeving_thematic_raster', 'digitaal_vlaanderen_dhmv', 'vmm_flood_hazard', + 'spw_bathymetry', ]) export function isDetectionImageryDataset(dataset: DatasetCreateResponse): boolean { diff --git a/frontend/src/lib/datasetDisplay.ts b/frontend/src/lib/datasetDisplay.ts index 3544a7ed..c80c3e23 100644 --- a/frontend/src/lib/datasetDisplay.ts +++ b/frontend/src/lib/datasetDisplay.ts @@ -36,6 +36,7 @@ const DATASET_SOURCE_LABELS: Record = { waterinfo: 'Waterinfo Vlaanderen', vmm_flood_hazard: 'Vlaamse Milieumaatschappij', vmm_vha_bathymetry_profiles: 'VMM / Vlaamse Hydrografische Atlas', + spw_bathymetry: 'Service public de Wallonie', department_omgeving_thematic_raster: 'Departement Omgeving', dov_soil_map: 'Databank Ondergrond Vlaanderen', } @@ -57,6 +58,9 @@ export function getDatasetDisplayName(dataset: DatasetCreateResponse): string { if (dataset.source_name === 'vmm_vha_bathymetry_profiles') { return 'VHA-dwarsprofielen waterbodem' } + if (dataset.source_name === 'spw_bathymetry') { + return 'SPW waterbodemhoogte 0,5 m' + } if (dataset.source_name === 'department_omgeving_thematic_raster') { const productName = dataset.source_metadata?.['product_display_name'] return typeof productName === 'string' && productName.trim() ? productName : 'Officieel Vlaams themaraster' diff --git a/frontend/src/lib/sourcePortfolio.ts b/frontend/src/lib/sourcePortfolio.ts index 400a336d..e454812f 100644 --- a/frontend/src/lib/sourcePortfolio.ts +++ b/frontend/src/lib/sourcePortfolio.ts @@ -414,11 +414,11 @@ export const OFFICIAL_SOURCE_PORTFOLIO: OfficialSourceDefinition[] = [ name: 'Waalse vaarweg- en stuwmeerbathymetrie', owner: 'Service public de Wallonie', coverage: 'Gemeten Waalse vaarwegen en stuwmeren, 0,5 m in mDNG', - value: 'Hoogwaardige bodemrasters voor federale uitbreiding buiten Vlaanderen.', - metricExamples: 'bodemhoogte, diepteprofiel en vergelijkbare meetcampagnes', - priority: 'planned', - url: 'https://geoportail.wallonie.be/catalogue/c450c28f-d357-48af-8423-62d524632cf9.html', - matches: (dataset) => sourceNameIs(dataset, 'spw_walloon_waterway_bathymetry'), + value: 'Officiële waterbodemhoogtes voor begrensde analyse buiten Vlaanderen.', + metricExamples: 'bodemhoogte, gemeten oppervlakte en brondekking', + priority: 'next', + url: 'https://geoportail.wallonie.be/catalogue/0a544b42-0b30-4c8e-85e7-38149b99eae0.html', + matches: (dataset) => sourceNameIs(dataset, 'spw_bathymetry'), }, ] diff --git a/frontend/src/services/api/datasets.ts b/frontend/src/services/api/datasets.ts index 0f9fd63e..f06ad2b6 100644 --- a/frontend/src/services/api/datasets.ts +++ b/frontend/src/services/api/datasets.ts @@ -23,6 +23,7 @@ import type { FloodHazardProductRead, FloodHazardSelectionResponse, BathymetryProfileAcquireRequest, + BathymetryRasterSelectionResponse, BathymetrySourceProbeRead, BathymetrySourceRead, DhmvProductRead, @@ -206,6 +207,15 @@ export const datasetsApi = { `/api/v1/projects/${projectId}/datasets/bathymetry/profiles/partitions/select`, payload, ), + selectBathymetryRaster: ( + projectId: string, + datasetId: string, + payload: { bbox: VectorSelectionRequest['bbox']; area_id?: string }, + ): Promise => + apiPost( + `/api/v1/projects/${projectId}/datasets/${datasetId}/raster/bathymetry/select`, + payload, + ), acquireThematicRaster: (projectId: string, payload: ThematicRasterAcquireRequest): Promise => apiPost(`/api/v1/projects/${projectId}/datasets/thematic-raster/acquire`, payload), listThematicRasterProducts: (projectId: string): Promise<{ items: ThematicRasterProductRead[]; total: number }> => diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 87eeb93d..08e530a0 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -542,6 +542,30 @@ export interface BathymetrySourceProbeRead { limitation_message: string } +export interface BathymetryRasterSelectionResponse { + dataset_id: string + product_key: 'spw_bathymetry_50cm_mdng' + selection_bbox: VectorSelectionBBox + selection_area_id?: string | null + selected_cell_count: number + valid_cell_count: number + coverage_ratio: number + resolution_m: number + vertical_reference: 'mDNG' + survey_period: string + summary: { + metric_label: string + metric_value: number + metric_unit: string + aggregation_method: string + primary_metric_key: string + metrics: VectorSelectionMetric[] + } + unsupported_metrics: string[] + limitation_message: string + generated_at: string +} + export interface ThematicRasterAcquireRequest { bbox: VectorSelectionBBox area_id?: string | null diff --git a/scripts/README.md b/scripts/README.md index 79d7c19c..4c667ddf 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1984,6 +1984,28 @@ docker exec geointel python /app/scripts/probe_mdk_bathymetry.py This performs only `GetCapabilities`, keeps strict TLS verification enabled and returns exit code `2` for an honest non-ready source. + +## Bounded SPW bathymetry raster + +Download or stage only the official SPW ZIP documented in +`docs/DATA_SOURCES.md`. The operator pins its SHA-256, validates its archive +and raster contract, and imports a bounded COG through the public GeoIntel API: + +```bash +docker exec geointel python /app/scripts/import_spw_bathymetry.py \ + --base-url http://127.0.0.1:8000 \ + --project-name "Belgium and North Sea Workbench" \ + --area "RC Golden - Wallonia urban-rural" \ + --bbox 4.85,50.45,4.87,50.47 \ + --raw-zip /app/storage/operator-evidence/spw-bathymetry/2023-05-23/raw/BATHY_50CM_ALTITUDE_DNG_GEOTIFF_3812.zip \ + --output-dir /app/storage/operator-evidence/spw-bathymetry/2023-05-23/derived +``` + +Use `--dry-run` to validate identity and scope without cropping or +persistence. Use `--download` only when the requested raw path does not yet +exist; download remains strict-TLS and fixed to the official source URL. +Selections above `--max-pixels` fail before raster materialization. + # Release backup and restore proof The release-candidate safety path is host-operated against the running diff --git a/scripts/audit_api_contracts.py b/scripts/audit_api_contracts.py index 150bf761..2c55c5b4 100644 --- a/scripts/audit_api_contracts.py +++ b/scripts/audit_api_contracts.py @@ -16,6 +16,7 @@ ALLOWED_NON_ENVELOPE_ENDPOINTS = { ("GET", "/api/v1/exports/{export_id}/download"), ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/image"), ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/terrain/image"), + ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/bathymetry/image"), ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/flood-hazard/image"), ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/thematic/image"), } diff --git a/scripts/import_spw_bathymetry.py b/scripts/import_spw_bathymetry.py new file mode 100644 index 00000000..6cea5045 --- /dev/null +++ b/scripts/import_spw_bathymetry.py @@ -0,0 +1,629 @@ +"""Validate, crop and import the official SPW bathymetry release. + +The operator is intentionally bounded and fail-closed. It accepts one pinned +official archive, keeps the source checksum in provenance, creates a compact +COG for a requested EPSG:4326 scope and persists it through DatasetService's +canonical upload API. It never writes directly to PostGIS or dataset storage. +""" + +from __future__ import annotations + +import argparse +from datetime import UTC, datetime +from hashlib import sha256 +import json +import math +import mimetypes +import os +from pathlib import Path, PurePosixPath +import tempfile +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen +import uuid +import zipfile + +from pyproj import Transformer +import rasterio +from rasterio.features import geometry_mask +from rasterio.mask import mask +from rasterio.shutil import copy as raster_copy +from shapely.geometry import box, mapping, shape +from shapely.ops import transform as shapely_transform + + +SOURCE_URL = ( + "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/" + "0a544b42-0b30-4c8e-85e7-38149b99eae0/" + "BATHY_50CM_ALTITUDE_DNG_GEOTIFF_3812.zip" +) +CATALOG_URL = ( + "https://geoportail.wallonie.be/catalogue/" + "0a544b42-0b30-4c8e-85e7-38149b99eae0.html" +) +SOURCE_SHA256 = "04122a1c5cecc7b77be025b580995d024545e82d8310e158b4c112f7fa5f8a6e" +SOURCE_MEMBER = "BATHY_50CM_ALTITUDE_DNG.tif" +SOURCE_VERSION = "SPW bathymetry 2023-05-23" +SURVEY_PERIOD = "2019-2022" +SOURCE_CRS = "EPSG:3812" +VERTICAL_REFERENCE = "mDNG" +NODATA = -9999.0 +MAX_ARCHIVE_BYTES = 600 * 1024 * 1024 +MAX_UNCOMPRESSED_BYTES = 10 * 1024 * 1024 * 1024 +MAX_ARCHIVE_MEMBERS = 32 +MAX_COMPRESSION_RATIO = 100.0 +DEFAULT_MAX_PIXELS = 30_000_000 +DEFAULT_PROJECT_NAME = "Belgium and North Sea Workbench" +DEFAULT_API_URL = "http://127.0.0.1:8000" + + +class SpwBathymetryImportError(RuntimeError): + pass + + +def sha256_file(path: Path) -> str: + digest = sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def download_source(path: Path, *, timeout: int) -> None: + request = Request( + SOURCE_URL, + headers={ + "Accept": "application/zip", + "User-Agent": "GeoIntel/1.0 governed-spw-bathymetry-operator", + }, + ) + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".partial") + digest = sha256() + size = 0 + try: + with urlopen(request, timeout=timeout) as response, temporary.open("wb") as output: + if "zip" not in str(response.headers.get("Content-Type") or "").lower(): + raise SpwBathymetryImportError("SPW response is not an official ZIP artifact") + for chunk in iter(lambda: response.read(8 * 1024 * 1024), b""): + size += len(chunk) + if size > MAX_ARCHIVE_BYTES: + raise SpwBathymetryImportError("SPW source archive exceeds the configured size limit") + output.write(chunk) + digest.update(chunk) + except (HTTPError, URLError, TimeoutError, OSError) as exc: + temporary.unlink(missing_ok=True) + raise SpwBathymetryImportError(f"Official SPW source download failed: {exc}") from exc + if digest.hexdigest() != SOURCE_SHA256: + temporary.unlink(missing_ok=True) + raise SpwBathymetryImportError("Downloaded SPW artifact checksum does not match the pinned release") + temporary.replace(path) + + +def validate_archive(path: Path) -> zipfile.ZipInfo: + if not path.is_file(): + raise SpwBathymetryImportError(f"SPW source archive does not exist: {path}") + if path.stat().st_size > MAX_ARCHIVE_BYTES: + raise SpwBathymetryImportError("SPW source archive exceeds the configured size limit") + actual_sha256 = sha256_file(path) + if actual_sha256 != SOURCE_SHA256: + raise SpwBathymetryImportError( + f"SPW source checksum mismatch: expected {SOURCE_SHA256}, received {actual_sha256}" + ) + try: + with zipfile.ZipFile(path) as archive: + members = archive.infolist() + if len(members) > MAX_ARCHIVE_MEMBERS: + raise SpwBathymetryImportError("SPW archive contains too many members") + total_size = 0 + selected: zipfile.ZipInfo | None = None + for member in members: + member_path = PurePosixPath(member.filename) + if member.is_dir(): + continue + if member_path.is_absolute() or ".." in member_path.parts: + raise SpwBathymetryImportError("SPW archive contains an unsafe member path") + total_size += member.file_size + compressed = max(1, member.compress_size) + if member.file_size / compressed > MAX_COMPRESSION_RATIO: + raise SpwBathymetryImportError("SPW archive contains an unsafe compression ratio") + if member_path.name == SOURCE_MEMBER: + selected = member + if total_size > MAX_UNCOMPRESSED_BYTES: + raise SpwBathymetryImportError("SPW archive expands beyond the configured size limit") + if selected is None: + raise SpwBathymetryImportError(f"SPW archive does not contain {SOURCE_MEMBER}") + return selected + except zipfile.BadZipFile as exc: + raise SpwBathymetryImportError("SPW source artifact is not a valid ZIP archive") from exc + + +def parse_bbox(value: str): + try: + coordinates = [float(item.strip()) for item in value.split(",")] + except ValueError as exc: + raise argparse.ArgumentTypeError("bbox must contain four numeric EPSG:4326 coordinates") from exc + if len(coordinates) != 4: + raise argparse.ArgumentTypeError("bbox must contain min_x,min_y,max_x,max_y") + min_x, min_y, max_x, max_y = coordinates + if ( + not all(math.isfinite(item) for item in coordinates) + or min_x >= max_x + or min_y >= max_y + or min_x < -180 + or max_x > 180 + or min_y < -90 + or max_y > 90 + ): + raise argparse.ArgumentTypeError("bbox is not a valid EPSG:4326 extent") + return box(min_x, min_y, max_x, max_y) + + +def unwrap(payload: Any, status_code: int) -> Any: + if not isinstance(payload, dict): + raise SpwBathymetryImportError(f"GeoIntel returned an invalid JSON envelope (HTTP {status_code})") + if status_code >= 400 or payload.get("error"): + error = payload.get("error") if isinstance(payload.get("error"), dict) else {} + message = error.get("message") or f"GeoIntel request failed with HTTP {status_code}" + raise SpwBathymetryImportError(str(message)) + if "data" not in payload: + raise SpwBathymetryImportError("GeoIntel response does not use the canonical data envelope") + return payload["data"] + + +def api_json( + base_url: str, + path: str, + *, + timeout: int, + method: str = "GET", + payload: dict[str, Any] | None = None, +) -> Any: + content = json.dumps(payload).encode("utf-8") if payload is not None else None + request = Request( + f"{base_url.rstrip('/')}{path}", + data=content, + method=method, + headers={ + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "GeoIntel/1.0 governed-spw-bathymetry-operator", + }, + ) + try: + with urlopen(request, timeout=timeout) as response: + status_code = int(response.status) + response_content = response.read() + except HTTPError as exc: + status_code = exc.code + response_content = exc.read() + except (URLError, TimeoutError, OSError) as exc: + raise SpwBathymetryImportError(f"GeoIntel API is unavailable: {exc}") from exc + try: + response_payload = json.loads(response_content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise SpwBathymetryImportError(f"GeoIntel returned non-JSON HTTP {status_code}") from exc + return unwrap(response_payload, status_code) + + +def paged_items(base_url: str, path: str, *, timeout: int) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + offset = 0 + while True: + separator = "&" if "?" in path else "?" + page = api_json( + base_url, + f"{path}{separator}limit=200&offset={offset}", + timeout=timeout, + ) + rows = list(page.get("items") or []) + items.extend(item for item in rows if isinstance(item, dict)) + total = int(page.get("total") or 0) + if not rows or len(items) >= total: + return items + offset += len(rows) + + +def resolve_project( + base_url: str, + project_name: str, + *, + project_id: str | None, + timeout: int, +) -> dict[str, Any]: + if project_id: + return api_json(base_url, f"/api/v1/projects/{project_id}", timeout=timeout) + projects = paged_items(base_url, "/api/v1/projects?status=active", timeout=timeout) + matches = [item for item in projects if str(item.get("name") or "").casefold() == project_name.casefold()] + if len(matches) != 1: + raise SpwBathymetryImportError( + f"Expected one active project named {project_name!r}, found {len(matches)}" + ) + return matches[0] + + +def resolve_area( + base_url: str, + project_id: str, + *, + area_id: str | None, + area_name: str | None, + timeout: int, +) -> dict[str, Any] | None: + if not area_id and not area_name: + return None + areas = paged_items(base_url, f"/api/v1/projects/{project_id}/areas", timeout=timeout) + if area_id: + matches = [item for item in areas if str(item.get("id")) == area_id] + else: + matches = [ + item + for item in areas + if str(area_name).casefold() in str(item.get("name") or "").casefold() + ] + if len(matches) != 1: + raise SpwBathymetryImportError(f"Expected exactly one matching persisted Area, found {len(matches)}") + if not isinstance(matches[0].get("geometry"), dict): + raise SpwBathymetryImportError("Selected Area has no persisted geometry") + return matches[0] + + +def source_path(archive_path: Path, member: zipfile.ZipInfo) -> str: + return f"/vsizip/{archive_path.resolve().as_posix()}/{member.filename}" + + +def selection_hash(selection) -> str: + payload = json.dumps(mapping(selection), sort_keys=True, separators=(",", ":")) + return sha256(payload.encode("utf-8")).hexdigest() + + +def crop_source( + archive_path: Path, + member: zipfile.ZipInfo, + selection_4326, + output_path: Path, + *, + max_pixels: int, +) -> dict[str, Any]: + try: + with rasterio.open(source_path(archive_path, member)) as source: + if source.crs is None or source.crs.to_epsg() != 3812: + raise SpwBathymetryImportError("SPW bathymetry source CRS must be EPSG:3812") + if source.count != 1 or source.dtypes[0] != "float32": + raise SpwBathymetryImportError("SPW bathymetry source must contain one Float32 band") + if source.nodata is None or not math.isclose(float(source.nodata), NODATA): + raise SpwBathymetryImportError("SPW bathymetry source nodata value must be -9999") + if not ( + math.isclose(abs(source.res[0]), 0.5, rel_tol=0.01) + and math.isclose(abs(source.res[1]), 0.5, rel_tol=0.01) + ): + raise SpwBathymetryImportError("SPW bathymetry source resolution must be approximately 0.5 m") + transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True) + selection_metric = shapely_transform(transformer.transform, selection_4326) + selection_metric = selection_metric.intersection(box(*source.bounds)) + if selection_metric.is_empty or selection_metric.area <= 0: + raise SpwBathymetryImportError("Requested scope does not overlap the official SPW bathymetry") + min_x, min_y, max_x, max_y = selection_metric.bounds + expected_pixels = math.ceil((max_x - min_x) / abs(source.res[0])) * math.ceil( + (max_y - min_y) / abs(source.res[1]) + ) + if expected_pixels > max_pixels: + raise SpwBathymetryImportError( + f"Requested scope requires {expected_pixels:,} cells; limit is {max_pixels:,}" + ) + clipped, transform = mask( + source, + [mapping(selection_metric)], + crop=True, + filled=True, + nodata=NODATA, + indexes=[1], + ) + profile = source.profile.copy() + profile.update( + driver="GTiff", + width=clipped.shape[2], + height=clipped.shape[1], + count=1, + transform=transform, + nodata=NODATA, + compress="DEFLATE", + predictor=3, + tiled=True, + blockxsize=512, + blockysize=512, + BIGTIFF="IF_SAFER", + ) + except SpwBathymetryImportError: + raise + except Exception as exc: + raise SpwBathymetryImportError(f"SPW bathymetry raster could not be read: {exc}") from exc + + output_path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(suffix=".tif", dir=output_path.parent, delete=False) as temporary: + temporary_path = Path(temporary.name) + try: + with rasterio.open(temporary_path, "w", **profile) as output: + output.write(clipped) + raster_copy( + temporary_path, + output_path, + driver="COG", + compress="DEFLATE", + predictor=3, + blocksize=512, + overview_resampling="average", + ) + finally: + temporary_path.unlink(missing_ok=True) + + with rasterio.open(output_path) as output: + values = output.read(1, masked=True) + valid = geometry_mask( + [mapping(selection_metric)], + out_shape=values.shape, + transform=output.transform, + invert=True, + ) & ~values.mask + valid_values = values.data[valid] + if valid_values.size == 0: + output_path.unlink(missing_ok=True) + raise SpwBathymetryImportError("Selected scope contains no surveyed SPW waterbed cells") + if not bool(((valid_values > -500.0) & (valid_values < 1000.0)).all()): + output_path.unlink(missing_ok=True) + raise SpwBathymetryImportError("SPW bathymetry contains values outside the governed mDNG range") + return { + "width": output.width, + "height": output.height, + "resolution_m": max(abs(output.res[0]), abs(output.res[1])), + "valid_cell_count": int(valid_values.size), + "valid_value_min": float(valid_values.min()), + "valid_value_max": float(valid_values.max()), + "valid_value_mean": float(valid_values.mean()), + "output_sha256": sha256_file(output_path), + } + + +def multipart_upload( + base_url: str, + project_id: str, + output_path: Path, + fields: dict[str, str], + *, + timeout: int, +) -> dict[str, Any]: + boundary = f"geointel-{uuid.uuid4().hex}" + body = bytearray() + for name, value in fields.items(): + body.extend(f"--{boundary}\r\n".encode()) + body.extend(f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode()) + body.extend(value.encode("utf-8")) + body.extend(b"\r\n") + body.extend(f"--{boundary}\r\n".encode()) + body.extend( + ( + f'Content-Disposition: form-data; name="file"; filename="{output_path.name}"\r\n' + f"Content-Type: {mimetypes.guess_type(output_path.name)[0] or 'image/tiff'}\r\n\r\n" + ).encode() + ) + body.extend(output_path.read_bytes()) + body.extend(f"\r\n--{boundary}--\r\n".encode()) + request = Request( + f"{base_url.rstrip('/')}/api/v1/projects/{project_id}/datasets/upload", + data=bytes(body), + method="POST", + headers={ + "Accept": "application/json", + "Content-Type": f"multipart/form-data; boundary={boundary}", + "User-Agent": "GeoIntel/1.0 governed-spw-bathymetry-operator", + }, + ) + try: + with urlopen(request, timeout=timeout) as response: + status_code = int(response.status) + response_content = response.read() + except HTTPError as exc: + status_code = exc.code + response_content = exc.read() + except (URLError, TimeoutError, OSError) as exc: + raise SpwBathymetryImportError(f"GeoIntel upload failed: {exc}") from exc + try: + response_payload = json.loads(response_content.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise SpwBathymetryImportError(f"GeoIntel upload returned non-JSON HTTP {status_code}") from exc + return unwrap(response_payload, status_code) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Import a bounded official SPW bathymetry raster.") + parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL)) + parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME) + parser.add_argument("--project-id") + parser.add_argument("--area") + parser.add_argument("--area-id") + parser.add_argument("--bbox", required=True, type=parse_bbox, help="min_x,min_y,max_x,max_y in EPSG:4326") + parser.add_argument("--raw-zip", required=True, type=Path) + parser.add_argument("--download", action="store_true", help="Download the pinned official ZIP when --raw-zip is absent.") + parser.add_argument("--output-dir", type=Path, default=Path("storage/operator-evidence/spw-bathymetry/derived")) + parser.add_argument("--max-pixels", type=int, default=DEFAULT_MAX_PIXELS) + parser.add_argument("--timeout", type=int, default=900) + parser.add_argument("--force-refresh", action="store_true") + parser.add_argument("--dry-run", action="store_true") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + archive_path = args.raw_zip.resolve() + if not archive_path.exists() and args.download: + download_source(archive_path, timeout=args.timeout) + member = validate_archive(archive_path) + project = resolve_project( + args.base_url, + args.project_name, + project_id=args.project_id, + timeout=args.timeout, + ) + project_id = str(project["id"]) + area = resolve_area( + args.base_url, + project_id, + area_id=args.area_id, + area_name=args.area, + timeout=args.timeout, + ) + selection = args.bbox + if area is not None: + selection = selection.intersection(shape(area["geometry"])) + if selection.is_empty or selection.area <= 0: + raise SpwBathymetryImportError("Requested bbox does not overlap the selected persisted Area") + + scope_hash = selection_hash(selection) + output_name = f"spw_bathymetry_{scope_hash[:16]}_3812.tif" + output_path = args.output_dir.resolve() / output_name + raw_sha256 = sha256_file(archive_path) + datasets = paged_items( + args.base_url, + f"/api/v1/projects/{project_id}/datasets", + timeout=args.timeout, + ) + existing = next( + ( + item + for item in datasets + if item.get("status") == "ready" + and item.get("source_name") == "spw_bathymetry" + and (item.get("source_metadata") or {}).get("source_artifact_sha256") == raw_sha256 + and (item.get("source_metadata") or {}).get("selection_hash") == scope_hash + ), + None, + ) + if existing and not args.force_refresh: + print(json.dumps({"status": "reused", "dataset_id": existing["id"]}, ensure_ascii=False)) + return 0 + + if args.dry_run: + print( + json.dumps( + { + "status": "validated", + "project_id": project_id, + "area_id": area.get("id") if area else None, + "bbox_epsg4326": list(selection.bounds), + "source_sha256": raw_sha256, + "source_member": member.filename, + "output_path": str(output_path), + }, + ensure_ascii=False, + ) + ) + return 0 + + diagnostics = crop_source( + archive_path, + member, + selection, + output_path, + max_pixels=args.max_pixels, + ) + source_metadata = { + "product_key": "spw_bathymetry_50cm_mdng", + "product_display_name": "SPW waterbodemhoogte 0,5 m", + "theme": "bathymetry", + "value_semantics": "bed_elevation", + "vertical_reference": VERTICAL_REFERENCE, + "source_crs": SOURCE_CRS, + "native_resolution_m": 0.5, + "analysis_resolution_m": diagnostics["resolution_m"], + "nodata_value": NODATA, + "survey_period": SURVEY_PERIOD, + "published_on": "2023-05-23", + "bbox_epsg4326": list(selection.bounds), + "coverage_zones": ["wallonia"], + "coverage_scope": "bounded_selection", + "selection_hash": scope_hash, + "source_artifact_sha256": raw_sha256, + "valid_cell_count": diagnostics["valid_cell_count"], + "valid_value_min": diagnostics["valid_value_min"], + "valid_value_max": diagnostics["valid_value_max"], + "valid_value_mean": diagnostics["valid_value_mean"], + "attribution": "Service public de Wallonie (SPW)", + "license_note": "CC BY 4.0", + "catalog_url": CATALOG_URL, + "limitation_message": ( + "Waterbodemhoogte in mDNG uit opmetingen 2019-2022; geen actuele waterdiepte of watervolume." + ), + } + provenance_metadata = { + "operator": "import_spw_bathymetry.py", + "official_source_url": SOURCE_URL, + "official_catalog_url": CATALOG_URL, + "source_artifact_sha256": raw_sha256, + "source_archive_member": member.filename, + "derived_artifact_sha256": diagnostics["output_sha256"], + "selection_geometry_epsg4326": mapping(selection), + "selection_hash": scope_hash, + "processed_at": datetime.now(UTC).isoformat(), + "water_depth_available": False, + "water_volume_available": False, + "vertical_datum_conversion_applied": False, + } + fields = { + "dataset_type": "raster", + "source": "SPW official operator archive", + "dataset_role": "source", + "source_name": "spw_bathymetry", + "source_metadata_json": json.dumps(source_metadata, separators=(",", ":")), + "provenance_metadata_json": json.dumps(provenance_metadata, separators=(",", ":")), + "valid_from": "2019-01-01T00:00:00Z", + "valid_to": "2022-12-31T23:59:59Z", + "temporal_granularity": "period", + "source_version": SOURCE_VERSION, + } + if area is not None: + fields["area_id"] = str(area["id"]) + created = multipart_upload( + args.base_url, + project_id, + output_path, + fields, + timeout=args.timeout, + ) + analysis = api_json( + args.base_url, + f"/api/v1/projects/{project_id}/datasets/{created['id']}/raster/bathymetry/select", + timeout=args.timeout, + method="POST", + payload={ + "bbox": { + "min_x": selection.bounds[0], + "min_y": selection.bounds[1], + "max_x": selection.bounds[2], + "max_y": selection.bounds[3], + "crs": "EPSG:4326", + }, + "area_id": area.get("id") if area else None, + }, + ) + print( + json.dumps( + { + "status": "imported", + "dataset_id": created["id"], + "output_path": str(output_path), + "source_sha256": raw_sha256, + "derived_sha256": diagnostics["output_sha256"], + "summary": analysis["summary"], + "coverage_ratio": analysis["coverage_ratio"], + }, + ensure_ascii=False, + ) + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except SpwBathymetryImportError as exc: + raise SystemExit(f"SPW_BATHYMETRY_IMPORT_FAILED: {exc}") from exc diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index b5064582..82dab99c 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -80,6 +80,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py ${PYTHON_BIN} -m py_compile scripts/provision_flanders_geographic_scope.py ${PYTHON_BIN} -m py_compile scripts/provision_flanders_bathymetry_profiles.py ${PYTHON_BIN} -m py_compile scripts/probe_mdk_bathymetry.py +${PYTHON_BIN} -m py_compile scripts/import_spw_bathymetry.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py