644 lines
34 KiB
Python
644 lines
34 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import time
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from email.parser import BytesParser
|
|
from email.policy import default
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlencode
|
|
from urllib.request import Request
|
|
from uuid import UUID
|
|
from xml.etree import ElementTree
|
|
|
|
from geoalchemy2.shape import to_shape
|
|
from pyproj import Transformer
|
|
from shapely.geometry import box, mapping
|
|
from shapely.ops import transform as shapely_transform
|
|
|
|
from app.core.config import Settings, get_settings
|
|
from app.core.errors import AppError
|
|
from app.services.outbound_request_guard import guarded_opener
|
|
from app.models import Area, Dataset, Project
|
|
from app.schemas.flood_hazard import FloodHazardAcquireRequest, FloodHazardAcquisitionResult, FloodHazardProductRead
|
|
from app.services.dataset_service import DatasetService
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class FloodHazardProduct:
|
|
key: str
|
|
display_name: str
|
|
mechanism: str
|
|
climate_context: str
|
|
probability_class: str
|
|
return_period_years: int
|
|
coverage_id: str
|
|
published_on: str
|
|
catalog_url: str
|
|
|
|
|
|
class FloodHazardAcquisitionService:
|
|
PROVIDER = "vmm_flood_hazard"
|
|
SOURCE_CRS = "EPSG:31370"
|
|
NATIVE_RESOLUTION_M = 2.0
|
|
SOURCE_VALUE_UNIT = "cm"
|
|
NORMALIZED_VALUE_UNIT = "m"
|
|
NODATA = -9999.0
|
|
SOURCE_VERSION = "VMM OGRK flood hazard maps"
|
|
ATTRIBUTION = "Bron: VMM"
|
|
LICENSE_NOTE = "Publieke toegang; gebruik en bronvermelding volgens de metadata van VMM/GDI-Vlaanderen."
|
|
# The VMM WCS rejects generated coverages above 4.88 MB. At the default
|
|
# 5 metre resolution a 5 km square stays below that provider-side limit.
|
|
WCS_TILE_SIDE_M = 5_000.0
|
|
WCS_REQUEST_INTERVAL_SECONDS = 1.0
|
|
WCS_RETRY_DELAY_SECONDS = 3.0
|
|
WCS_TRANSIENT_STATUS_CODES = frozenset({400, 429, 502, 503, 504})
|
|
# The VMM WCS rounds the grid size of partial edge tiles to an integer
|
|
# number of cells. Keep that provider artefact bounded and auditable.
|
|
WCS_EDGE_RESOLUTION_REL_TOLERANCE = 0.05
|
|
WCS_EDGE_RESOLUTION_ABS_TOLERANCE_M = 0.25
|
|
SERVICE_CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/publieke-inspire-coverage-service-van-ogrk"
|
|
LIMITATION = (
|
|
"Gemodelleerde maximale overstromingsdiepte voor een vast kans- en klimaatscenario. "
|
|
"Dit is geen actuele waterstand, geen bathymetrie en geen permanente diepte of inhoud van een waterlichaam."
|
|
)
|
|
|
|
@staticmethod
|
|
def _products() -> dict[str, FloodHazardProduct]:
|
|
products: list[FloodHazardProduct] = []
|
|
probability = {
|
|
10: ("grote kans", "grote-kans"),
|
|
100: ("middelgrote kans", "middelgrote-kans"),
|
|
1000: ("kleine kans", "kleine-kans"),
|
|
}
|
|
for mechanism, code in (("pluviaal", "PLU"), ("fluviaal", "FLU")):
|
|
for climate_key, climate_code, climate_label, published_on in (
|
|
("current", "noCC", "huidig klimaat", "2021-08-31"),
|
|
("future_2050", "hCC", "klimaatprojectie 2050", "2021-08-31" if mechanism == "fluviaal" else "2019-12-22"),
|
|
):
|
|
for period, (probability_label, probability_slug) in probability.items():
|
|
climate_slug = (
|
|
"huidig-klimaat"
|
|
if climate_key == "current"
|
|
else "toekomstig-klimaat-met-klimaatprojectie-2050"
|
|
)
|
|
catalog_url = (
|
|
"https://www.vlaanderen.be/datavindplaats/catalogus/"
|
|
f"overstromingsgevaarkaart-waterdiepte-{mechanism}-{climate_slug}-{probability_slug}"
|
|
)
|
|
products.append(
|
|
FloodHazardProduct(
|
|
key=f"{mechanism}_{climate_key}_t{period}",
|
|
display_name=(
|
|
f"{mechanism.capitalize()} - {climate_label} - {probability_label} (T{period})"
|
|
),
|
|
mechanism=mechanism,
|
|
climate_context=climate_label,
|
|
probability_class=probability_label,
|
|
return_period_years=period,
|
|
coverage_id=(
|
|
f"Overstromingsgevaarkaarten-{code.replace('PLU', 'PLUVIAAL').replace('FLU', 'FLUVIAAL')}:"
|
|
f"waterdiepte_{code}_{climate_code}_T{period}"
|
|
),
|
|
published_on=published_on,
|
|
catalog_url=catalog_url,
|
|
)
|
|
)
|
|
return {product.key: product for product in products}
|
|
|
|
@staticmethod
|
|
def list_products() -> list[dict[str, Any]]:
|
|
return [
|
|
FloodHazardProductRead(
|
|
key=product.key,
|
|
display_name=product.display_name,
|
|
mechanism=product.mechanism,
|
|
climate_context=product.climate_context,
|
|
probability_class=product.probability_class,
|
|
return_period_years=product.return_period_years,
|
|
coverage_id=product.coverage_id,
|
|
native_resolution_m=FloodHazardAcquisitionService.NATIVE_RESOLUTION_M,
|
|
source_crs=FloodHazardAcquisitionService.SOURCE_CRS,
|
|
source_value_unit=FloodHazardAcquisitionService.SOURCE_VALUE_UNIT,
|
|
normalized_value_unit=FloodHazardAcquisitionService.NORMALIZED_VALUE_UNIT,
|
|
published_on=product.published_on,
|
|
catalog_url=product.catalog_url,
|
|
attribution=FloodHazardAcquisitionService.ATTRIBUTION,
|
|
limitation_message=FloodHazardAcquisitionService.LIMITATION,
|
|
).model_dump()
|
|
for product in FloodHazardAcquisitionService._products().values()
|
|
]
|
|
|
|
@staticmethod
|
|
def _product(product_key: str) -> FloodHazardProduct:
|
|
product = FloodHazardAcquisitionService._products().get(product_key.strip().lower())
|
|
if product is None:
|
|
raise AppError(
|
|
code="FLOOD_HAZARD_PRODUCT_NOT_SUPPORTED",
|
|
message="Select a governed VMM fluvial or pluvial flood-depth scenario",
|
|
details={"product_key": product_key},
|
|
status_code=422,
|
|
)
|
|
return product
|
|
|
|
@staticmethod
|
|
def _prepared_request(payload: FloodHazardAcquireRequest, settings: Settings) -> dict[str, Any]:
|
|
if not settings.flood_hazard_enabled:
|
|
raise AppError(code="FLOOD_HAZARD_NOT_CONFIGURED", message="VMM flood-hazard acquisition is disabled", status_code=503)
|
|
product = FloodHazardAcquisitionService._product(payload.product_key)
|
|
resolution_m = float(payload.resolution_m or settings.flood_hazard_resolution_m)
|
|
values = (payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
|
|
if not all(math.isfinite(value) for value in values) or payload.bbox.min_x >= payload.bbox.max_x or payload.bbox.min_y >= payload.bbox.max_y:
|
|
raise AppError(code="INVALID_BBOX", message="Flood-hazard selection must be a finite non-empty rectangle", status_code=400)
|
|
transformer = Transformer.from_crs("EPSG:4326", FloodHazardAcquisitionService.SOURCE_CRS, always_xy=True)
|
|
metric_bounds = transformer.transform_bounds(*values, densify_pts=21)
|
|
width_m = float(metric_bounds[2] - metric_bounds[0])
|
|
height_m = float(metric_bounds[3] - metric_bounds[1])
|
|
if width_m < settings.flood_hazard_min_side_m or height_m < settings.flood_hazard_min_side_m:
|
|
raise AppError(
|
|
code="FLOOD_HAZARD_SELECTION_TOO_SMALL",
|
|
message=f"Select an area of at least {settings.flood_hazard_min_side_m:g} by {settings.flood_hazard_min_side_m:g} metres",
|
|
status_code=422,
|
|
)
|
|
if width_m > settings.flood_hazard_max_side_m or height_m > settings.flood_hazard_max_side_m:
|
|
raise AppError(
|
|
code="FLOOD_HAZARD_SELECTION_TOO_LARGE",
|
|
message=f"Select an area no larger than {settings.flood_hazard_max_side_m:g} by {settings.flood_hazard_max_side_m:g} metres",
|
|
details={"width_m": width_m, "height_m": height_m},
|
|
status_code=422,
|
|
)
|
|
width = max(1, math.ceil(width_m / resolution_m))
|
|
height = max(1, math.ceil(height_m / resolution_m))
|
|
if width * height > settings.flood_hazard_max_pixels:
|
|
raise AppError(
|
|
code="FLOOD_HAZARD_SELECTION_TOO_LARGE",
|
|
message="Flood-hazard selection exceeds the configured raster cell limit",
|
|
details={"pixel_count": width * height, "max_pixels": settings.flood_hazard_max_pixels},
|
|
status_code=422,
|
|
)
|
|
bbox_4326 = [float(value) for value in values]
|
|
bbox_31370 = [float(value) for value in metric_bounds]
|
|
identity = {
|
|
"provider": FloodHazardAcquisitionService.PROVIDER,
|
|
"coverage_id": product.coverage_id,
|
|
"bbox_epsg4326": [round(value, 8) for value in bbox_4326],
|
|
"bbox_epsg31370": [round(value, 3) for value in bbox_31370],
|
|
"resolution_m": resolution_m,
|
|
"area_id": str(payload.area_id) if payload.area_id else None,
|
|
}
|
|
request_hash = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
|
|
return {
|
|
**identity,
|
|
"product": product,
|
|
"request_hash": request_hash,
|
|
"bbox_epsg4326": bbox_4326,
|
|
"bbox_epsg31370": bbox_31370,
|
|
"width": width,
|
|
"height": height,
|
|
}
|
|
|
|
@staticmethod
|
|
def _wcs_request_url(settings: Settings, product: FloodHazardProduct, bounds: tuple[float, float, float, float], resolution_m: float) -> str:
|
|
crs = "urn:ogc:def:crs:EPSG::31370"
|
|
query = [
|
|
("SERVICE", "WCS"),
|
|
("VERSION", "1.1.0"),
|
|
("REQUEST", "GetCoverage"),
|
|
("IDENTIFIER", product.coverage_id),
|
|
("BOUNDINGBOX", f"{bounds[0]:.3f},{bounds[1]:.3f},{bounds[2]:.3f},{bounds[3]:.3f},{crs}"),
|
|
("FORMAT", "image/tiff"),
|
|
("GRIDBASECRS", crs),
|
|
("GRIDCS", "urn:ogc:def:cs:OGC:0.0:Grid2dSquareCS"),
|
|
("GRIDTYPE", "urn:ogc:def:method:WCS:1.1:2dSimpleGrid"),
|
|
("GRIDORIGIN", f"{bounds[0]:.3f},{bounds[3]:.3f}"),
|
|
("GRIDOFFSETS", f"{resolution_m:g},-{resolution_m:g}"),
|
|
]
|
|
return f"{settings.flood_hazard_wcs_url}?{urlencode(query)}"
|
|
|
|
@staticmethod
|
|
def _tile_bounds(prepared: dict[str, Any]) -> list[tuple[float, float, float, float]]:
|
|
min_x, min_y, max_x, max_y = prepared["bbox_epsg31370"]
|
|
tiles: list[tuple[float, float, float, float]] = []
|
|
y = min_y
|
|
while y < max_y:
|
|
tile_max_y = min(y + FloodHazardAcquisitionService.WCS_TILE_SIDE_M, max_y)
|
|
x = min_x
|
|
while x < max_x:
|
|
tile_max_x = min(x + FloodHazardAcquisitionService.WCS_TILE_SIDE_M, max_x)
|
|
tiles.append((x, y, tile_max_x, tile_max_y))
|
|
x = tile_max_x
|
|
y = tile_max_y
|
|
return tiles
|
|
|
|
@staticmethod
|
|
def _scope_geometry(db, project_id: UUID, area_id: UUID | None, bbox_epsg4326: list[float]):
|
|
if not db.get(Project, project_id):
|
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
|
selection = box(*bbox_epsg4326)
|
|
if area_id is None:
|
|
return selection
|
|
area = db.get(Area, area_id)
|
|
if not area:
|
|
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
|
if area.project_id != project_id:
|
|
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
|
|
intersection = to_shape(area.geometry).intersection(selection)
|
|
if intersection.is_empty or intersection.area <= 0:
|
|
raise AppError(code="FLOOD_HAZARD_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422)
|
|
return intersection
|
|
|
|
@staticmethod
|
|
def _fetch(request_url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, str]:
|
|
request = Request(request_url, headers={"Accept": "*/*", "User-Agent": "GeoIntel/0.1 bounded-vmm-flood-hazard-acquisition"})
|
|
max_bytes = settings.flood_hazard_max_response_mb * 1024 * 1024
|
|
try:
|
|
with (opener or guarded_opener(request_url))(request, timeout=settings.flood_hazard_timeout_seconds) as response:
|
|
content_type = str(response.headers.get("Content-Type", ""))
|
|
content_length = response.headers.get("Content-Length")
|
|
if content_length and int(content_length) > max_bytes:
|
|
raise AppError(code="FLOOD_HAZARD_RESPONSE_TOO_LARGE", message="Official VMM response exceeds the configured size limit", status_code=502)
|
|
content = response.read(max_bytes + 1)
|
|
except AppError:
|
|
raise
|
|
except HTTPError as exc:
|
|
preview = exc.read(300).decode("utf-8", errors="replace")
|
|
raise AppError(
|
|
code="FLOOD_HAZARD_PROVIDER_UNAVAILABLE",
|
|
message="The official VMM WCS could not complete the bounded request",
|
|
details={"reason": str(exc), "provider_status_code": int(exc.code), "response_preview": preview},
|
|
status_code=502,
|
|
) from exc
|
|
except (URLError, TimeoutError, OSError) as exc:
|
|
raise AppError(
|
|
code="FLOOD_HAZARD_PROVIDER_UNAVAILABLE",
|
|
message="The official VMM WCS could not complete the bounded request",
|
|
details={"reason": str(exc)},
|
|
status_code=502,
|
|
) from exc
|
|
if len(content) > max_bytes:
|
|
raise AppError(code="FLOOD_HAZARD_RESPONSE_TOO_LARGE", message="Official VMM response exceeds the configured size limit", status_code=502)
|
|
return content, content_type
|
|
|
|
@staticmethod
|
|
def _extract_geotiff(content: bytes, content_type: str) -> bytes:
|
|
if content.startswith((b"II*\x00", b"MM\x00*")):
|
|
return content
|
|
if "multipart" not in content_type.lower():
|
|
provider_exception = None
|
|
if "xml" in content_type.lower() or content.lstrip().startswith(b"<"):
|
|
try:
|
|
root = ElementTree.fromstring(content)
|
|
exception_texts = [
|
|
(element.text or "").strip()
|
|
for element in root.iter()
|
|
if element.tag.rsplit("}", 1)[-1] in {"ExceptionText", "ServiceException"}
|
|
and (element.text or "").strip()
|
|
]
|
|
provider_exception = " ".join(exception_texts) or None
|
|
except ElementTree.ParseError:
|
|
pass
|
|
raise AppError(
|
|
code="FLOOD_HAZARD_PROVIDER_INVALID_RESPONSE",
|
|
message="The official VMM service did not return a GeoTIFF coverage",
|
|
details={
|
|
"content_type": content_type,
|
|
"provider_exception": provider_exception,
|
|
"response_preview": content[:300].decode("utf-8", errors="replace"),
|
|
},
|
|
status_code=502,
|
|
)
|
|
message = BytesParser(policy=default).parsebytes(f"Content-Type: {content_type}\r\nMIME-Version: 1.0\r\n\r\n".encode() + content)
|
|
for part in message.walk():
|
|
payload = part.get_payload(decode=True) or b""
|
|
if part.get_content_type() == "image/tiff" and payload.startswith((b"II*\x00", b"MM\x00*")):
|
|
return payload
|
|
raise AppError(
|
|
code="FLOOD_HAZARD_PROVIDER_INVALID_RESPONSE",
|
|
message="The official VMM multipart response contains no valid GeoTIFF coverage",
|
|
status_code=502,
|
|
)
|
|
|
|
@staticmethod
|
|
def _mosaic_geotiffs(
|
|
coverages: list[bytes],
|
|
expected_resolution_m: float,
|
|
diagnostics: dict[str, Any] | None = None,
|
|
) -> bytes:
|
|
if len(coverages) == 1:
|
|
return coverages[0]
|
|
try:
|
|
from rasterio.io import MemoryFile
|
|
from rasterio.merge import merge
|
|
except ImportError as exc:
|
|
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio is required to assemble VMM flood-hazard tiles", status_code=503) from exc
|
|
memories = [MemoryFile(content) for content in coverages]
|
|
sources = []
|
|
try:
|
|
sources = [memory.open() for memory in memories]
|
|
invalid_crs = [
|
|
index
|
|
for index, source in enumerate(sources)
|
|
if source.crs is None or source.crs.to_epsg() != 31370
|
|
]
|
|
invalid_bands = [index for index, source in enumerate(sources) if source.count != 1]
|
|
tile_resolutions = [
|
|
[abs(float(source.res[0])), abs(float(source.res[1]))]
|
|
for source in sources
|
|
]
|
|
invalid_resolution = [
|
|
{
|
|
"tile_index": index,
|
|
"resolution": tile_resolutions[index],
|
|
}
|
|
for index, source in enumerate(sources)
|
|
if not all(
|
|
math.isclose(
|
|
abs(float(value)),
|
|
expected_resolution_m,
|
|
rel_tol=FloodHazardAcquisitionService.WCS_EDGE_RESOLUTION_REL_TOLERANCE,
|
|
abs_tol=FloodHazardAcquisitionService.WCS_EDGE_RESOLUTION_ABS_TOLERANCE_M,
|
|
)
|
|
for value in source.res
|
|
)
|
|
]
|
|
if invalid_crs or invalid_bands or invalid_resolution:
|
|
raise AppError(
|
|
code="FLOOD_HAZARD_TILE_MISMATCH",
|
|
message="VMM coverage tiles do not match the governed CRS, band layout and resolution",
|
|
details={
|
|
"invalid_crs_tile_indexes": invalid_crs,
|
|
"invalid_band_tile_indexes": invalid_bands,
|
|
"invalid_resolution_tiles": invalid_resolution,
|
|
"expected_resolution_m": expected_resolution_m,
|
|
},
|
|
status_code=502,
|
|
)
|
|
harmonized_tile_indexes = [
|
|
index
|
|
for index, resolution in enumerate(tile_resolutions)
|
|
if not all(
|
|
math.isclose(value, expected_resolution_m, rel_tol=0.02, abs_tol=0.05)
|
|
for value in resolution
|
|
)
|
|
]
|
|
if diagnostics is not None:
|
|
diagnostics.update(
|
|
{
|
|
"source_tile_resolutions_m": tile_resolutions,
|
|
"target_resolution_m": expected_resolution_m,
|
|
"harmonized_tile_indexes": harmonized_tile_indexes,
|
|
"harmonization_method": (
|
|
"rasterio_merge_target_resolution" if harmonized_tile_indexes else None
|
|
),
|
|
}
|
|
)
|
|
mosaic, transform = merge(sources, res=(expected_resolution_m, expected_resolution_m), nodata=0.0, dtype="float32")
|
|
profile = sources[0].profile.copy()
|
|
profile.pop("blockxsize", None)
|
|
profile.pop("blockysize", None)
|
|
profile.update(driver="GTiff", width=mosaic.shape[2], height=mosaic.shape[1], count=1, dtype="float32", crs=FloodHazardAcquisitionService.SOURCE_CRS, transform=transform, nodata=0.0, compress="deflate", predictor=3)
|
|
with MemoryFile() as output_memory:
|
|
with output_memory.open(**profile) as output:
|
|
output.write(mosaic)
|
|
return output_memory.read()
|
|
except AppError:
|
|
raise
|
|
except Exception as exc:
|
|
raise AppError(code="FLOOD_HAZARD_TILE_MOSAIC_FAILED", message="VMM flood-hazard tiles could not be assembled", details={"reason": str(exc)}, status_code=502) from exc
|
|
finally:
|
|
for source in sources:
|
|
source.close()
|
|
for memory in memories:
|
|
memory.close()
|
|
|
|
@staticmethod
|
|
def _fetch_coverage(prepared: dict[str, Any], settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, dict[str, Any]]:
|
|
product: FloodHazardProduct = prepared["product"]
|
|
request_urls = [
|
|
FloodHazardAcquisitionService._wcs_request_url(settings, product, bounds, prepared["resolution_m"])
|
|
for bounds in FloodHazardAcquisitionService._tile_bounds(prepared)
|
|
]
|
|
raw_hash = hashlib.sha256()
|
|
coverage_hash = hashlib.sha256()
|
|
content_types: list[str] = []
|
|
coverages: list[bytes] = []
|
|
for index, request_url in enumerate(request_urls):
|
|
if index > 0 and opener is None:
|
|
time.sleep(FloodHazardAcquisitionService.WCS_REQUEST_INTERVAL_SECONDS)
|
|
try:
|
|
raw_content, content_type = FloodHazardAcquisitionService._fetch(request_url, settings, opener)
|
|
except AppError as exc:
|
|
provider_status = (exc.details or {}).get("provider_status_code")
|
|
if opener is not None or provider_status not in FloodHazardAcquisitionService.WCS_TRANSIENT_STATUS_CODES:
|
|
raise
|
|
time.sleep(FloodHazardAcquisitionService.WCS_RETRY_DELAY_SECONDS)
|
|
raw_content, content_type = FloodHazardAcquisitionService._fetch(request_url, settings, opener)
|
|
coverage = FloodHazardAcquisitionService._extract_geotiff(raw_content, content_type)
|
|
raw_hash.update(len(raw_content).to_bytes(8, "big"))
|
|
raw_hash.update(raw_content)
|
|
coverage_hash.update(len(coverage).to_bytes(8, "big"))
|
|
coverage_hash.update(coverage)
|
|
content_types.append(content_type)
|
|
coverages.append(coverage)
|
|
mosaic_diagnostics: dict[str, Any] = {}
|
|
mosaic = FloodHazardAcquisitionService._mosaic_geotiffs(
|
|
coverages,
|
|
prepared["resolution_m"],
|
|
diagnostics=mosaic_diagnostics,
|
|
)
|
|
return mosaic, {
|
|
"tile_count": len(request_urls),
|
|
"request_urls": request_urls,
|
|
"response_content_types": content_types,
|
|
"response_sha256": raw_hash.hexdigest(),
|
|
"coverage_sha256": coverage_hash.hexdigest(),
|
|
"grid_harmonization": mosaic_diagnostics,
|
|
}
|
|
|
|
@staticmethod
|
|
def _normalize_raster(content: bytes, scope_geometry_4326, prepared: dict[str, Any]) -> tuple[bytes, dict[str, Any]]:
|
|
try:
|
|
import numpy as np
|
|
from rasterio.io import MemoryFile
|
|
from rasterio.mask import mask
|
|
except ImportError as exc:
|
|
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for flood-hazard validation", status_code=503) from exc
|
|
try:
|
|
with MemoryFile(content) as source_memory, source_memory.open() as source:
|
|
if source.crs is None or source.crs.to_epsg() != 31370:
|
|
raise AppError(code="FLOOD_HAZARD_INVALID_CRS", message="VMM flood-hazard coverage must use EPSG:31370", status_code=502)
|
|
if source.count != 1:
|
|
raise AppError(code="FLOOD_HAZARD_INVALID_BANDS", message="VMM flood-hazard coverage must contain one depth band", status_code=502)
|
|
resolution = max(abs(float(source.res[0])), abs(float(source.res[1])))
|
|
if not math.isclose(resolution, prepared["resolution_m"], rel_tol=0.02, abs_tol=0.05):
|
|
raise AppError(code="FLOOD_HAZARD_INVALID_RESOLUTION", message="VMM coverage resolution differs from the governed request", status_code=502)
|
|
transformer = Transformer.from_crs("EPSG:4326", FloodHazardAcquisitionService.SOURCE_CRS, always_xy=True)
|
|
scope_metric = shapely_transform(transformer.transform, scope_geometry_4326)
|
|
clipped, transform = mask(source, [mapping(scope_metric)], crop=True, filled=False, indexes=[1])
|
|
source_values = np.ma.asarray(clipped[0], dtype="float32")
|
|
raw_cm = np.asarray(source_values.filled(0.0), dtype="float32")
|
|
positive = (~np.ma.getmaskarray(source_values)) & np.isfinite(raw_cm) & (raw_cm > 0.0)
|
|
normalized_m = np.full(raw_cm.shape, FloodHazardAcquisitionService.NODATA, dtype="float32")
|
|
normalized_m[positive] = raw_cm[positive] / 100.0
|
|
valid_values = normalized_m[positive].astype("float64")
|
|
profile = source.profile.copy()
|
|
profile.pop("blockxsize", None)
|
|
profile.pop("blockysize", None)
|
|
profile.update(driver="GTiff", width=normalized_m.shape[1], height=normalized_m.shape[0], count=1, dtype="float32", crs=FloodHazardAcquisitionService.SOURCE_CRS, transform=transform, nodata=FloodHazardAcquisitionService.NODATA, compress="deflate", predictor=3)
|
|
with MemoryFile() as output_memory:
|
|
with output_memory.open(**profile) as output:
|
|
output.write(normalized_m, 1)
|
|
normalized_content = output_memory.read()
|
|
return normalized_content, {
|
|
"width": int(normalized_m.shape[1]),
|
|
"height": int(normalized_m.shape[0]),
|
|
"inundated_pixel_count": int(positive.sum()),
|
|
"nodata_value": FloodHazardAcquisitionService.NODATA,
|
|
"resolution_m": resolution,
|
|
"minimum_depth_m": float(valid_values.min()) if valid_values.size else None,
|
|
"maximum_depth_m": float(valid_values.max()) if valid_values.size else None,
|
|
"source_value_unit": FloodHazardAcquisitionService.SOURCE_VALUE_UNIT,
|
|
"normalized_value_unit": FloodHazardAcquisitionService.NORMALIZED_VALUE_UNIT,
|
|
}
|
|
except AppError:
|
|
raise
|
|
except Exception as exc:
|
|
raise AppError(code="FLOOD_HAZARD_RASTER_INVALID", message="The official VMM response is not a valid georeferenced flood-depth raster", details={"reason": str(exc)}, status_code=502) from exc
|
|
|
|
@staticmethod
|
|
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
|
|
candidate = (
|
|
db.query(Dataset)
|
|
.filter(Dataset.project_id == project_id, Dataset.name == filename, Dataset.source_name == FloodHazardAcquisitionService.PROVIDER, Dataset.status == "ready")
|
|
.order_by(Dataset.imported_at.desc())
|
|
.first()
|
|
)
|
|
if candidate and candidate.storage_path and Path(candidate.storage_path).is_file():
|
|
return candidate
|
|
return None
|
|
|
|
@staticmethod
|
|
def acquire(db, project_id: UUID, payload: FloodHazardAcquireRequest, *, settings: Settings | None = None, opener: Callable[..., Any] | None = None) -> dict[str, Any]:
|
|
resolved_settings = settings or get_settings()
|
|
prepared = FloodHazardAcquisitionService._prepared_request(payload, resolved_settings)
|
|
product: FloodHazardProduct = prepared["product"]
|
|
scope_geometry = FloodHazardAcquisitionService._scope_geometry(db, project_id, payload.area_id, prepared["bbox_epsg4326"])
|
|
resolution_token = f"{prepared['resolution_m']:g}".replace(".", "p")
|
|
filename = f"vmm_flood_depth_{product.key}_{resolution_token}m_{prepared['request_hash'][:12]}.tif"
|
|
if not payload.force_refresh:
|
|
cached = FloodHazardAcquisitionService._cached_dataset(db, project_id, filename)
|
|
if cached is not None:
|
|
metadata = cached.source_metadata or {}
|
|
raster = cached.metadata_json or {}
|
|
return FloodHazardAcquisitionResult(
|
|
output_dataset_id=cached.id,
|
|
reused=True,
|
|
provider=FloodHazardAcquisitionService.PROVIDER,
|
|
product_key=product.key,
|
|
display_name=product.display_name,
|
|
mechanism=product.mechanism,
|
|
climate_context=product.climate_context,
|
|
probability_class=product.probability_class,
|
|
return_period_years=product.return_period_years,
|
|
coverage_id=product.coverage_id,
|
|
resolution_m=float(metadata.get("analysis_resolution_m", prepared["resolution_m"])),
|
|
width=int(raster.get("width", prepared["width"])),
|
|
height=int(raster.get("height", prepared["height"])),
|
|
inundated_pixel_count=int(metadata.get("inundated_pixel_count", 0)),
|
|
bbox_epsg4326=prepared["bbox_epsg4326"],
|
|
bbox_epsg31370=prepared["bbox_epsg31370"],
|
|
attribution=FloodHazardAcquisitionService.ATTRIBUTION,
|
|
limitation_message=FloodHazardAcquisitionService.LIMITATION,
|
|
).model_dump(mode="json")
|
|
content, transfer = FloodHazardAcquisitionService._fetch_coverage(prepared, resolved_settings, opener)
|
|
normalized, validation = FloodHazardAcquisitionService._normalize_raster(content, scope_geometry, prepared)
|
|
acquired_at = datetime.now(UTC)
|
|
dataset = DatasetService.import_raster_bytes(
|
|
db,
|
|
project_id=project_id,
|
|
area_id=payload.area_id,
|
|
filename=filename,
|
|
content=normalized,
|
|
source=f"VMM OGRK WCS {product.coverage_id}",
|
|
source_name=FloodHazardAcquisitionService.PROVIDER,
|
|
source_version=FloodHazardAcquisitionService.SOURCE_VERSION,
|
|
content_type="image/tiff",
|
|
source_metadata={
|
|
"provider": FloodHazardAcquisitionService.PROVIDER,
|
|
"service": "WCS",
|
|
"service_version": "1.1.0",
|
|
"product_key": product.key,
|
|
"product_display_name": product.display_name,
|
|
"mechanism": product.mechanism,
|
|
"climate_context": product.climate_context,
|
|
"probability_class": product.probability_class,
|
|
"return_period_years": product.return_period_years,
|
|
"coverage_id": product.coverage_id,
|
|
"native_resolution_m": FloodHazardAcquisitionService.NATIVE_RESOLUTION_M,
|
|
"analysis_resolution_m": validation["resolution_m"],
|
|
"source_crs": FloodHazardAcquisitionService.SOURCE_CRS,
|
|
"source_value_unit": FloodHazardAcquisitionService.SOURCE_VALUE_UNIT,
|
|
"normalized_value_unit": FloodHazardAcquisitionService.NORMALIZED_VALUE_UNIT,
|
|
"inundated_pixel_count": validation["inundated_pixel_count"],
|
|
"minimum_depth_m": validation["minimum_depth_m"],
|
|
"maximum_depth_m": validation["maximum_depth_m"],
|
|
"bbox_epsg4326": prepared["bbox_epsg4326"],
|
|
"bbox_epsg31370": prepared["bbox_epsg31370"],
|
|
"published_on": product.published_on,
|
|
"catalog_url": product.catalog_url,
|
|
"service_catalog_url": FloodHazardAcquisitionService.SERVICE_CATALOG_URL,
|
|
"attribution": FloodHazardAcquisitionService.ATTRIBUTION,
|
|
"license_note": FloodHazardAcquisitionService.LICENSE_NOTE,
|
|
"theme": "flood_hazard",
|
|
"layer_name": "modelled_flood_depth",
|
|
"coverage_scope": "municipality" if payload.area_id else "bounded_selection",
|
|
},
|
|
provenance_metadata={
|
|
"acquisition": "explicit_bounded_tiled_wcs_coverage",
|
|
"acquired_at": acquired_at.isoformat(),
|
|
"request_hash": prepared["request_hash"],
|
|
"tile_count": transfer["tile_count"],
|
|
"tile_request_urls": transfer["request_urls"],
|
|
"response_content_types": transfer["response_content_types"],
|
|
"response_sha256": transfer["response_sha256"],
|
|
"coverage_sha256": transfer["coverage_sha256"],
|
|
"grid_harmonization": transfer["grid_harmonization"],
|
|
"normalized_sha256": hashlib.sha256(normalized).hexdigest(),
|
|
"bbox_epsg4326": prepared["bbox_epsg4326"],
|
|
"bbox_epsg31370": prepared["bbox_epsg31370"],
|
|
"requested_resolution_m": prepared["resolution_m"],
|
|
"clipped_to_area_id": str(payload.area_id) if payload.area_id else None,
|
|
"validation": validation,
|
|
"limitation_message": FloodHazardAcquisitionService.LIMITATION,
|
|
"bathymetry_available": False,
|
|
"permanent_water_depth_available": False,
|
|
"permanent_water_volume_available": False,
|
|
"concurrent_flood_volume_available": False,
|
|
},
|
|
)
|
|
return FloodHazardAcquisitionResult(
|
|
output_dataset_id=dataset.id,
|
|
reused=False,
|
|
provider=FloodHazardAcquisitionService.PROVIDER,
|
|
product_key=product.key,
|
|
display_name=product.display_name,
|
|
mechanism=product.mechanism,
|
|
climate_context=product.climate_context,
|
|
probability_class=product.probability_class,
|
|
return_period_years=product.return_period_years,
|
|
coverage_id=product.coverage_id,
|
|
resolution_m=validation["resolution_m"],
|
|
width=validation["width"],
|
|
height=validation["height"],
|
|
inundated_pixel_count=validation["inundated_pixel_count"],
|
|
bbox_epsg4326=prepared["bbox_epsg4326"],
|
|
bbox_epsg31370=prepared["bbox_epsg31370"],
|
|
attribution=FloodHazardAcquisitionService.ATTRIBUTION,
|
|
limitation_message=FloodHazardAcquisitionService.LIMITATION,
|
|
).model_dump(mode="json")
|