Complete regional raster exploration
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-16 14:21:32 +02:00
parent 153cff06c0
commit 45dc730e76
24 changed files with 1162 additions and 123 deletions
+12
View File
@@ -1246,6 +1246,12 @@ and a complete failure summary. Persistence remains inside the canonical
DHMV acquisition service and Dataset/DatasetVersion/Job flow; the operator
does not fetch WCS bytes or write raster metadata directly.
The complete live matrix contains 56 ready Datasets and 56 DatasetVersions
across 28 Areas. On the complete Kempen Area the Map workspace presents those
partitions as one logical DTM/DSM layer. `POST .../datasets/raster/terrain/select`
opens only partitions intersecting the drawn rectangle and computes exact
global cell statistics. It does not create a hidden regional mosaic.
Settings: `DHMV_ENABLED`, `DHMV_WCS_URL`, `DHMV_RESOLUTION_M`,
`DHMV_MIN_SIDE_M`, `DHMV_MAX_SIDE_M`, `DHMV_MAX_PIXELS`,
`DHMV_TIMEOUT_SECONDS` and `DHMV_MAX_RESPONSE_MB`.
@@ -1295,6 +1301,12 @@ The full Kempen scope with all products means 28 municipalities times 12
scenario rasters. This is intentionally explicit operator work, not startup
work and not a browser-side provider fetch.
The complete live matrix contains 336 ready Datasets and 336 DatasetVersions.
The regional Map workspace deduplicates them into twelve scenario choices,
renders all municipality image partitions for the selected scenario and uses
`POST .../datasets/raster/flood-hazard/select` for exact bounded cross-boundary
analysis. The same 12-million-cell guard prevents unsafe full-region reads.
Use `--products pluviaal_current_t100`, `--resolution-m 5` or `--force` for an
explicit subset/refresh. `POST .../raster/flood-hazard/select` returns mapped
inundated hectares, selection share and local modeled maximum-depth statistics.
+21 -4
View File
@@ -4,7 +4,6 @@ import json
from datetime import datetime
from typing import Any
from uuid import UUID
from uuid import UUID as _UUID
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response
from fastapi import UploadFile
@@ -23,8 +22,10 @@ from app.schemas import (
RasterNdbiRequest,
OrthophotoAcquireRequest,
DhmvAcquireRequest,
TerrainPartitionSelectionRequest,
TerrainSelectionRequest,
FloodHazardAcquireRequest,
FloodHazardPartitionSelectionRequest,
FloodHazardSelectionRequest,
ThematicRasterAcquireRequest,
ThematicRasterSelectionRequest,
@@ -32,14 +33,12 @@ from app.schemas import (
VectorBufferRequest,
VectorClipRequest,
VectorIntersectRequest,
VectorSelectionBBox,
VectorSelectionBBox, # noqa: F401 - retained as a route-module compatibility export
VectorSelectionDeriveRequest,
VectorSelectionRequest,
VectorSelectionResponse,
)
from app.schemas.job import JobCreate
from app.schemas.dataset import DatasetCreateResponse, DatasetTemporalUpdate
from app.schemas.operations import VectorOperationResult
from app.services.job_service import JobService
from app.services.raster_operations_service import RasterOperationsService
from app.services.vector_operations_service import VectorOperationsService
@@ -550,6 +549,15 @@ def raster_terrain_selection(
return envelope(TerrainAnalysisService.analyze(db, project_id, dataset_id, payload))
@router.post("/datasets/raster/terrain/select", response_model=dict)
def partitioned_raster_terrain_selection(
project_id: UUID,
payload: TerrainPartitionSelectionRequest,
db: Session = Depends(get_db),
):
return envelope(TerrainAnalysisService.analyze_partitions(db, project_id, payload))
@router.get("/datasets/{dataset_id}/raster/terrain/image")
def raster_terrain_image(
project_id: UUID,
@@ -574,6 +582,15 @@ def raster_flood_hazard_selection(
return envelope(FloodHazardAnalysisService.analyze(db, project_id, dataset_id, payload))
@router.post("/datasets/raster/flood-hazard/select", response_model=dict)
def partitioned_raster_flood_hazard_selection(
project_id: UUID,
payload: FloodHazardPartitionSelectionRequest,
db: Session = Depends(get_db),
):
return envelope(FloodHazardAnalysisService.analyze_partitions(db, project_id, payload))
@router.get("/datasets/{dataset_id}/raster/flood-hazard/image")
def raster_flood_hazard_image(
project_id: UUID,
+4
View File
@@ -38,6 +38,7 @@ from .dhmv import (
DhmvAcquisitionResult,
DhmvProductRead,
TerrainMetric,
TerrainPartitionSelectionRequest,
TerrainSelectionRequest,
TerrainSelectionResponse,
TerrainSelectionSummary,
@@ -46,6 +47,7 @@ from .flood_hazard import (
FloodHazardAcquireRequest,
FloodHazardAcquisitionResult,
FloodHazardMetric,
FloodHazardPartitionSelectionRequest,
FloodHazardProductRead,
FloodHazardSelectionRequest,
FloodHazardSelectionResponse,
@@ -166,12 +168,14 @@ __all__ = [
"DhmvAcquisitionResult",
"DhmvProductRead",
"TerrainMetric",
"TerrainPartitionSelectionRequest",
"TerrainSelectionRequest",
"TerrainSelectionResponse",
"TerrainSelectionSummary",
"FloodHazardAcquireRequest",
"FloodHazardAcquisitionResult",
"FloodHazardMetric",
"FloodHazardPartitionSelectionRequest",
"FloodHazardProductRead",
"FloodHazardSelectionRequest",
"FloodHazardSelectionResponse",
+6
View File
@@ -56,6 +56,10 @@ class TerrainSelectionRequest(BaseModel):
area_id: UUID | None = None
class TerrainPartitionSelectionRequest(TerrainSelectionRequest):
product_key: str = "dtm_1m"
class TerrainMetric(BaseModel):
metric_key: str
metric_label: str
@@ -76,6 +80,8 @@ class TerrainSelectionSummary(BaseModel):
class TerrainSelectionResponse(BaseModel):
dataset_id: UUID
dataset_ids: list[UUID] = Field(default_factory=list)
partition_count: int = Field(default=1, ge=1)
product_key: str
surface_model: str
selection_bbox: VectorSelectionBBox
+6
View File
@@ -59,6 +59,10 @@ class FloodHazardSelectionRequest(BaseModel):
area_id: UUID | None = None
class FloodHazardPartitionSelectionRequest(FloodHazardSelectionRequest):
product_key: str = "pluviaal_current_t100"
class FloodHazardMetric(BaseModel):
metric_key: str
metric_label: str
@@ -79,6 +83,8 @@ class FloodHazardSelectionSummary(BaseModel):
class FloodHazardSelectionResponse(BaseModel):
dataset_id: UUID
dataset_ids: list[UUID] = Field(default_factory=list)
partition_count: int = Field(default=1, ge=1)
product_key: str
mechanism: str
climate_context: str
@@ -16,11 +16,13 @@ from app.core.errors import AppError
from app.models import Area, Dataset
from app.schemas.flood_hazard import (
FloodHazardMetric,
FloodHazardPartitionSelectionRequest,
FloodHazardSelectionRequest,
FloodHazardSelectionResponse,
FloodHazardSelectionSummary,
)
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
from app.services.raster_partition_analysis_service import RasterPartitionAnalysisService
class FloodHazardAnalysisService:
@@ -170,6 +172,8 @@ class FloodHazardAnalysisService:
primary = metrics[0]
response = FloodHazardSelectionResponse(
dataset_id=dataset.id,
dataset_ids=[dataset.id],
partition_count=1,
product_key=product.key,
mechanism=product.mechanism,
climate_context=product.climate_context,
@@ -195,6 +199,129 @@ class FloodHazardAnalysisService:
)
return response.model_dump(mode="json")
@staticmethod
def analyze_partitions(
db,
project_id: UUID,
payload: FloodHazardPartitionSelectionRequest,
*,
settings: Settings | None = None,
) -> dict:
resolved_settings = settings or get_settings()
product = FloodHazardAcquisitionService._products().get(payload.product_key.strip().lower())
if product is None:
raise AppError(
code="FLOOD_HAZARD_PRODUCT_NOT_SUPPORTED",
message="Select a governed VMM fluvial or pluvial flood-depth scenario",
details={"product_key": payload.product_key},
status_code=422,
)
selection_4326 = FloodHazardAnalysisService._selection_geometry(db, project_id, payload)
partition = RasterPartitionAnalysisService.select(
db,
project_id,
source_name=FloodHazardAcquisitionService.PROVIDER,
product_key=product.key,
selection_geometry_4326=selection_4326,
nodata=FloodHazardAcquisitionService.NODATA,
max_pixels=resolved_settings.flood_hazard_max_pixels,
)
try:
import numpy as np
except ImportError as exc:
raise AppError(
code="RASTER_PROCESSING_UNAVAILABLE",
message="Numpy is required for partitioned flood-hazard analysis",
status_code=503,
) from exc
raw = partition.values
valid = (
partition.selected_cells
& np.isfinite(raw)
& (raw != FloodHazardAcquisitionService.NODATA)
& (raw > 0.0)
)
values = raw[valid]
selected_cell_count = int(partition.selected_cells.sum())
inundated_cell_count = int(values.size)
cell_area_m2 = partition.resolution_x * partition.resolution_y
def metric(key: str, label: str, value: float, unit: str, method: str) -> FloodHazardMetric:
return FloodHazardMetric(
metric_key=key,
metric_label=label,
metric_value=round(float(value), 4),
metric_unit=unit,
aggregation_method=method,
)
inundated_area_ha = inundated_cell_count * cell_area_m2 / 10_000.0
metrics = [
metric(
"modelled_inundated_area_ha",
"Gemodelleerd overstroomd oppervlak",
inundated_area_ha,
"ha",
"positive_depth_cells_times_cell_area",
),
metric(
"modelled_inundated_share_pct",
"Aandeel selectie met gemodelleerde diepte",
inundated_cell_count / max(1, selected_cell_count) * 100.0,
"%",
"positive_depth_cells_divided_by_selected_cells",
),
]
if inundated_cell_count:
metrics.extend(
[
metric("modelled_depth_mean_m", "Gemiddelde gemodelleerde maximumdiepte", values.mean(), "m", "mean_positive_depth_cells"),
metric("modelled_depth_p90_m", "90e percentiel gemodelleerde maximumdiepte", np.percentile(values, 90), "m", "percentile_90_positive_depth_cells"),
metric("modelled_depth_max_m", "Hoogste gemodelleerde maximumdiepte", values.max(), "m", "maximum_positive_depth_cells"),
metric(
"modelled_max_depth_area_integral_m3",
"Diepte-oppervlakte-integraal (geen gelijktijdig volume)",
values.sum() * cell_area_m2,
"m3",
"sum_local_max_depth_times_cell_area",
),
]
)
primary = metrics[0]
first_dataset = partition.datasets[0]
response = FloodHazardSelectionResponse(
dataset_id=first_dataset.id,
dataset_ids=[dataset.id for dataset in partition.datasets],
partition_count=len(partition.datasets),
product_key=product.key,
mechanism=product.mechanism,
climate_context=product.climate_context,
probability_class=product.probability_class,
return_period_years=product.return_period_years,
selection_bbox=payload.bbox,
selection_area_id=payload.area_id,
selected_cell_count=selected_cell_count,
inundated_cell_count=inundated_cell_count,
inundated_fraction=round(inundated_cell_count / max(1, selected_cell_count), 6),
resolution_m=round(max(partition.resolution_x, partition.resolution_y), 4),
summary=FloodHazardSelectionSummary(
metric_label=primary.metric_label,
metric_value=primary.metric_value,
metric_unit=primary.metric_unit,
aggregation_method=primary.aggregation_method,
primary_metric_key=primary.metric_key,
metrics=metrics,
),
unsupported_metrics=FloodHazardAnalysisService.UNSUPPORTED_METRICS,
limitation_message=(
f"{FloodHazardAnalysisService.LIMITATION} De selectie werd exact berekend over "
f"{len(partition.datasets)} persistente gemeentelijke rasterpartities."
),
generated_at=datetime.now(UTC).isoformat(),
)
return response.model_dump(mode="json")
@staticmethod
def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes:
dataset = FloodHazardAnalysisService._load_dataset(db, project_id, dataset_id)
@@ -0,0 +1,200 @@
from __future__ import annotations
import math
from contextlib import ExitStack
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from uuid import UUID
from pyproj import Transformer
from shapely.geometry import mapping
from shapely.ops import transform as shapely_transform
from app.core.errors import AppError
from app.models import Dataset
@dataclass(frozen=True)
class RasterPartitionSelection:
datasets: list[Dataset]
values: Any
selected_cells: Any
resolution_x: float
resolution_y: float
class RasterPartitionAnalysisService:
MAX_PARTITIONS = 64
@staticmethod
def _bbox_intersects(dataset: Dataset, bbox: tuple[float, float, float, float]) -> bool:
source_bbox = (dataset.source_metadata or {}).get("bbox_epsg4326")
if not isinstance(source_bbox, list) or len(source_bbox) != 4:
return True
try:
min_x, min_y, max_x, max_y = (float(value) for value in source_bbox)
except (TypeError, ValueError):
return True
return not (
max_x <= bbox[0]
or min_x >= bbox[2]
or max_y <= bbox[1]
or min_y >= bbox[3]
)
@staticmethod
def _candidate_datasets(
db,
project_id: UUID,
*,
source_name: str,
product_key: str,
bbox: tuple[float, float, float, float],
) -> list[Dataset]:
rows = (
db.query(Dataset)
.filter(
Dataset.project_id == project_id,
Dataset.source_name == source_name,
Dataset.dataset_type == "raster",
Dataset.status == "ready",
)
.all()
)
candidates = [
dataset
for dataset in rows
if str((dataset.source_metadata or {}).get("product_key") or "") == product_key
and dataset.storage_path
and Path(dataset.storage_path).is_file()
and RasterPartitionAnalysisService._bbox_intersects(dataset, bbox)
]
candidates.sort(key=lambda dataset: (str(dataset.area_id or ""), str(dataset.id)))
if not candidates:
raise AppError(
code="RASTER_PARTITIONS_NOT_FOUND",
message="No persisted raster partitions cover this selection",
details={"source_name": source_name, "product_key": product_key},
status_code=404,
)
if len(candidates) > RasterPartitionAnalysisService.MAX_PARTITIONS:
raise AppError(
code="RASTER_PARTITION_LIMIT_EXCEEDED",
message="The selection intersects too many raster partitions",
details={
"partition_count": len(candidates),
"max_partitions": RasterPartitionAnalysisService.MAX_PARTITIONS,
},
status_code=422,
)
return candidates
@staticmethod
def select(
db,
project_id: UUID,
*,
source_name: str,
product_key: str,
selection_geometry_4326,
nodata: float,
max_pixels: int,
) -> RasterPartitionSelection:
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.merge import merge
except ImportError as exc:
raise AppError(
code="RASTER_PROCESSING_UNAVAILABLE",
message="Rasterio and numpy are required for partitioned raster analysis",
status_code=503,
) from exc
bbox = tuple(float(value) for value in selection_geometry_4326.bounds)
datasets = RasterPartitionAnalysisService._candidate_datasets(
db,
project_id,
source_name=source_name,
product_key=product_key,
bbox=bbox,
)
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
selection_metric = shapely_transform(transformer.transform, selection_geometry_4326)
min_x, min_y, max_x, max_y = selection_metric.bounds
try:
with ExitStack() as stack:
sources = [stack.enter_context(rasterio.open(dataset.storage_path)) for dataset in datasets]
invalid_sources = [
index
for index, source in enumerate(sources)
if source.crs is None or source.crs.to_epsg() != 31370 or source.count != 1
]
if invalid_sources:
raise AppError(
code="RASTER_PARTITION_MISMATCH",
message="Raster partitions do not share the governed CRS and band layout",
details={"invalid_partition_indexes": invalid_sources},
status_code=409,
)
target_resolution = max(abs(float(sources[0].res[0])), abs(float(sources[0].res[1])))
invalid_resolutions = [
{
"partition_index": index,
"resolution": [abs(float(source.res[0])), abs(float(source.res[1]))],
}
for index, source in enumerate(sources)
if not all(
math.isclose(abs(float(value)), target_resolution, rel_tol=0.001, abs_tol=0.01)
for value in source.res
)
]
if invalid_resolutions:
raise AppError(
code="RASTER_PARTITION_MISMATCH",
message="Raster partitions do not share one analysis resolution",
details={"invalid_resolutions": invalid_resolutions},
status_code=409,
)
width = max(1, math.ceil((max_x - min_x) / target_resolution))
height = max(1, math.ceil((max_y - min_y) / target_resolution))
if width * height > max_pixels:
raise AppError(
code="RASTER_PARTITION_SELECTION_TOO_LARGE",
message="Select a smaller rectangle for regional raster analysis",
details={"pixel_count": width * height, "max_pixels": max_pixels},
status_code=422,
)
mosaic, transform = merge(
sources,
bounds=(min_x, min_y, max_x, max_y),
res=(target_resolution, target_resolution),
nodata=nodata,
dtype="float32",
)
values = np.asarray(mosaic[0], dtype="float64")
selected_cells = geometry_mask(
[mapping(selection_metric)],
out_shape=values.shape,
transform=transform,
invert=True,
)
return RasterPartitionSelection(
datasets=datasets,
values=values,
selected_cells=selected_cells,
resolution_x=target_resolution,
resolution_y=target_resolution,
)
except AppError:
raise
except Exception as exc:
raise AppError(
code="RASTER_PARTITION_ANALYSIS_FAILED",
message="Persisted raster partitions could not be assembled for this selection",
details={"reason": str(exc)},
status_code=500,
) from exc
@@ -14,8 +14,15 @@ from shapely.ops import transform as shapely_transform
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import Area, Dataset
from app.schemas.dhmv import TerrainMetric, TerrainSelectionRequest, TerrainSelectionResponse, TerrainSelectionSummary
from app.schemas.dhmv import (
TerrainMetric,
TerrainPartitionSelectionRequest,
TerrainSelectionRequest,
TerrainSelectionResponse,
TerrainSelectionSummary,
)
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.raster_partition_analysis_service import RasterPartitionAnalysisService
class TerrainAnalysisService:
@@ -177,6 +184,8 @@ class TerrainAnalysisService:
selected_cell_count = int(selected_cells.sum())
response = TerrainSelectionResponse(
dataset_id=dataset.id,
dataset_ids=[dataset.id],
partition_count=1,
product_key=product_key,
surface_model=surface_model,
selection_bbox=payload.bbox,
@@ -200,6 +209,139 @@ class TerrainAnalysisService:
)
return response.model_dump(mode="json")
@staticmethod
def analyze_partitions(
db,
project_id: UUID,
payload: TerrainPartitionSelectionRequest,
*,
settings: Settings | None = None,
) -> dict:
resolved_settings = settings or get_settings()
product = DhmvAcquisitionService._products().get(payload.product_key.strip().lower())
if product is None:
raise AppError(
code="DHMV_PRODUCT_NOT_SUPPORTED",
message="Select a governed DHMV terrain or surface product",
details={"product_key": payload.product_key},
status_code=422,
)
selection_4326 = TerrainAnalysisService._selection_geometry(db, project_id, payload)
partition = RasterPartitionAnalysisService.select(
db,
project_id,
source_name=DhmvAcquisitionService.PROVIDER,
product_key=product.key,
selection_geometry_4326=selection_4326,
nodata=DhmvAcquisitionService.NODATA,
max_pixels=resolved_settings.dhmv_max_pixels,
)
surface_models = {
str((dataset.source_metadata or {}).get("surface_model") or "")
for dataset in partition.datasets
}
if surface_models != {product.surface_model}:
raise AppError(
code="INVALID_TERRAIN_METADATA",
message="DHMV partition provenance is incomplete",
details={"surface_models": sorted(surface_models)},
status_code=409,
)
try:
import numpy as np
except ImportError as exc:
raise AppError(
code="RASTER_PROCESSING_UNAVAILABLE",
message="Numpy is required for partitioned terrain analysis",
status_code=503,
) from exc
raw = partition.values
invalid = ~np.isfinite(raw) | (raw == DhmvAcquisitionService.NODATA)
valid_mask = partition.selected_cells & ~invalid
values = raw[valid_mask]
if values.size == 0:
raise AppError(
code="TERRAIN_NO_VALID_DATA",
message="No valid DHMV height cells occur in this selection",
status_code=422,
)
slope_values = np.asarray([], dtype="float64")
if raw.shape[0] >= 2 and raw.shape[1] >= 2:
surface = np.where(valid_mask, raw, np.nan)
gradient_y, gradient_x = np.gradient(
surface,
partition.resolution_y,
partition.resolution_x,
)
slope = np.degrees(np.arctan(np.hypot(gradient_x, gradient_y)))
slope_values = slope[np.isfinite(slope) & valid_mask]
def metric(key: str, label: str, value: float, unit: str, method: str) -> TerrainMetric:
return TerrainMetric(
metric_key=key,
metric_label=label,
metric_value=round(float(value), 4),
metric_unit=unit,
aggregation_method=method,
)
prefix = "terrain" if product.surface_model == "terrain" else "surface"
elevation_label = (
"Gemiddelde maaiveldhoogte"
if product.surface_model == "terrain"
else "Gemiddelde oppervlaktehoogte"
)
metrics = [
metric(f"{prefix}_elevation_mean_m", elevation_label, values.mean(), "m TAW", "mean_valid_cells"),
metric(f"{prefix}_elevation_min_m", "Laagste hoogte", values.min(), "m TAW", "minimum_valid_cells"),
metric(f"{prefix}_elevation_max_m", "Hoogste hoogte", values.max(), "m TAW", "maximum_valid_cells"),
metric(f"{prefix}_elevation_p10_m", "10e percentiel hoogte", np.percentile(values, 10), "m TAW", "percentile_10_valid_cells"),
metric(f"{prefix}_elevation_p90_m", "90e percentiel hoogte", np.percentile(values, 90), "m TAW", "percentile_90_valid_cells"),
metric("relief_m", "Reliëfverschil", values.max() - values.min(), "m", "maximum_minus_minimum"),
]
if slope_values.size:
metrics.extend(
[
metric("slope_mean_deg", "Gemiddelde helling", slope_values.mean(), "°", "mean_finite_gradient"),
metric("slope_p90_deg", "90e percentiel helling", np.percentile(slope_values, 90), "°", "percentile_90_finite_gradient"),
metric("slope_max_deg", "Steilste helling", slope_values.max(), "°", "maximum_finite_gradient"),
]
)
primary = metrics[0]
selected_cell_count = int(partition.selected_cells.sum())
first_dataset = partition.datasets[0]
response = TerrainSelectionResponse(
dataset_id=first_dataset.id,
dataset_ids=[dataset.id for dataset in partition.datasets],
partition_count=len(partition.datasets),
product_key=product.key,
surface_model=product.surface_model,
selection_bbox=payload.bbox,
selection_area_id=payload.area_id,
sample_count=int(values.size),
slope_sample_count=int(slope_values.size),
coverage_ratio=round(float(values.size / max(1, selected_cell_count)), 6),
resolution_m=round(max(partition.resolution_x, partition.resolution_y), 4),
vertical_reference=DhmvAcquisitionService.VERTICAL_REFERENCE,
summary=TerrainSelectionSummary(
metric_label=primary.metric_label,
metric_value=primary.metric_value,
metric_unit=primary.metric_unit,
aggregation_method=primary.aggregation_method,
primary_metric_key=primary.metric_key,
metrics=metrics,
),
unsupported_metrics=TerrainAnalysisService.UNSUPPORTED_METRICS,
limitation_message=(
f"{TerrainAnalysisService.LIMITATION} De selectie werd exact berekend over "
f"{len(partition.datasets)} persistente gemeentelijke rasterpartities."
),
generated_at=datetime.now(UTC).isoformat(),
)
return response.model_dump(mode="json")
@staticmethod
def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes:
dataset = TerrainAnalysisService._load_dataset(db, project_id, dataset_id)
+93 -1
View File
@@ -18,7 +18,7 @@ from app.core.errors import AppError
from app.db.session import get_db
from app.main import app
from app.models import Area, Dataset, DatasetVersion, Job, Project
from app.schemas.dhmv import DhmvAcquireRequest, TerrainSelectionRequest
from app.schemas.dhmv import DhmvAcquireRequest, TerrainPartitionSelectionRequest, TerrainSelectionRequest
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.terrain_analysis_service import TerrainAnalysisService
@@ -39,6 +39,9 @@ class FakeQuery:
def first(self):
return self.result
def all(self):
return self.result if isinstance(self.result, list) else []
class FakeSession:
def __init__(self, rows=None, query_result=None):
@@ -115,6 +118,23 @@ def elevation_tiff(*, left: float, top: float, width: int, height: int, resoluti
return memory.read()
def constant_elevation_tiff(*, left: float, top: float, value: float) -> bytes:
values = np.full((20, 20), value, dtype="float32")
with MemoryFile() as memory:
with memory.open(
driver="GTiff",
width=20,
height=20,
count=1,
dtype="float32",
crs="EPSG:31370",
transform=from_origin(left, top, 5.0, 5.0),
nodata=-9999.0,
) as output:
output.write(values, 1)
return memory.read()
def edge_elevation_tiff(*, left: float, top: float, x_resolution: float, y_resolution: float = 5.0) -> bytes:
rows, columns = np.indices((20, 20))
values = (20.0 + columns * 0.5 + rows).astype("float32")
@@ -359,6 +379,61 @@ def test_terrain_analysis_returns_governed_elevation_relief_and_slope(tmp_path)
assert "Waterdiepte" in result["limitation_message"]
def test_partitioned_terrain_analysis_is_exact_across_municipality_boundaries(tmp_path) -> None:
project_id = uuid4()
transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
min_x, min_y = transformer.transform(200_000, 210_000)
middle_x, _ = transformer.transform(200_100, 210_000)
max_x, max_y = transformer.transform(200_200, 210_100)
paths = [tmp_path / "left-terrain.tif", tmp_path / "right-terrain.tif"]
paths[0].write_bytes(constant_elevation_tiff(left=200_000, top=210_100, value=10.0))
paths[1].write_bytes(constant_elevation_tiff(left=200_100, top=210_100, value=20.0))
datasets = [
Dataset(
id=uuid4(),
project_id=project_id,
area_id=uuid4(),
name=path.name,
dataset_type="raster",
source="official WCS",
source_name="digitaal_vlaanderen_dhmv",
source_metadata={
"product_key": "dtm_1m",
"surface_model": "terrain",
"bbox_epsg4326": [left, min_y, right, max_y],
},
status="ready",
storage_path=str(path),
)
for path, left, right in (
(paths[0], min_x, middle_x),
(paths[1], middle_x, max_x),
)
]
db = FakeSession(query_result=datasets)
payload = TerrainPartitionSelectionRequest(
bbox={"min_x": min_x, "min_y": min_y, "max_x": max_x, "max_y": max_y, "crs": "EPSG:4326"},
product_key="dtm_1m",
)
result = TerrainAnalysisService.analyze_partitions(
db,
project_id,
payload,
settings=Settings(_env_file=None),
)
metrics = {item["metric_key"]: item["metric_value"] for item in result["summary"]["metrics"]}
assert result["partition_count"] == 2
assert set(result["dataset_ids"]) == {str(dataset.id) for dataset in datasets}
assert result["sample_count"] >= 790
assert metrics["terrain_elevation_mean_m"] == pytest.approx(15.0, abs=0.1)
assert metrics["terrain_elevation_min_m"] == 10.0
assert metrics["terrain_elevation_max_m"] == 20.0
assert metrics["terrain_elevation_p90_m"] == 20.0
assert "2 persistente gemeentelijke rasterpartities" in result["limitation_message"]
def test_terrain_analysis_rejects_non_dhmv_raster(tmp_path) -> None:
project_id = uuid4()
dataset_id = uuid4()
@@ -425,6 +500,16 @@ def test_dhmv_endpoints_use_canonical_envelopes(monkeypatch) -> None:
"unsupported_metrics": ["water_depth_m", "water_volume_m3"],
},
)
monkeypatch.setattr(
TerrainAnalysisService,
"analyze_partitions",
lambda *_args, **_kwargs: {
"dataset_id": str(output_dataset_id),
"dataset_ids": [str(output_dataset_id)],
"partition_count": 1,
"sample_count": 100,
},
)
app.dependency_overrides[get_db] = lambda: db
try:
products = TestClient(app).get(f"/api/v1/projects/{project_id}/datasets/dhmv/products")
@@ -436,6 +521,10 @@ def test_dhmv_endpoints_use_canonical_envelopes(monkeypatch) -> None:
f"/api/v1/projects/{project_id}/datasets/{output_dataset_id}/raster/terrain/select",
json={"bbox": lambert_bbox_payload().bbox.model_dump()},
)
regional_terrain = TestClient(app).post(
f"/api/v1/projects/{project_id}/datasets/raster/terrain/select",
json={"bbox": lambert_bbox_payload().bbox.model_dump(), "product_key": "dtm_1m"},
)
finally:
app.dependency_overrides.clear()
@@ -450,6 +539,9 @@ def test_dhmv_endpoints_use_canonical_envelopes(monkeypatch) -> None:
assert set(terrain.json()) == {"data"}
assert terrain.json()["data"]["sample_count"] == 100
assert terrain.json()["data"]["unsupported_metrics"] == ["water_depth_m", "water_volume_m3"]
assert regional_terrain.status_code == 200
assert set(regional_terrain.json()) == {"data"}
assert regional_terrain.json()["data"]["partition_count"] == 1
assert any(isinstance(item, Job) for item in db.added)
@@ -16,7 +16,11 @@ from app.core.errors import AppError
from app.db.session import get_db
from app.main import app
from app.models import Dataset, Job, Project
from app.schemas.flood_hazard import FloodHazardAcquireRequest, FloodHazardSelectionRequest
from app.schemas.flood_hazard import (
FloodHazardAcquireRequest,
FloodHazardPartitionSelectionRequest,
FloodHazardSelectionRequest,
)
from app.schemas.assistant import AssistantQueryRequest
from app.services.geo_assistant_service import GeoAssistantService
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
@@ -120,6 +124,23 @@ def edge_depth_tiff(*, left: float, top: float, x_resolution: float, y_resolutio
return memory.read()
def normalized_depth_tiff(*, left: float, top: float, value: float) -> bytes:
values = np.full((20, 20), value, dtype="float32")
with MemoryFile() as memory:
with memory.open(
driver="GTiff",
width=20,
height=20,
count=1,
dtype="float32",
crs="EPSG:31370",
transform=from_origin(left, top, 5.0, 5.0),
nodata=-9999.0,
) as output:
output.write(values, 1)
return memory.read()
def test_flood_hazard_registry_is_complete_and_semantically_honest() -> None:
products = FloodHazardAcquisitionService.list_products()
@@ -274,6 +295,61 @@ def test_flood_hazard_analysis_reports_scenario_metrics_without_claiming_waterbo
assert "geen gelijktijdig" in result["limitation_message"]
def test_partitioned_flood_analysis_is_exact_across_municipality_boundaries(tmp_path) -> None:
project_id = uuid4()
transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
min_x, min_y = transformer.transform(200_000, 210_000)
middle_x, _ = transformer.transform(200_100, 210_000)
max_x, max_y = transformer.transform(200_200, 210_100)
paths = [tmp_path / "left-flood.tif", tmp_path / "right-flood.tif"]
paths[0].write_bytes(normalized_depth_tiff(left=200_000, top=210_100, value=1.0))
paths[1].write_bytes(normalized_depth_tiff(left=200_100, top=210_100, value=2.0))
datasets = [
Dataset(
id=uuid4(),
project_id=project_id,
area_id=uuid4(),
name=path.name,
dataset_type="raster",
source="VMM",
source_name=FloodHazardAcquisitionService.PROVIDER,
source_metadata={
"product_key": "pluviaal_current_t100",
"normalized_value_unit": "m",
"bbox_epsg4326": [left, min_y, right, max_y],
},
status="ready",
storage_path=str(path),
)
for path, left, right in (
(paths[0], min_x, middle_x),
(paths[1], middle_x, max_x),
)
]
db = FakeSession(query_result=datasets)
payload = FloodHazardPartitionSelectionRequest(
bbox={"min_x": min_x, "min_y": min_y, "max_x": max_x, "max_y": max_y, "crs": "EPSG:4326"},
product_key="pluviaal_current_t100",
)
result = FloodHazardAnalysisService.analyze_partitions(
db,
project_id,
payload,
settings=Settings(_env_file=None),
)
metrics = {item["metric_key"]: item["metric_value"] for item in result["summary"]["metrics"]}
assert result["partition_count"] == 2
assert set(result["dataset_ids"]) == {str(dataset.id) for dataset in datasets}
assert result["inundated_cell_count"] >= 790
assert result["inundated_fraction"] == pytest.approx(1.0)
assert metrics["modelled_depth_mean_m"] == pytest.approx(1.5, abs=0.01)
assert metrics["modelled_depth_p90_m"] == 2.0
assert metrics["modelled_inundated_area_ha"] == pytest.approx(2.0, abs=0.03)
assert "2 persistente gemeentelijke rasterpartities" in result["limitation_message"]
def test_flood_hazard_renderer_returns_transparent_png(tmp_path) -> None:
project_id = uuid4()
dataset_id = uuid4()
@@ -314,6 +390,16 @@ def test_flood_hazard_api_uses_canonical_envelopes(monkeypatch) -> None:
"unsupported_metrics": ["permanent_water_volume_m3"],
},
)
monkeypatch.setattr(
FloodHazardAnalysisService,
"analyze_partitions",
lambda *_args, **_kwargs: {
"dataset_id": str(output_dataset_id),
"dataset_ids": [str(output_dataset_id)],
"partition_count": 1,
"inundated_cell_count": 4,
},
)
app.dependency_overrides[get_db] = lambda: db
try:
client = TestClient(app)
@@ -326,6 +412,10 @@ def test_flood_hazard_api_uses_canonical_envelopes(monkeypatch) -> None:
f"/api/v1/projects/{project_id}/datasets/{output_dataset_id}/raster/flood-hazard/select",
json={"bbox": flood_payload().bbox.model_dump()},
)
regional_selection = client.post(
f"/api/v1/projects/{project_id}/datasets/raster/flood-hazard/select",
json={"bbox": flood_payload().bbox.model_dump(), "product_key": "pluviaal_current_t100"},
)
finally:
app.dependency_overrides.clear()
@@ -334,6 +424,8 @@ def test_flood_hazard_api_uses_canonical_envelopes(monkeypatch) -> None:
assert acquisition.status_code == 200 and set(acquisition.json()) == {"data"}
assert acquisition.json()["data"]["job_type"] == "raster.flood_hazard.acquire"
assert selection.status_code == 200 and set(selection.json()) == {"data"}
assert regional_selection.status_code == 200 and set(regional_selection.json()) == {"data"}
assert regional_selection.json()["data"]["partition_count"] == 1
assert any(isinstance(item, Job) for item in db.added)
@@ -0,0 +1,52 @@
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
def test_partitioned_raster_routes_are_canonical_and_documented() -> None:
routes = (ROOT / "backend/app/api/routes/datasets.py").read_text(encoding="utf-8")
contracts = (ROOT / "docs/API_CONTRACTS.md").read_text(encoding="utf-8")
for path in (
"/datasets/raster/terrain/select",
"/datasets/raster/flood-hazard/select",
):
assert f'@router.post("{path}", response_model=dict)' in routes
assert path in contracts
assert "envelope(TerrainAnalysisService.analyze_partitions" in routes
assert "envelope(FloodHazardAnalysisService.analyze_partitions" in routes
def test_regional_map_uses_logical_partition_groups_and_exact_analysis() -> None:
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
hook = (ROOT / "frontend/src/hooks/useMapThemeSelectionInsights.ts").read_text(encoding="utf-8")
api = (ROOT / "frontend/src/services/api/datasets.ts").read_text(encoding="utf-8")
assert "regionalScopeSelected" in workspace
assert "rasterPartitionsForDataset" in workspace
assert "imageOverlays={activeImageOverlays}" in workspace
assert "de juiste gemeentelijke rasters worden automatisch gecombineerd" in workspace
assert "selectTerrainPartitions" in hook
assert "selectFloodHazardPartitions" in hook
assert "/datasets/raster/terrain/select" in api
assert "/datasets/raster/flood-hazard/select" in api
def test_maplibre_supports_multiple_persisted_raster_overlays() -> None:
map_source = (ROOT / "frontend/src/components/GeoMap.tsx").read_text(encoding="utf-8")
assert "imageOverlays?: MapImageOverlay[]" in map_source
assert "imageOverlayIdsRef" in map_source
assert "imageOverlays.forEach" in map_source
assert "bounded-raster-" in map_source
def test_regional_analysis_does_not_create_an_authoritative_mosaic() -> None:
service = (ROOT / "backend/app/services/raster_partition_analysis_service.py").read_text(encoding="utf-8")
storage = (ROOT / "docs/STORAGE_ARCHITECTURE.md").read_text(encoding="utf-8")
assert "from rasterio.merge import merge" in service
assert "DatasetService" not in service
assert "12-million-cell limit" in storage
assert "does not create another authoritative raster" in storage