feat: add governed hydrology and historical imagery
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-15 12:17:45 +02:00
parent 5b1156e989
commit fb38eb3e91
32 changed files with 1646 additions and 55 deletions
+27 -3
View File
@@ -1121,16 +1121,40 @@ is never presented as a complete result.
## Bounded official orthophoto acquisition
`POST /api/v1/projects/{project_id}/datasets/orthophoto/acquire` accepts an
explicit EPSG:4326 map rectangle and stores the official Digitaal Vlaanderen
`OMWRGBMRVL`/`Ortho` response as a canonical EPSG:31370 raster Dataset. The
`GET /api/v1/projects/{project_id}/datasets/orthophoto/products` lists the
governed product allowlist. `POST .../datasets/orthophoto/acquire` accepts an
explicit EPSG:4326 map rectangle plus `product_key` and stores the official
Digitaal Vlaanderen WMS response as a canonical EPSG:31370 raster Dataset. The
default safety envelope is 128-1,024 m per side, 1 m/pixel, 32 MiB and a
24-hour exact-request cache. It runs synchronously behind the existing Job
abstraction and never during startup.
Available products cover the most recent winter image, annual winter mosaics
for 2012-2025, three older winter periods, RGB 1979-1990 and panchromatic 1971.
Historical products persist validity metadata and are deliberately excluded
from configured-YOLO/current-GRB QA. `GET .../datasets/{dataset_id}/raster/image`
is the constrained binary PNG endpoint used by the MapLibre image overlay.
Settings: `ORTHOPHOTO_ENABLED`, `ORTHOPHOTO_WMS_URL`,
`ORTHOPHOTO_WMS_LAYER`, `ORTHOPHOTO_RESOLUTION_M`,
`ORTHOPHOTO_MIN_SIDE_M`, `ORTHOPHOTO_MAX_SIDE_M`,
`ORTHOPHOTO_TIMEOUT_SECONDS`, `ORTHOPHOTO_MAX_RESPONSE_MB` and
`ORTHOPHOTO_CACHE_TTL_HOURS`. Keep the official HTTPS URL and 1 m profile
unless a separately verified deployment/model profile requires a change.
## Waterinfo station histories
Run the explicit operator after the regional workspace and Mol Area exist:
```bash
docker exec geointel python /app/scripts/provision_waterinfo_station_history.py \
--project-name "Kempen Regional Workbench" \
--area-name "Gemeente Mol" \
--from-year 2013 --to-year 2025
```
The command retains raw KiWIS JSON/checksums and imports only real annual
observations through the canonical dataset upload API. Every station has its
own temporal-series key. Water levels and discharges remain Point measurements;
they are never averaged across stations or presented as municipal water volume.
Use `--fetch-only` to prepare and audit artifacts without persistence.
+24 -2
View File
@@ -6,10 +6,10 @@ from typing import Any
from uuid import UUID
from uuid import UUID as _UUID
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response
from fastapi import UploadFile
from sqlalchemy.orm import Session
from app.models import Area
from app.models import Area, Project
from app.core.errors import AppError
from app.db.session import get_db
@@ -143,6 +143,14 @@ def acquire_bounded_orthophoto(
return envelope(job)
@router.get("/datasets/orthophoto/products", response_model=dict)
def list_orthophoto_products(project_id: UUID, db: Session = Depends(get_db)):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
items = OrthophotoAcquisitionService.list_products()
return envelope({"items": items, "total": len(items)})
@router.get("/datasets", response_model=dict)
def list_datasets(
project_id: UUID,
@@ -426,6 +434,20 @@ def raster_preview_readiness(
return envelope(RasterOperationsService.preview(db, dataset_id))
@router.get("/datasets/{dataset_id}/raster/image")
def raster_orthophoto_image(
project_id: UUID,
dataset_id: UUID,
db: Session = Depends(get_db),
):
content = OrthophotoAcquisitionService.render_png(db, project_id, dataset_id)
return Response(
content=content,
media_type="image/png",
headers={"Cache-Control": "private, max-age=86400"},
)
@router.get("/datasets/{dataset_id}/raster/stats", response_model=dict)
def raster_stats(
project_id: UUID,
+2 -1
View File
@@ -32,7 +32,7 @@ from .segmentation import (
)
from .health import HealthResponse, SystemCapabilities
from .job import JobCreate, JobList, JobRead, JobStatus
from .orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult
from .orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult, OrthophotoProductRead
from .external import (
ExternalFetchRequest,
ExternalFetchResponse,
@@ -134,6 +134,7 @@ __all__ = [
"JobStatus",
"OrthophotoAcquireRequest",
"OrthophotoAcquisitionResult",
"OrthophotoProductRead",
"VectorBBoxResponse",
"VectorClipRequest",
"VectorBufferRequest",
+18
View File
@@ -10,13 +10,31 @@ from .operations import VectorSelectionBBox
class OrthophotoAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str = "most_recent"
force_refresh: bool = False
class OrthophotoProductRead(BaseModel):
key: str
display_name: str
observation_label: str
temporal_granularity: str
native_resolution_m: float
supports_detection: bool
color_mode: str
catalog_url: str
limitation_message: str
class OrthophotoAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
observation_label: str
temporal_granularity: str
supports_detection: bool
layer: str
width: int
height: int
+17 -1
View File
@@ -422,6 +422,11 @@ class DatasetService:
source_metadata: dict[str, Any],
provenance_metadata: dict[str, Any],
area_id: UUID | None = None,
temporal_series_key: str | None = None,
observed_at: datetime | None = None,
valid_from: datetime | None = None,
valid_to: datetime | None = None,
temporal_granularity: str | None = None,
source_version: str | None = None,
content_type: str = "image/tiff",
) -> DatasetCreateResponse:
@@ -438,6 +443,14 @@ class DatasetService:
safe_filename = DatasetService._validate_upload_filename(filename)
if DatasetService._extension_for_path(safe_filename) not in DatasetService.RASTER_EXTENSIONS:
raise AppError(code="INVALID_UPLOAD", message="Raster artifacts require a GeoTIFF filename", status_code=415)
temporal = DatasetService._validate_temporal_metadata(
temporal_series_key=temporal_series_key,
observed_at=observed_at,
valid_from=valid_from,
valid_to=valid_to,
temporal_granularity=temporal_granularity,
source_version=source_version,
)
dataset_id = uuid.uuid4()
storage_info = StorageService.persist_dataset_file(
@@ -462,7 +475,7 @@ class DatasetService:
source_metadata=source_metadata,
provenance_metadata=provenance_metadata,
imported_at=datetime.now(timezone.utc),
source_version=source_version,
**temporal,
storage_path=storage_info["storage_path"],
original_filename=storage_info["original_filename"],
stored_filename=storage_info["stored_filename"],
@@ -483,6 +496,9 @@ class DatasetService:
version=1,
storage_path=dataset.storage_path,
source_version=dataset.source_version,
observed_at=dataset.observed_at,
valid_from=dataset.valid_from,
valid_to=dataset.valid_to,
checksum_sha256=dataset.checksum_sha256,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
@@ -349,6 +349,7 @@ class GeoAssistantService:
"measurement_quality": (
"schatting" if summary["is_estimate"] else "exact_binnen_bronrepresentatie"
),
"warning": summary.get("warning"),
}
)
if dataset.id not in source_dataset_ids:
@@ -1,9 +1,11 @@
from __future__ import annotations
import hashlib
import io
import json
import math
import warnings
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any, Callable
@@ -20,21 +22,169 @@ from shapely.ops import transform as shapely_transform
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import Area, Dataset, Project
from app.schemas.orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult
from app.schemas.orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult, OrthophotoProductRead
from app.services.dataset_service import DatasetService
@dataclass(frozen=True)
class OrthophotoProduct:
key: str
display_name: str
observation_label: str
temporal_granularity: str
native_resolution_m: float
wms_url: str
layer: str
catalog_url: str
limitation_message: str
supports_detection: bool = False
color_mode: str = "rgb"
observed_at: datetime | None = None
valid_from: datetime | None = None
valid_to: datetime | None = None
class OrthophotoAcquisitionService:
PROVIDER = "digitaal_vlaanderen_orthophoto"
ATTRIBUTION = "Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen"
CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/orthofotomozaiek-middenschalig-winteropnamen-kleur-meest-recent-vlaanderen"
LIMITATION = "Meest recente samengestelde winterorthofoto op het moment van de aanvraag; geen historische opnamedatum per pixel."
HISTORICAL_WINTER_WMS_URL = "https://geo.api.vlaanderen.be/OMW/wms"
HISTORICAL_WINTER_CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/wmts-orthofotomozaiek-middenschalig-winteropnamen"
HISTORICAL_SUMMER_WMS_URL = "https://geo.api.vlaanderen.be/OKZ/wms"
HISTORICAL_SUMMER_CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/orthofotomozaiek-kleinschalig-zomeropnamen"
@staticmethod
def _products(settings: Settings) -> dict[str, OrthophotoProduct]:
products: list[OrthophotoProduct] = [
OrthophotoProduct(
key="most_recent",
display_name="Meest recente winterluchtbeeld",
observation_label="Meest recent beschikbaar",
temporal_granularity="snapshot",
native_resolution_m=0.15,
wms_url=settings.orthophoto_wms_url,
layer=settings.orthophoto_wms_layer,
catalog_url=OrthophotoAcquisitionService.CATALOG_URL,
limitation_message=OrthophotoAcquisitionService.LIMITATION,
supports_detection=True,
)
]
for year in range(2025, 2011, -1):
products.append(
OrthophotoProduct(
key=str(year),
display_name=f"Winterluchtbeeld {year}",
observation_label=str(year),
temporal_granularity="year",
native_resolution_m=0.15 if year >= 2022 else 0.25,
wms_url=OrthophotoAcquisitionService.HISTORICAL_WINTER_WMS_URL,
layer=f"OMWRGB{year % 100:02d}VL",
catalog_url=OrthophotoAcquisitionService.HISTORICAL_WINTER_CATALOG_URL,
limitation_message=(
"Officiële samengestelde winterorthofoto voor deze jaargang; de exacte opnamedatum kan per tegel verschillen. "
"Historische beelden worden niet met de actuele GRB-toestand gevalideerd."
),
observed_at=datetime(year, 1, 1, tzinfo=UTC),
valid_from=datetime(year, 1, 1, tzinfo=UTC),
valid_to=datetime(year, 12, 31, 23, 59, 59, tzinfo=UTC),
)
)
for key, start_year, end_year, layer in (
("2008_2011", 2008, 2011, "OMWRGB08_11VL"),
("2005_2007", 2005, 2007, "OMWRGB05_07VL"),
("2000_2003", 2000, 2003, "OMWRGB00_03VL"),
):
products.append(
OrthophotoProduct(
key=key,
display_name=f"Winterluchtbeeld {start_year}-{end_year}",
observation_label=f"{start_year}-{end_year}",
temporal_granularity="period",
native_resolution_m=0.25,
wms_url=OrthophotoAcquisitionService.HISTORICAL_WINTER_WMS_URL,
layer=layer,
catalog_url=OrthophotoAcquisitionService.HISTORICAL_WINTER_CATALOG_URL,
limitation_message=(
"Officiële samengestelde winterorthofoto uit een meerjarige opnameperiode; dit is geen exacte jaaropname. "
"Historische beelden worden niet met de actuele GRB-toestand gevalideerd."
),
observed_at=datetime(start_year, 1, 1, tzinfo=UTC),
valid_from=datetime(start_year, 1, 1, tzinfo=UTC),
valid_to=datetime(end_year, 12, 31, 23, 59, 59, tzinfo=UTC),
)
)
products.extend(
[
OrthophotoProduct(
key="1979_1990",
display_name="Zomerluchtbeeld 1979-1990",
observation_label="1979-1990",
temporal_granularity="period",
native_resolution_m=1.0,
wms_url=OrthophotoAcquisitionService.HISTORICAL_SUMMER_WMS_URL,
layer="OKZRGB79_90VL",
catalog_url=OrthophotoAcquisitionService.HISTORICAL_SUMMER_CATALOG_URL,
limitation_message="Kleinschalig RGB-mozaïek uit meerdere zomervluchten tussen 1979 en 1990; geen exacte jaartoestand.",
observed_at=datetime(1979, 1, 1, tzinfo=UTC),
valid_from=datetime(1979, 1, 1, tzinfo=UTC),
valid_to=datetime(1990, 12, 31, 23, 59, 59, tzinfo=UTC),
),
OrthophotoProduct(
key="1971",
display_name="Zomerluchtbeeld 1971",
observation_label="1971",
temporal_granularity="year",
native_resolution_m=1.0,
wms_url=OrthophotoAcquisitionService.HISTORICAL_SUMMER_WMS_URL,
layer="OKZPAN71VL",
catalog_url=OrthophotoAcquisitionService.HISTORICAL_SUMMER_CATALOG_URL,
limitation_message="Kleinschalig panchromatisch mozaïek uit 1971; zwart-wit en niet geschikt voor het huidige RGB-detectiemodel.",
color_mode="panchromatic",
observed_at=datetime(1971, 1, 1, tzinfo=UTC),
valid_from=datetime(1971, 1, 1, tzinfo=UTC),
valid_to=datetime(1971, 12, 31, 23, 59, 59, tzinfo=UTC),
),
]
)
return {product.key: product for product in products}
@staticmethod
def list_products(settings: Settings | None = None) -> list[dict[str, Any]]:
resolved_settings = settings or get_settings()
return [
OrthophotoProductRead(
key=product.key,
display_name=product.display_name,
observation_label=product.observation_label,
temporal_granularity=product.temporal_granularity,
native_resolution_m=product.native_resolution_m,
supports_detection=product.supports_detection,
color_mode=product.color_mode,
catalog_url=product.catalog_url,
limitation_message=product.limitation_message,
).model_dump()
for product in OrthophotoAcquisitionService._products(resolved_settings).values()
]
@staticmethod
def _product(product_key: str, settings: Settings) -> OrthophotoProduct:
product = OrthophotoAcquisitionService._products(settings).get(product_key.strip().lower())
if product is None:
raise AppError(
code="ORTHOPHOTO_PRODUCT_NOT_SUPPORTED",
message="Select an orthophoto product from the official product registry",
details={"product_key": product_key},
status_code=422,
)
return product
@staticmethod
def _prepared_request(
payload: OrthophotoAcquireRequest,
settings: Settings,
) -> dict[str, Any]:
product = OrthophotoAcquisitionService._product(payload.product_key, settings)
if payload.bbox.crs.upper() != "EPSG:4326":
raise AppError(code="INVALID_CRS", message="Orthophoto selection bbox must use EPSG:4326", status_code=400)
min_x = float(payload.bbox.min_x)
@@ -68,8 +218,9 @@ class OrthophotoAcquisitionService:
bbox_31370 = [float(value) for value in lambert_bounds]
request_identity = {
"provider": OrthophotoAcquisitionService.PROVIDER,
"wms_url": settings.orthophoto_wms_url,
"layer": settings.orthophoto_wms_layer,
"product_key": product.key,
"wms_url": product.wms_url,
"layer": product.layer,
"bbox_epsg4326": [round(value, 8) for value in bbox_4326],
"bbox_epsg31370": [round(value, 3) for value in bbox_31370],
"width": width,
@@ -77,11 +228,18 @@ class OrthophotoAcquisitionService:
"resolution_m": settings.orthophoto_resolution_m,
}
request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode("utf-8")).hexdigest()
spatial_identity = {
"bbox_epsg4326": request_identity["bbox_epsg4326"],
"width": width,
"height": height,
"resolution_m": settings.orthophoto_resolution_m,
}
spatial_hash = hashlib.sha256(json.dumps(spatial_identity, sort_keys=True).encode("utf-8")).hexdigest()
params = {
"SERVICE": "WMS",
"VERSION": "1.3.0",
"REQUEST": "GetMap",
"LAYERS": settings.orthophoto_wms_layer,
"LAYERS": product.layer,
"STYLES": "",
"FORMAT": "image/tiff",
"CRS": "EPSG:31370",
@@ -91,8 +249,10 @@ class OrthophotoAcquisitionService:
}
return {
**request_identity,
"product": product,
"spatial_hash": spatial_hash,
"request_hash": request_hash,
"request_url": f"{settings.orthophoto_wms_url}?{urlencode(params)}",
"request_url": f"{product.wms_url}?{urlencode(params)}",
"params": params,
"bbox_epsg4326": bbox_4326,
"bbox_epsg31370": bbox_31370,
@@ -124,8 +284,14 @@ class OrthophotoAcquisitionService:
)
@staticmethod
def _cached_dataset(db, project_id: UUID, filename: str, settings: Settings) -> Dataset | None:
if settings.orthophoto_cache_ttl_hours <= 0:
def _cached_dataset(
db,
project_id: UUID,
filename: str,
settings: Settings,
product: OrthophotoProduct,
) -> Dataset | None:
if product.key == "most_recent" and settings.orthophoto_cache_ttl_hours <= 0:
return None
candidate = (
db.query(Dataset)
@@ -145,7 +311,7 @@ class OrthophotoAcquisitionService:
return None
if imported_at.tzinfo is None:
imported_at = imported_at.replace(tzinfo=UTC)
if datetime.now(UTC) - imported_at > timedelta(hours=settings.orthophoto_cache_ttl_hours):
if product.key == "most_recent" and datetime.now(UTC) - imported_at > timedelta(hours=settings.orthophoto_cache_ttl_hours):
return None
return candidate
@@ -196,7 +362,9 @@ class OrthophotoAcquisitionService:
with warnings.catch_warnings():
warnings.simplefilter("ignore", NotGeoreferencedWarning)
with source_memory.open() as source:
if source.width != prepared["width"] or source.height != prepared["height"] or source.count < 3:
product: OrthophotoProduct = prepared["product"]
minimum_band_count = 1 if product.color_mode == "panchromatic" else 3
if source.width != prepared["width"] or source.height != prepared["height"] or source.count < minimum_band_count:
raise AppError(
code="ORTHOPHOTO_PROVIDER_INVALID_RESPONSE",
message="Official orthophoto dimensions or RGB bands do not match the bounded request",
@@ -216,7 +384,7 @@ class OrthophotoAcquisitionService:
with output_memory.open(**profile) as output:
output.write(image)
output.update_tags(
source="Digitaal Vlaanderen OMWRGBMRVL WMS Ortho layer",
source=f"Digitaal Vlaanderen WMS {product.layer}",
source_url=prepared["request_url"],
attribution=OrthophotoAcquisitionService.ATTRIBUTION,
acquisition="explicit_bounded_map_selection",
@@ -245,44 +413,74 @@ class OrthophotoAcquisitionService:
if not resolved_settings.orthophoto_enabled:
raise AppError(code="ORTHOPHOTO_NOT_CONFIGURED", message="Official orthophoto acquisition is disabled", status_code=503)
prepared = OrthophotoAcquisitionService._prepared_request(payload, resolved_settings)
product: OrthophotoProduct = prepared["product"]
OrthophotoAcquisitionService._validate_area_scope(db, project_id, payload.area_id, prepared["bbox_epsg4326"])
filename = f"orthofoto_selectie_{prepared['request_hash'][:12]}.tif"
filename = f"orthofoto_{product.key}_{prepared['request_hash'][:12]}.tif"
cached = None if payload.force_refresh else OrthophotoAcquisitionService._cached_dataset(db, project_id, filename, resolved_settings)
cached = None if payload.force_refresh else OrthophotoAcquisitionService._cached_dataset(
db,
project_id,
filename,
resolved_settings,
product,
)
if cached is not None:
return OrthophotoAcquisitionResult(
output_dataset_id=cached.id,
reused=True,
provider=OrthophotoAcquisitionService.PROVIDER,
layer=resolved_settings.orthophoto_wms_layer,
product_key=product.key,
display_name=product.display_name,
observation_label=product.observation_label,
temporal_granularity=product.temporal_granularity,
supports_detection=product.supports_detection,
layer=product.layer,
width=prepared["width"],
height=prepared["height"],
resolution_m=resolved_settings.orthophoto_resolution_m,
bbox_epsg4326=prepared["bbox_epsg4326"],
bbox_epsg31370=prepared["bbox_epsg31370"],
attribution=OrthophotoAcquisitionService.ATTRIBUTION,
limitation_message=OrthophotoAcquisitionService.LIMITATION,
limitation_message=product.limitation_message,
).model_dump(mode="json")
raw_content, response_content_type = OrthophotoAcquisitionService._fetch(prepared["request_url"], resolved_settings, opener)
geotiff_content = OrthophotoAcquisitionService._georeference_tiff(raw_content, prepared)
acquired_at = datetime.now(UTC)
observed_at = product.observed_at or acquired_at
dataset = DatasetService.import_raster_bytes(
db,
project_id=project_id,
area_id=payload.area_id,
filename=filename,
content=geotiff_content,
source="Digitaal Vlaanderen OMWRGBMRVL WMS",
source=f"Digitaal Vlaanderen WMS {product.layer}",
source_name=OrthophotoAcquisitionService.PROVIDER,
source_version=f"most_recent_at_{acquired_at.date().isoformat()}",
temporal_series_key=f"digitaal-vlaanderen:orthophoto:{prepared['spatial_hash'][:24]}",
observed_at=observed_at,
valid_from=product.valid_from or observed_at,
valid_to=product.valid_to,
temporal_granularity=product.temporal_granularity,
source_version=(
f"most_recent_at_{acquired_at.date().isoformat()}"
if product.key == "most_recent"
else product.key
),
content_type="image/tiff",
source_metadata={
"provider": OrthophotoAcquisitionService.PROVIDER,
"service": "WMS",
"service_version": "1.3.0",
"layer": resolved_settings.orthophoto_wms_layer,
"catalog_url": OrthophotoAcquisitionService.CATALOG_URL,
"product_key": product.key,
"product_display_name": product.display_name,
"observation_label": product.observation_label,
"observation_date_precision": product.temporal_granularity,
"native_resolution_m": product.native_resolution_m,
"requested_resolution_m": resolved_settings.orthophoto_resolution_m,
"color_mode": product.color_mode,
"supports_detection": product.supports_detection,
"layer": product.layer,
"catalog_url": product.catalog_url,
"attribution": OrthophotoAcquisitionService.ATTRIBUTION,
"license_note": "Gebruik volgens het gebruiksrecht geografische webdiensten van Digitaal Vlaanderen.",
},
@@ -290,6 +488,7 @@ class OrthophotoAcquisitionService:
"acquisition": "explicit_bounded_map_selection",
"acquired_at": acquired_at.isoformat(),
"request_hash": prepared["request_hash"],
"spatial_hash": prepared["spatial_hash"],
"request_url": prepared["request_url"],
"response_content_type": response_content_type,
"bbox_epsg4326": prepared["bbox_epsg4326"],
@@ -297,19 +496,70 @@ class OrthophotoAcquisitionService:
"width": prepared["width"],
"height": prepared["height"],
"resolution_m": resolved_settings.orthophoto_resolution_m,
"limitation_message": OrthophotoAcquisitionService.LIMITATION,
"limitation_message": product.limitation_message,
},
)
return OrthophotoAcquisitionResult(
output_dataset_id=dataset.id,
reused=False,
provider=OrthophotoAcquisitionService.PROVIDER,
layer=resolved_settings.orthophoto_wms_layer,
product_key=product.key,
display_name=product.display_name,
observation_label=product.observation_label,
temporal_granularity=product.temporal_granularity,
supports_detection=product.supports_detection,
layer=product.layer,
width=prepared["width"],
height=prepared["height"],
resolution_m=resolved_settings.orthophoto_resolution_m,
bbox_epsg4326=prepared["bbox_epsg4326"],
bbox_epsg31370=prepared["bbox_epsg31370"],
attribution=OrthophotoAcquisitionService.ATTRIBUTION,
limitation_message=OrthophotoAcquisitionService.LIMITATION,
limitation_message=product.limitation_message,
).model_dump(mode="json")
@staticmethod
def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1600) -> bytes:
dataset = db.get(Dataset, dataset_id)
if (
dataset is None
or dataset.project_id != project_id
or dataset.source_name != OrthophotoAcquisitionService.PROVIDER
or dataset.status != "ready"
or not dataset.storage_path
or not Path(dataset.storage_path).is_file()
):
raise AppError(code="ORTHOPHOTO_NOT_FOUND", message="Orthophoto dataset not found", status_code=404)
try:
import numpy as np
import rasterio
from PIL import Image
from rasterio.enums import Resampling
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Raster preview dependencies are unavailable", status_code=503) from exc
try:
with rasterio.open(dataset.storage_path) as source:
scale = min(1.0, max_dimension / max(source.width, source.height))
width = max(1, round(source.width * scale))
height = max(1, round(source.height * scale))
indexes = [1] if source.count == 1 else list(range(1, min(source.count, 3) + 1))
pixels = source.read(indexes, out_shape=(len(indexes), height, width), resampling=Resampling.bilinear)
if pixels.dtype != np.uint8:
pixels = np.clip(pixels, 0, 255).astype(np.uint8)
if len(indexes) == 1:
image = Image.fromarray(pixels[0])
else:
image = Image.fromarray(np.moveaxis(pixels[:3], 0, 2))
output = io.BytesIO()
image.save(output, format="PNG", optimize=True)
return output.getvalue()
except AppError:
raise
except Exception as exc:
raise AppError(
code="ORTHOPHOTO_PREVIEW_FAILED",
message="The persisted orthophoto could not be rendered",
details={"reason": str(exc)},
status_code=500,
) from exc
@@ -23,6 +23,7 @@ FULL_AREA_CLIPPED_OPERATOR_TOOLS = {
"provision_official_landuse_timeseries.py",
"provision_regional_grb_buildings.py",
"provision_regional_grb_context.py",
"provision_waterinfo_station_history.py",
}
@@ -439,7 +440,7 @@ class VectorFeatureService:
length_m = db.query(func.coalesce(func.sum(length_expression), 0.0)).filter(*metric_filter).scalar()
divisor = 1_000.0 if unit == "km" else 1.0
metric_value = float(length_m or 0.0) / divisor
elif method in {"sum", "area_weighted_sum"}:
elif method in {"sum", "mean", "area_weighted_sum"}:
property_name = str(config.get("property") or "").strip()
if not property_name:
raise AppError(
@@ -457,8 +458,9 @@ class VectorFeatureService:
)
coverage_ratio = intersection_area / func.nullif(source_area, 0.0)
value_expression = numeric_value * coverage_ratio
aggregate_function = func.avg if method == "mean" else func.sum
aggregate_value = (
db.query(func.coalesce(func.sum(value_expression), 0.0))
db.query(func.coalesce(aggregate_function(value_expression), 0.0))
.filter(*selection_filter)
.filter(VectorFeature.properties_json.op("->>")(property_name).isnot(None))
.scalar()
@@ -86,7 +86,13 @@ class FakeImageResponse:
return self.content[:limit]
def _selection_payload(*, side_m: float = 512.0, force_refresh: bool = True, area_id=None) -> OrthophotoAcquireRequest:
def _selection_payload(
*,
side_m: float = 512.0,
force_refresh: bool = True,
area_id=None,
product_key: str = "most_recent",
) -> OrthophotoAcquireRequest:
west, south = 199_000.0, 210_000.0
transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
min_lon, min_lat = transformer.transform(west, south)
@@ -100,6 +106,7 @@ def _selection_payload(*, side_m: float = 512.0, force_refresh: bool = True, are
"crs": "EPSG:4326",
},
area_id=area_id,
product_key=product_key,
force_refresh=force_refresh,
)
@@ -136,6 +143,25 @@ def test_orthophoto_request_is_bounded_and_uses_official_wms_contract() -> None:
assert len(prepared["request_hash"]) == 64
def test_orthophoto_product_registry_exposes_only_governed_official_layers() -> None:
settings = Settings(_env_file=None)
products = OrthophotoAcquisitionService.list_products(settings)
keys = [item["key"] for item in products]
assert keys[0] == "most_recent"
assert {"2025", "2012", "2008_2011", "2000_2003", "1979_1990", "1971"}.issubset(keys)
assert next(item for item in products if item["key"] == "most_recent")["supports_detection"] is True
assert all(item["supports_detection"] is False for item in products if item["key"] != "most_recent")
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings)
assert prepared["params"]["LAYERS"] == "OKZPAN71VL"
assert prepared["wms_url"] == "https://geo.api.vlaanderen.be/OKZ/wms"
with pytest.raises(AppError) as exc_info:
OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="arbitrary-layer"), settings)
assert exc_info.value.code == "ORTHOPHOTO_PRODUCT_NOT_SUPPORTED"
@pytest.mark.parametrize(
("side_m", "expected_code"),
[(64.0, "ORTHOPHOTO_SELECTION_TOO_SMALL"), (1_200.0, "ORTHOPHOTO_SELECTION_TOO_LARGE")],
@@ -240,7 +266,7 @@ def test_orthophoto_acquisition_reuses_fresh_exact_request_without_provider_call
cached = Dataset(
id=uuid4(),
project_id=project_id,
name=f"orthofoto_selectie_{prepared['request_hash'][:12]}.tif",
name=f"orthofoto_most_recent_{prepared['request_hash'][:12]}.tif",
dataset_type="raster",
source="Digitaal Vlaanderen",
source_name="digitaal_vlaanderen_orthophoto",
@@ -277,6 +303,54 @@ def test_orthophoto_provider_rejects_non_image_response() -> None:
assert exc_info.value.code == "ORTHOPHOTO_PROVIDER_INVALID_RESPONSE"
def test_historical_orthophoto_persists_temporal_product_provenance(tmp_path) -> None:
project_id = uuid4()
payload = _selection_payload(product_key="2020")
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
settings = Settings(_env_file=None, storage_root=str(tmp_path), orthophoto_resolution_m=1.0)
prepared = OrthophotoAcquisitionService._prepared_request(payload, settings)
response = FakeImageResponse(_source_tiff(prepared["width"], prepared["height"]))
result = OrthophotoAcquisitionService.acquire(
db,
project_id,
payload,
settings=settings,
opener=lambda *_args, **_kwargs: response,
)
dataset = next(row for row in db.added if isinstance(row, Dataset))
assert result["product_key"] == "2020"
assert result["supports_detection"] is False
assert dataset.observed_at.year == 2020
assert dataset.temporal_granularity == "year"
assert dataset.source_metadata["layer"] == "OMWRGB20VL"
assert dataset.source_metadata["product_key"] == "2020"
assert dataset.provenance_metadata["spatial_hash"] == prepared["spatial_hash"]
def test_persisted_orthophoto_renders_browser_png(tmp_path) -> None:
project_id = uuid4()
dataset_id = uuid4()
path = tmp_path / "ortho.tif"
path.write_bytes(_source_tiff(32, 24))
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="ortho.tif",
dataset_type="raster",
source="Digitaal Vlaanderen",
source_name="digitaal_vlaanderen_orthophoto",
status="ready",
storage_path=str(path),
)
db = FakeSession({(Dataset, dataset_id): dataset})
png = OrthophotoAcquisitionService.render_png(db, project_id, dataset_id)
assert png.startswith(b"\x89PNG\r\n\x1a\n")
def test_orthophoto_endpoint_returns_canonical_job_envelope(monkeypatch) -> None:
project_id = uuid4()
output_dataset_id = uuid4()
@@ -307,6 +381,22 @@ def test_orthophoto_endpoint_returns_canonical_job_envelope(monkeypatch) -> None
assert any(isinstance(row, Job) for row in db.added)
def test_orthophoto_product_endpoint_returns_canonical_envelope() -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
app.dependency_overrides[get_db] = lambda: db
try:
response = TestClient(app).get(f"/api/v1/projects/{project_id}/datasets/orthophoto/products")
finally:
app.dependency_overrides.clear()
assert response.status_code == 200
body = response.json()
assert set(body) == {"data"}
assert body["data"]["total"] == len(body["data"]["items"])
assert body["data"]["items"][0]["key"] == "most_recent"
def test_frontend_connects_map_selection_to_existing_detection_and_qa_flows() -> None:
app_source = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8")
hook_source = (ROOT / "frontend" / "src" / "hooks" / "useMapOrthophotoAnalysis.ts").read_text(encoding="utf-8")
@@ -114,6 +114,36 @@ def test_population_keeps_configured_metric_and_adds_sector_count() -> None:
}
def test_station_measurement_uses_numeric_mean_without_area_extrapolation() -> None:
dataset = themed_dataset("water", method="mean")
dataset.source_name = "waterinfo"
dataset.source_metadata.update(
{
"semantic_metrics": False,
"selection_aggregation": {
"metric_key": "water_level",
"method": "mean",
"property": "annual_mean_water_level_m",
"label": "Jaargemiddelde waterstand",
"unit": "m",
"warning": "Puntmeting; geen gebiedsdekkend watervolume.",
},
}
)
result = VectorFeatureService.summarize_features_by_bbox(
SequenceScalarSession([30.455]),
dataset=dataset,
bbox=BBOX,
total_feature_count=1,
)
assert result["metric_value"] == 30.455
assert result["aggregation_method"] == "mean"
assert result["metric_unit"] == "m"
assert result["warning"] == "Puntmeting; geen gebiedsdekkend watervolume."
def test_future_regional_imports_persist_semantic_aggregation_configuration() -> None:
buildings = (ROOT / "scripts/provision_regional_grb_buildings.py").read_text(encoding="utf-8")
context = (ROOT / "scripts/provision_regional_grb_context.py").read_text(encoding="utf-8")
@@ -0,0 +1,135 @@
from __future__ import annotations
import importlib.util
import sys
from pathlib import Path
from shapely.geometry import box
ROOT = Path(__file__).resolve().parents[2]
def load_operator():
script_path = ROOT / "scripts" / "provision_waterinfo_station_history.py"
spec = importlib.util.spec_from_file_location("waterinfo_history_operator", script_path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
class JsonResponse:
ok = True
status_code = 200
text = ""
def __init__(self, payload):
self.payload = payload
def raise_for_status(self):
return None
def json(self):
return self.payload
class JsonSession:
def __init__(self, payloads):
self.payloads = iter(payloads)
self.calls = []
def get(self, url, *, params, timeout):
self.calls.append((url, params, timeout))
return JsonResponse(next(self.payloads))
def test_waterinfo_station_discovery_filters_exact_area_and_uses_annual_group() -> None:
module = load_operator()
payload = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.1, 51.2]},
"properties": {"ts_id": 5319042, "station_no": "L10_089", "station_name": "Mol/ScheppelijkeNete"},
},
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [6.0, 52.0]},
"properties": {"ts_id": 999, "station_no": "outside", "station_name": "Outside"},
},
],
}
session = JsonSession([payload])
raw, stations = module.discover_station_series(
session,
module.PARAMETERS["water_level"],
box(5.0, 51.0, 5.3, 51.4),
timeout=30,
)
assert raw == payload
assert [item["ts_id"] for item in stations] == ["5319042"]
assert session.calls[0][1]["timeseriesgroup_id"] == "192784"
assert session.calls[0][1]["request"] == "getTimeseriesValueLayer"
def test_waterinfo_annual_values_reject_invalid_sentinel_and_keep_real_zero() -> None:
module = load_operator()
payload = [
{
"ts_id": 5319042,
"data": [
["2013-01-01T00:00:00.000+01:00", 30.46],
["2014-01-01T00:00:00.000+01:00", -9999],
["2015-01-01T00:00:00.000+01:00", 0.0],
["2026-01-01T00:00:00.000+01:00", 99.0],
],
}
]
session = JsonSession([payload])
raw, values = module.fetch_annual_values(session, "5319042", from_year=2013, to_year=2025, timeout=30)
assert raw == payload
assert values == {2013: 30.46, 2015: 0.0}
assert session.calls[0][1]["request"] == "getTimeseriesValues"
def test_waterinfo_snapshot_and_series_keep_station_identity_and_honest_metric() -> None:
module = load_operator()
parameter = module.PARAMETERS["water_level"]
station = {
"ts_id": "5319042",
"geometry": {"type": "Point", "coordinates": [5.1, 51.2]},
"properties": {
"station_id": "123",
"station_no": "L10_089",
"station_name": "Mol/ScheppelijkeNete",
"ts_unitsymbol": "m",
},
}
snapshot = module.build_snapshot(parameter, station, 2025, 30.455)
feature = snapshot["features"][0]
assert module.series_key(parameter, station) == "waterinfo:water_level:annual:l10-089"
assert feature["geometry"]["type"] == "Point"
assert feature["properties"]["annual_mean_water_level_m"] == 30.455
assert feature["properties"]["timeseries_id"] == "5319042"
assert "volume" in parameter.limitation
def test_waterinfo_operator_is_packaged_and_readiness_checked() -> None:
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
vector_service = (ROOT / "backend" / "app" / "services" / "vector_feature_service.py").read_text(encoding="utf-8")
assert "py_compile scripts/provision_waterinfo_station_history.py" in readiness
assert "COPY scripts/provision_waterinfo_station_history.py" in dockerfile
assert '"provision_waterinfo_station_history.py"' in vector_service
assert '"sum", "mean", "area_weighted_sum"' in vector_service