feat: add cross-domain Mol data profile
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-16 03:06:36 +02:00
parent 4aa0d6da24
commit 035ec3b233
39 changed files with 3049 additions and 22 deletions
+37
View File
@@ -1289,6 +1289,43 @@ Settings: `FLOOD_HAZARD_ENABLED`, `FLOOD_HAZARD_WCS_URL`,
`FLOOD_HAZARD_MAX_SIDE_M`, `FLOOD_HAZARD_MAX_PIXELS`,
`FLOOD_HAZARD_TIMEOUT_SECONDS` and `FLOOD_HAZARD_MAX_RESPONSE_MB`.
## Cross-domain thematic rasters and DOV soil
The governed thematic registry exposes five fixed MercatorNet products through
`GET .../datasets/thematic-raster/products`. Acquisition uses
`POST .../datasets/thematic-raster/acquire`; selection and PNG rendering use
`POST .../raster/thematic/select` and `GET .../raster/thematic/image`.
Provision every product for the exact persisted Mol Area:
```bash
docker exec geointel python /app/scripts/provision_thematic_rasters.py
```
Inspect the complete 28-municipality matrix without writes, then run it after
the Mol source/runtime gate passes:
```bash
docker exec geointel python /app/scripts/provision_thematic_rasters.py \
--project-name "Kempen Regional Workbench" --all-municipalities --dry-run
```
Settings: `THEMATIC_RASTER_ENABLED`, `THEMATIC_RASTER_WCS_URL`,
`THEMATIC_RASTER_MIN_SIDE_M`, `THEMATIC_RASTER_MAX_SIDE_M`,
`THEMATIC_RASTER_MAX_PIXELS`, `THEMATIC_RASTER_TIMEOUT_SECONDS` and
`THEMATIC_RASTER_MAX_RESPONSE_MB`.
Provision the official DOV soil polygons for Mol through the existing vector
upload path:
```bash
docker exec geointel python /app/scripts/provision_mol_soil_map.py
```
Use `--fetch-only` to retain and validate source evidence without importing.
The operator never writes directly to PostGIS. Soil drainage and related map
classes represent the 1949-1971 survey and are not current observations.
## Waterinfo station histories
Run the explicit operator after the regional workspace and Mol Area exist:
+52
View File
@@ -26,6 +26,8 @@ from app.schemas import (
TerrainSelectionRequest,
FloodHazardAcquireRequest,
FloodHazardSelectionRequest,
ThematicRasterAcquireRequest,
ThematicRasterSelectionRequest,
VectorBBoxResponse,
VectorBufferRequest,
VectorClipRequest,
@@ -48,6 +50,8 @@ from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.terrain_analysis_service import TerrainAnalysisService
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}", tags=["datasets"])
@@ -207,6 +211,30 @@ def list_flood_hazard_products(project_id: UUID, db: Session = Depends(get_db)):
return envelope({"items": items, "total": len(items)})
@router.post("/datasets/thematic-raster/acquire", response_model=dict)
def acquire_bounded_thematic_raster(
project_id: UUID,
payload: ThematicRasterAcquireRequest,
db: Session = Depends(get_db),
):
job = JobService.run_sync_job(
db=db,
project_id=project_id,
job_type="raster.thematic.acquire",
parameters=payload.model_dump(mode="json"),
operation=lambda: ThematicRasterAcquisitionService.acquire(db, project_id, payload),
)
return envelope(job)
@router.get("/datasets/thematic-raster/products", response_model=dict)
def list_thematic_raster_products(project_id: UUID, db: Session = Depends(get_db)):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
items = ThematicRasterAcquisitionService.list_products()
return envelope({"items": items, "total": len(items)})
@router.get("/datasets", response_model=dict)
def list_datasets(
project_id: UUID,
@@ -560,6 +588,30 @@ def raster_flood_hazard_image(
)
@router.post("/datasets/{dataset_id}/raster/thematic/select", response_model=dict)
def raster_thematic_selection(
project_id: UUID,
dataset_id: UUID,
payload: ThematicRasterSelectionRequest,
db: Session = Depends(get_db),
):
return envelope(ThematicRasterAnalysisService.analyze(db, project_id, dataset_id, payload))
@router.get("/datasets/{dataset_id}/raster/thematic/image")
def raster_thematic_image(
project_id: UUID,
dataset_id: UUID,
db: Session = Depends(get_db),
):
content = ThematicRasterAnalysisService.render_png(db, project_id, dataset_id)
return Response(
content=content,
media_type="image/png",
headers={"Cache-Control": "private, max-age=86400"},
)
@router.get("/datasets/{dataset_id}/raster/stats", response_model=dict)
def raster_stats(
project_id: UUID,
+10
View File
@@ -53,6 +53,16 @@ class Settings(BaseSettings):
flood_hazard_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="FLOOD_HAZARD_MAX_PIXELS")
flood_hazard_timeout_seconds: int = Field(default=300, ge=1, validation_alias="FLOOD_HAZARD_TIMEOUT_SECONDS")
flood_hazard_max_response_mb: int = Field(default=160, ge=1, validation_alias="FLOOD_HAZARD_MAX_RESPONSE_MB")
thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED")
thematic_raster_wcs_url: str = Field(
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
validation_alias="THEMATIC_RASTER_WCS_URL",
)
thematic_raster_min_side_m: float = Field(default=100.0, gt=0, validation_alias="THEMATIC_RASTER_MIN_SIDE_M")
thematic_raster_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="THEMATIC_RASTER_MAX_SIDE_M")
thematic_raster_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS")
thematic_raster_timeout_seconds: int = Field(default=300, ge=1, validation_alias="THEMATIC_RASTER_TIMEOUT_SECONDS")
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
+16
View File
@@ -51,6 +51,15 @@ from .flood_hazard import (
FloodHazardSelectionResponse,
FloodHazardSelectionSummary,
)
from .thematic_raster import (
ThematicRasterAcquireRequest,
ThematicRasterAcquisitionResult,
ThematicRasterMetric,
ThematicRasterProductRead,
ThematicRasterSelectionRequest,
ThematicRasterSelectionResponse,
ThematicRasterSelectionSummary,
)
from .external import (
ExternalFetchRequest,
ExternalFetchResponse,
@@ -167,6 +176,13 @@ __all__ = [
"FloodHazardSelectionRequest",
"FloodHazardSelectionResponse",
"FloodHazardSelectionSummary",
"ThematicRasterAcquireRequest",
"ThematicRasterAcquisitionResult",
"ThematicRasterMetric",
"ThematicRasterProductRead",
"ThematicRasterSelectionRequest",
"ThematicRasterSelectionResponse",
"ThematicRasterSelectionSummary",
"VectorBBoxResponse",
"VectorClipRequest",
"VectorBufferRequest",
+96
View File
@@ -0,0 +1,96 @@
from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel
from .operations import VectorSelectionBBox
class ThematicRasterAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
product_key: str
force_refresh: bool = False
class ThematicRasterProductRead(BaseModel):
key: str
display_name: str
theme: str
metric_kind: str
coverage_id: str
native_resolution_m: float
source_crs: str
source_value_unit: str
observation_year: int
source_version: str
catalog_url: str
attribution: str
license_note: str
legend_min_label: str
legend_max_label: str
limitation_message: str
class ThematicRasterAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
theme: str
metric_kind: str
coverage_id: str
resolution_m: float
width: int
height: int
valid_pixel_count: int
bbox_epsg4326: list[float]
bbox_epsg31370: list[float]
observation_year: int
source_value_unit: str
attribution: str
limitation_message: str
class ThematicRasterSelectionRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
class ThematicRasterMetric(BaseModel):
metric_key: str
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
derived: bool = True
is_estimate: bool = True
class ThematicRasterSelectionSummary(BaseModel):
metric_label: str
metric_value: float
metric_unit: str
aggregation_method: str
primary_metric_key: str
metrics: list[ThematicRasterMetric]
class ThematicRasterSelectionResponse(BaseModel):
dataset_id: UUID
product_key: str
theme: str
metric_kind: str
selection_bbox: VectorSelectionBBox
selection_area_id: UUID | None = None
selected_cell_count: int
valid_cell_count: int
coverage_ratio: float
resolution_m: float
observation_year: int
summary: ThematicRasterSelectionSummary
unsupported_metrics: list[str]
limitation_message: str
generated_at: str
@@ -22,8 +22,11 @@ from app.schemas.assistant import (
AssistantTemporalSeries,
)
from app.schemas.flood_hazard import FloodHazardSelectionRequest
from app.schemas.thematic_raster import ThematicRasterSelectionRequest
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
from app.services.vector_feature_service import VectorFeatureService
@@ -42,9 +45,17 @@ class GeoAssistantService:
)
ESTIMATE_TOPIC_TERMS = {
"population": ("bevolk", "inwoner"),
"space_occupation": ("ruimtebeslag",),
"open_space": ("open ruimte",),
"accessibility": ("bereikbaar", "knooppunt"),
"services": ("voorziening",),
}
ESTIMATE_TOPIC_LABELS = {
"population": "bevolkingswaarden",
"space_occupation": "ruimtebeslagoppervlakten",
"open_space": "openruimte-oppervlakten",
"accessibility": "bereikbaarheidsscores",
"services": "voorzieningenscores",
}
@classmethod
@@ -264,6 +275,19 @@ class GeoAssistantService:
if dataset.dataset_type == "raster" and dataset.source_name == FloodHazardAcquisitionService.PROVIDER
and (area is None or dataset.area_id is None or dataset.area_id == area.id)
]
thematic_candidates = [
dataset
for dataset in datasets
if dataset.dataset_type == "raster" and dataset.source_name == ThematicRasterAcquisitionService.PROVIDER
and (area is None or dataset.area_id is None or dataset.area_id == area.id)
]
thematic_by_product: dict[str, Dataset] = {}
for dataset in thematic_candidates:
product_key = str((dataset.source_metadata or {}).get("product_key") or "")
current = thematic_by_product.get(product_key)
if product_key and (current is None or (dataset.imported_at or datetime.min.replace(tzinfo=timezone.utc)) > (current.imported_at or datetime.min.replace(tzinfo=timezone.utc))):
thematic_by_product[product_key] = dataset
thematic_datasets = list(thematic_by_product.values())
warnings: list[str] = []
context_metrics: list[AssistantContextMetric] = []
source_dataset_ids: list[UUID] = []
@@ -323,6 +347,47 @@ class GeoAssistantService:
}
)
for dataset in sorted(thematic_datasets, key=lambda item: str((item.source_metadata or {}).get("product_key") or item.name)):
try:
result = ThematicRasterAnalysisService.analyze(
db,
project_id,
dataset.id,
ThematicRasterSelectionRequest(bbox=bbox, area_id=area.id if area is not None else None),
settings=self.settings,
)
except AppError as exc:
warnings.append(f"{dataset.name}: {exc.message}")
continue
serialized_metrics: list[dict[str, Any]] = []
for metric in result["summary"]["metrics"]:
item = AssistantContextMetric(
theme=result["theme"],
label=str(metric["metric_label"]),
value=float(metric["metric_value"]),
unit=str(metric["metric_unit"]),
source=ThematicRasterAcquisitionService.ATTRIBUTION,
dataset_id=dataset.id,
observed_at=dataset.observed_at,
is_estimate=bool(metric.get("is_estimate", True)),
)
context_metrics.append(item)
serialized_metrics.append(item.model_dump(mode="json"))
serialized_metrics[-1]["measurement_quality"] = "resolutiegebonden_bronmeting"
source_dataset_ids.append(dataset.id)
current_context.append(
{
"dataset_name": dataset.name,
"dataset_id": str(dataset.id),
"theme": result["theme"],
"source": ThematicRasterAcquisitionService.ATTRIBUTION,
"observed_at": dataset.observed_at.isoformat() if dataset.observed_at else None,
"metrics": serialized_metrics,
"unsupported_metrics": result["unsupported_metrics"],
"warning": result["limitation_message"],
}
)
for dataset in sorted(
flood_hazard_datasets,
key=lambda item: str((item.source_metadata or {}).get("product_key") or item.name),
@@ -425,6 +490,7 @@ class GeoAssistantService:
"water_volume_available": False,
"water_volume_reason": "Geen bathymetrie gekoppeld voor de permanente inhoud van waterlichamen.",
"flood_hazard_scenarios_available": bool(flood_hazard_datasets),
"thematic_policy_rasters_available": bool(thematic_datasets),
"flood_depth_area_integral_is_concurrent_volume": False,
"object_counts_are_supporting_metrics": True,
"causal_explanations_available": False,
@@ -0,0 +1,597 @@
from __future__ import annotations
import hashlib
import json
import math
import time
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from uuid import UUID
from geoalchemy2.shape import to_shape
from pyproj import Transformer
from shapely.geometry import box, mapping
from shapely.ops import transform as shapely_transform
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import Area, Dataset, Project
from app.schemas.thematic_raster import (
ThematicRasterAcquireRequest,
ThematicRasterAcquisitionResult,
ThematicRasterProductRead,
)
from app.services.dataset_service import DatasetService
@dataclass(frozen=True)
class ThematicRasterProduct:
key: str
display_name: str
theme: str
metric_kind: str
coverage_id: str
native_resolution_m: float
source_value_unit: str
observation_year: int
source_version: str
catalog_url: str
legend_min_label: str
legend_max_label: str
limitation_message: str
class ThematicRasterAcquisitionService:
"""Acquire bounded, allowlisted policy rasters from MercatorNet WCS."""
PROVIDER = "department_omgeving_thematic_raster"
SOURCE_CRS = "EPSG:31370"
WCS_VERSION = "1.0.0"
NODATA = -9999.0
WCS_TILE_SIDE_M = 10_000.0
WCS_REQUEST_INTERVAL_SECONDS = 0.5
ATTRIBUTION = "Bron: Departement Omgeving, MercatorNet"
LICENSE_NOTE = "Publieke GDI-Vlaanderen bron; bronvermelding en productspecifieke gebruiksvoorwaarden blijven van toepassing."
@staticmethod
def _products() -> dict[str, ThematicRasterProduct]:
products = (
ThematicRasterProduct(
key="space_occupation_2025",
display_name="Ruimtebeslag Vlaanderen 2025",
theme="space_occupation",
metric_kind="binary_area",
coverage_id="lu:lu_ruibes_vlaa_2025_v3",
native_resolution_m=10.0,
source_value_unit="class_0_1",
observation_year=2025,
source_version="Toestand 2025 versie 3",
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/ruimtebeslag-vlaanderen-toestand-2025",
legend_min_label="Geen ruimtebeslag",
legend_max_label="Ruimtebeslag",
limitation_message=(
"Binaire 10 m-kaart volgens de beleidsdefinitie van ruimtebeslag. Celgebaseerde oppervlakte is een "
"resolutiegebonden schatting en is niet gelijk aan uitsluitend bebouwde oppervlakte of verharding."
),
),
ThematicRasterProduct(
key="open_space_2022",
display_name="Open ruimte Vlaanderen 2022",
theme="open_space",
metric_kind="binary_area",
coverage_id="lu:lu_openruimte_vlaa_2022_v3",
native_resolution_m=10.0,
source_value_unit="class_0_1",
observation_year=2022,
source_version="Toestand 2022 versie 3",
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/open-ruimte-vlaanderen-toestand-2022",
legend_min_label="Geen open ruimte",
legend_max_label="Open ruimte",
limitation_message=(
"Binaire 10 m-beleidskaart afgeleid uit landgebruik, ruimtebeslag en kernen. Open ruimte is niet "
"synoniem met natuur, bos, publieke toegankelijkheid of planologische bestemming."
),
),
ThematicRasterProduct(
key="population_density_2019",
display_name="Inwonersdichtheid per hectare 2019",
theme="population",
metric_kind="population_density",
coverage_id="ni:ni_inw_ha_vlaa_2019",
native_resolution_m=100.0,
source_value_unit="inhabitants_per_hectare",
observation_year=2019,
source_version="Toestand 2019",
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/inwonersdichtheid-per-ha-vlaanderen-toestand-2019",
legend_min_label="0 inwoners/ha",
legend_max_label="Hogere dichtheid",
limitation_message=(
"Statistische 1 ha-rasterinschatting voor 2019, gecorrigeerd op statistische-sectorbasis. De som "
"binnen een getekende grens is een rasterraming en geen actuele registertelling."
),
),
ThematicRasterProduct(
key="node_value_2022",
display_name="Knooppuntwaarde collectief vervoer 2022",
theme="accessibility",
metric_kind="index_score",
coverage_id="lu:lu_knptw_ha_2022_v3",
native_resolution_m=100.0,
source_value_unit="source_index_score",
observation_year=2022,
source_version="Toestand 2022 versie 3",
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/knooppuntwaarde-per-ha-toestand-2022",
legend_min_label="Lagere knooppuntwaarde",
legend_max_label="Hogere knooppuntwaarde",
limitation_message=(
"Bronindex per hectare op basis van collectief-vervoerknooppunten en afstandsverval. De score is "
"geen percentage, reistijd, dienstregeling van vandaag of garantie op bereikbaarheid."
),
),
ThematicRasterProduct(
key="service_level_2022",
display_name="Totaal voorzieningenniveau 2022",
theme="services",
metric_kind="normalized_score",
coverage_id="lu:lu_totvznv_ha_2022_v3",
native_resolution_m=100.0,
source_value_unit="score_0_1",
observation_year=2022,
source_version="Toestand 2022 versie 3",
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/totaal-voorzieningenniveau-toestand-2022",
legend_min_label="Lager voorzieningenniveau",
legend_max_label="Hoger voorzieningenniveau",
limitation_message=(
"Genormaliseerde 0-1 nabijheidsscore voor basis-, regionale en metropolitane voorzieningen in "
"referentiejaar 2022. Dit is geen objecttelling, openingsurencontrole of actuele reistijd."
),
),
)
return {product.key: product for product in products}
@staticmethod
def list_products() -> list[dict[str, Any]]:
return [
ThematicRasterProductRead(
key=product.key,
display_name=product.display_name,
theme=product.theme,
metric_kind=product.metric_kind,
coverage_id=product.coverage_id,
native_resolution_m=product.native_resolution_m,
source_crs=ThematicRasterAcquisitionService.SOURCE_CRS,
source_value_unit=product.source_value_unit,
observation_year=product.observation_year,
source_version=product.source_version,
catalog_url=product.catalog_url,
attribution=ThematicRasterAcquisitionService.ATTRIBUTION,
license_note=ThematicRasterAcquisitionService.LICENSE_NOTE,
legend_min_label=product.legend_min_label,
legend_max_label=product.legend_max_label,
limitation_message=product.limitation_message,
).model_dump()
for product in ThematicRasterAcquisitionService._products().values()
]
@staticmethod
def _product(product_key: str) -> ThematicRasterProduct:
product = ThematicRasterAcquisitionService._products().get(product_key.strip().lower())
if product is None:
raise AppError(
code="THEMATIC_RASTER_PRODUCT_NOT_SUPPORTED",
message="Select a product from the governed Flemish thematic raster registry",
details={"product_key": product_key},
status_code=422,
)
return product
@staticmethod
def _prepared_request(payload: ThematicRasterAcquireRequest, settings: Settings) -> dict[str, Any]:
if not settings.thematic_raster_enabled:
raise AppError(code="THEMATIC_RASTER_NOT_CONFIGURED", message="Official thematic raster acquisition is disabled", status_code=503)
product = ThematicRasterAcquisitionService._product(payload.product_key)
values = (payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
if payload.bbox.crs.upper() != "EPSG:4326":
raise AppError(code="INVALID_BBOX_CRS", message="Thematic raster selection requires EPSG:4326", status_code=400)
if not all(math.isfinite(value) for value in values) or values[0] >= values[2] or values[1] >= values[3]:
raise AppError(code="INVALID_BBOX", message="Thematic raster selection must be a finite non-empty rectangle", status_code=400)
transformer = Transformer.from_crs("EPSG:4326", ThematicRasterAcquisitionService.SOURCE_CRS, always_xy=True)
raw_bounds = transformer.transform_bounds(*values, densify_pts=21)
resolution = product.native_resolution_m
lambert_bounds = (
math.floor(raw_bounds[0] / resolution) * resolution,
math.floor(raw_bounds[1] / resolution) * resolution,
math.ceil(raw_bounds[2] / resolution) * resolution,
math.ceil(raw_bounds[3] / resolution) * resolution,
)
width_m = lambert_bounds[2] - lambert_bounds[0]
height_m = lambert_bounds[3] - lambert_bounds[1]
if width_m < settings.thematic_raster_min_side_m or height_m < settings.thematic_raster_min_side_m:
raise AppError(
code="THEMATIC_RASTER_SELECTION_TOO_SMALL",
message=f"Select an area of at least {settings.thematic_raster_min_side_m:g} by {settings.thematic_raster_min_side_m:g} metres",
status_code=422,
)
if width_m > settings.thematic_raster_max_side_m or height_m > settings.thematic_raster_max_side_m:
raise AppError(
code="THEMATIC_RASTER_SELECTION_TOO_LARGE",
message=f"Select an area no larger than {settings.thematic_raster_max_side_m:g} by {settings.thematic_raster_max_side_m:g} metres",
details={"width_m": width_m, "height_m": height_m},
status_code=422,
)
width = max(1, round(width_m / resolution))
height = max(1, round(height_m / resolution))
if width * height > settings.thematic_raster_max_pixels:
raise AppError(
code="THEMATIC_RASTER_SELECTION_TOO_LARGE",
message="Thematic raster selection exceeds the configured cell limit",
details={"pixel_count": width * height, "max_pixels": settings.thematic_raster_max_pixels},
status_code=422,
)
request_identity = {
"provider": ThematicRasterAcquisitionService.PROVIDER,
"coverage_id": product.coverage_id,
"bbox_epsg4326": [round(float(value), 8) for value in values],
"bbox_epsg31370": [round(float(value), 3) for value in lambert_bounds],
"resolution_m": resolution,
"area_id": str(payload.area_id) if payload.area_id else None,
}
request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode()).hexdigest()
return {
**request_identity,
"product": product,
"request_hash": request_hash,
"width": width,
"height": height,
}
@staticmethod
def _wcs_request_url(settings: Settings, product: ThematicRasterProduct, bounds: tuple[float, float, float, float]) -> str:
query = {
"SERVICE": "WCS",
"VERSION": ThematicRasterAcquisitionService.WCS_VERSION,
"REQUEST": "GetCoverage",
"COVERAGE": product.coverage_id,
"CRS": ThematicRasterAcquisitionService.SOURCE_CRS,
"BBOX": ",".join(f"{value:.3f}" for value in bounds),
"RESX": f"{product.native_resolution_m:g}",
"RESY": f"{product.native_resolution_m:g}",
"FORMAT": "image/tiff",
"RESPONSE_CRS": ThematicRasterAcquisitionService.SOURCE_CRS,
}
return f"{settings.thematic_raster_wcs_url}?{urlencode(query)}"
@staticmethod
def _tile_bounds(prepared: dict[str, Any]) -> list[tuple[float, float, float, float]]:
min_x, min_y, max_x, max_y = prepared["bbox_epsg31370"]
resolution = prepared["product"].native_resolution_m
side = max(resolution, math.floor(ThematicRasterAcquisitionService.WCS_TILE_SIDE_M / resolution) * resolution)
tiles: list[tuple[float, float, float, float]] = []
y = min_y
while y < max_y:
tile_max_y = min(y + side, max_y)
x = min_x
while x < max_x:
tile_max_x = min(x + side, max_x)
tiles.append((x, y, tile_max_x, tile_max_y))
x = tile_max_x
y = tile_max_y
return tiles
@staticmethod
def _scope_geometry(db, project_id: UUID, area_id: UUID | None, bbox_epsg4326: list[float]):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
selection = box(*bbox_epsg4326)
if area_id is None:
return selection
area = db.get(Area, area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
intersection = to_shape(area.geometry).intersection(selection)
if intersection.is_empty or intersection.area <= 0:
raise AppError(code="THEMATIC_RASTER_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422)
return intersection
@staticmethod
def _fetch(request_url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, str]:
request = Request(request_url, headers={"Accept": "image/tiff,*/*", "User-Agent": "GeoIntel/0.1 bounded-thematic-raster"})
max_bytes = settings.thematic_raster_max_response_mb * 1024 * 1024
try:
with (opener or urlopen)(request, timeout=settings.thematic_raster_timeout_seconds) as response:
content_type = str(response.headers.get("Content-Type", ""))
content_length = response.headers.get("Content-Length")
if content_length and int(content_length) > max_bytes:
raise AppError(code="THEMATIC_RASTER_RESPONSE_TOO_LARGE", message="Official raster response exceeds the configured size limit", status_code=502)
content = response.read(max_bytes + 1)
except AppError:
raise
except HTTPError as exc:
preview = exc.read(300).decode("utf-8", errors="replace")
raise AppError(
code="THEMATIC_RASTER_PROVIDER_UNAVAILABLE",
message="The official MercatorNet WCS could not complete the bounded request",
details={"reason": str(exc), "provider_status_code": int(exc.code), "response_preview": preview},
status_code=502,
) from exc
except (URLError, TimeoutError, OSError) as exc:
raise AppError(
code="THEMATIC_RASTER_PROVIDER_UNAVAILABLE",
message="The official MercatorNet WCS could not complete the bounded request",
details={"reason": str(exc)},
status_code=502,
) from exc
if len(content) > max_bytes:
raise AppError(code="THEMATIC_RASTER_RESPONSE_TOO_LARGE", message="Official raster response exceeds the configured size limit", status_code=502)
if not content.startswith((b"II*\x00", b"MM\x00*")):
preview = content[:300].decode("utf-8", errors="replace")
raise AppError(
code="THEMATIC_RASTER_PROVIDER_INVALID_RESPONSE",
message="The official MercatorNet service did not return a GeoTIFF coverage",
details={"content_type": content_type, "response_preview": preview},
status_code=502,
)
return content, content_type
@staticmethod
def _mosaic(coverages: list[bytes], product: ThematicRasterProduct) -> bytes:
if len(coverages) == 1:
return coverages[0]
try:
import rasterio
from rasterio.io import MemoryFile
from rasterio.merge import merge
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio is required to assemble thematic raster tiles", status_code=503) from exc
memories = [MemoryFile(content) for content in coverages]
sources = []
try:
sources = [memory.open() for memory in memories]
for source in sources:
if source.crs is None or source.crs.to_epsg() != 31370 or source.count != 1:
raise AppError(code="THEMATIC_RASTER_TILE_MISMATCH", message="Thematic raster tiles have incompatible CRS or bands", status_code=502)
if not all(math.isclose(abs(float(value)), product.native_resolution_m, abs_tol=0.05) for value in source.res):
raise AppError(code="THEMATIC_RASTER_TILE_MISMATCH", message="Thematic raster tile resolution differs from the registry", status_code=502)
mosaic, transform = merge(sources, res=(product.native_resolution_m, product.native_resolution_m), nodata=ThematicRasterAcquisitionService.NODATA, dtype="float32")
profile = sources[0].profile.copy()
profile.pop("blockxsize", None)
profile.pop("blockysize", None)
profile.update(driver="GTiff", width=mosaic.shape[2], height=mosaic.shape[1], count=1, dtype="float32", crs=ThematicRasterAcquisitionService.SOURCE_CRS, transform=transform, nodata=ThematicRasterAcquisitionService.NODATA, compress="deflate", predictor=3)
with MemoryFile() as output_memory:
with output_memory.open(**profile) as output:
output.write(mosaic)
return output_memory.read()
except AppError:
raise
except Exception as exc:
raise AppError(code="THEMATIC_RASTER_TILE_MOSAIC_FAILED", message="Thematic raster tiles could not be assembled", details={"reason": str(exc)}, status_code=502) from exc
finally:
for source in sources:
source.close()
for memory in memories:
memory.close()
@staticmethod
def _fetch_coverage(prepared: dict[str, Any], settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, dict[str, Any]]:
product: ThematicRasterProduct = prepared["product"]
request_urls = [ThematicRasterAcquisitionService._wcs_request_url(settings, product, bounds) for bounds in ThematicRasterAcquisitionService._tile_bounds(prepared)]
coverages: list[bytes] = []
digest = hashlib.sha256()
content_types: list[str] = []
for index, request_url in enumerate(request_urls):
if index and opener is None:
time.sleep(ThematicRasterAcquisitionService.WCS_REQUEST_INTERVAL_SECONDS)
content, content_type = ThematicRasterAcquisitionService._fetch(request_url, settings, opener)
digest.update(len(content).to_bytes(8, "big"))
digest.update(content)
coverages.append(content)
content_types.append(content_type)
return ThematicRasterAcquisitionService._mosaic(coverages, product), {
"tile_count": len(request_urls),
"request_urls": request_urls,
"response_content_types": content_types,
"coverage_sha256": digest.hexdigest(),
}
@staticmethod
def _validate_values(values, product: ThematicRasterProduct) -> None:
import numpy as np
if values.size == 0:
raise AppError(code="THEMATIC_RASTER_NO_VALID_DATA", message="The official product contains no valid cells in this selection", status_code=422)
minimum = float(values.min())
maximum = float(values.max())
if minimum < 0:
raise AppError(code="THEMATIC_RASTER_INVALID_VALUES", message="Official thematic raster contains unexpected negative values", details={"minimum": minimum}, status_code=502)
if product.metric_kind == "binary_area" and not set(np.unique(values).tolist()).issubset({0.0, 1.0}):
raise AppError(code="THEMATIC_RASTER_INVALID_VALUES", message="Binary thematic raster contains classes outside 0 and 1", status_code=502)
if product.metric_kind == "normalized_score" and maximum > 1.0001:
raise AppError(code="THEMATIC_RASTER_INVALID_VALUES", message="Normalized thematic score falls outside the documented 0-1 range", details={"maximum": maximum}, status_code=502)
@staticmethod
def _normalize_raster(content: bytes, scope_geometry_4326, prepared: dict[str, Any]) -> tuple[bytes, dict[str, Any]]:
try:
import numpy as np
from rasterio.io import MemoryFile
from rasterio.mask import mask
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for thematic raster validation", status_code=503) from exc
product: ThematicRasterProduct = prepared["product"]
try:
with MemoryFile(content) as source_memory, source_memory.open() as source:
if source.crs is None or source.crs.to_epsg() != 31370:
raise AppError(code="THEMATIC_RASTER_INVALID_CRS", message="Official thematic raster must use EPSG:31370", status_code=502)
if source.count != 1:
raise AppError(code="THEMATIC_RASTER_INVALID_BANDS", message="Official thematic raster must contain one band", status_code=502)
if not all(math.isclose(abs(float(value)), product.native_resolution_m, abs_tol=0.05) for value in source.res):
raise AppError(code="THEMATIC_RASTER_INVALID_RESOLUTION", message="Official thematic raster resolution differs from the registry", status_code=502)
transformer = Transformer.from_crs("EPSG:4326", ThematicRasterAcquisitionService.SOURCE_CRS, always_xy=True)
scope_metric = shapely_transform(transformer.transform, scope_geometry_4326)
clipped, transform = mask(source, [mapping(scope_metric)], crop=True, filled=False, indexes=[1])
band = np.ma.asarray(clipped[0], dtype="float32")
raw = np.asarray(band.filled(np.nan), dtype="float32")
invalid = np.ma.getmaskarray(band) | ~np.isfinite(raw)
if source.nodata is not None:
invalid |= np.isclose(raw, float(source.nodata))
normalized = np.ma.array(raw, mask=invalid)
values = normalized.compressed().astype("float64")
ThematicRasterAcquisitionService._validate_values(values, product)
profile = source.profile.copy()
profile.pop("blockxsize", None)
profile.pop("blockysize", None)
profile.update(driver="GTiff", width=normalized.shape[1], height=normalized.shape[0], count=1, dtype="float32", crs=ThematicRasterAcquisitionService.SOURCE_CRS, transform=transform, nodata=ThematicRasterAcquisitionService.NODATA, compress="deflate", predictor=3)
with MemoryFile() as output_memory:
with output_memory.open(**profile) as output:
output.write(normalized.filled(ThematicRasterAcquisitionService.NODATA), 1)
normalized_content = output_memory.read()
return normalized_content, {
"width": int(normalized.shape[1]),
"height": int(normalized.shape[0]),
"valid_pixel_count": int(values.size),
"nodata_value": ThematicRasterAcquisitionService.NODATA,
"resolution_m": product.native_resolution_m,
"minimum_value": float(values.min()),
"maximum_value": float(values.max()),
"p02_value": float(np.percentile(values, 2)),
"p98_value": float(np.percentile(values, 98)),
}
except AppError:
raise
except Exception as exc:
raise AppError(code="THEMATIC_RASTER_INVALID", message="The official thematic raster could not be validated", details={"reason": str(exc)}, status_code=502) from exc
@staticmethod
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
candidate = (
db.query(Dataset)
.filter(Dataset.project_id == project_id, Dataset.name == filename, Dataset.source_name == ThematicRasterAcquisitionService.PROVIDER, Dataset.status == "ready")
.order_by(Dataset.imported_at.desc())
.first()
)
return candidate if candidate and candidate.storage_path and Path(candidate.storage_path).is_file() else None
@staticmethod
def acquire(db, project_id: UUID, payload: ThematicRasterAcquireRequest, *, settings: Settings | None = None, opener: Callable[..., Any] | None = None) -> dict[str, Any]:
resolved_settings = settings or get_settings()
prepared = ThematicRasterAcquisitionService._prepared_request(payload, resolved_settings)
product: ThematicRasterProduct = prepared["product"]
scope_geometry = ThematicRasterAcquisitionService._scope_geometry(db, project_id, payload.area_id, prepared["bbox_epsg4326"])
filename = f"thematic_{product.key}_{prepared['request_hash'][:12]}.tif"
if not payload.force_refresh:
cached = ThematicRasterAcquisitionService._cached_dataset(db, project_id, filename)
if cached is not None:
metadata = cached.source_metadata or {}
raster_metadata = cached.metadata_json or {}
return ThematicRasterAcquisitionResult(
output_dataset_id=cached.id,
reused=True,
provider=ThematicRasterAcquisitionService.PROVIDER,
product_key=product.key,
display_name=product.display_name,
theme=product.theme,
metric_kind=product.metric_kind,
coverage_id=product.coverage_id,
resolution_m=product.native_resolution_m,
width=int(raster_metadata.get("width", prepared["width"])),
height=int(raster_metadata.get("height", prepared["height"])),
valid_pixel_count=int(metadata.get("valid_pixel_count", 0)),
bbox_epsg4326=prepared["bbox_epsg4326"],
bbox_epsg31370=prepared["bbox_epsg31370"],
observation_year=product.observation_year,
source_value_unit=product.source_value_unit,
attribution=ThematicRasterAcquisitionService.ATTRIBUTION,
limitation_message=product.limitation_message,
).model_dump(mode="json")
coverage, transfer = ThematicRasterAcquisitionService._fetch_coverage(prepared, resolved_settings, opener)
normalized, validation = ThematicRasterAcquisitionService._normalize_raster(coverage, scope_geometry, prepared)
acquired_at = datetime.now(UTC)
observed_at = datetime(product.observation_year, 12, 31, 23, 59, 59, tzinfo=UTC)
scope_key = str(payload.area_id) if payload.area_id else prepared["request_hash"][:24]
dataset = DatasetService.import_raster_bytes(
db,
project_id=project_id,
area_id=payload.area_id,
filename=filename,
content=normalized,
source=f"Departement Omgeving MercatorNet WCS {product.coverage_id}",
source_name=ThematicRasterAcquisitionService.PROVIDER,
temporal_series_key=f"department-omgeving:thematic-raster:{product.key}:{scope_key}",
observed_at=observed_at,
valid_from=datetime(product.observation_year, 1, 1, tzinfo=UTC),
valid_to=observed_at,
temporal_granularity="year",
source_version=product.source_version,
source_metadata={
"provider": ThematicRasterAcquisitionService.PROVIDER,
"service": "WCS",
"service_version": ThematicRasterAcquisitionService.WCS_VERSION,
"product_key": product.key,
"product_display_name": product.display_name,
"theme": product.theme,
"metric_kind": product.metric_kind,
"coverage_id": product.coverage_id,
"native_resolution_m": product.native_resolution_m,
"analysis_resolution_m": product.native_resolution_m,
"source_crs": ThematicRasterAcquisitionService.SOURCE_CRS,
"source_value_unit": product.source_value_unit,
"observation_year": product.observation_year,
"observation_date_precision": "year",
"valid_pixel_count": validation["valid_pixel_count"],
"minimum_value": validation["minimum_value"],
"maximum_value": validation["maximum_value"],
"render_min_value": validation["p02_value"],
"render_max_value": validation["p98_value"],
"bbox_epsg4326": prepared["bbox_epsg4326"],
"bbox_epsg31370": prepared["bbox_epsg31370"],
"catalog_url": product.catalog_url,
"attribution": ThematicRasterAcquisitionService.ATTRIBUTION,
"license_note": ThematicRasterAcquisitionService.LICENSE_NOTE,
"legend_min_label": product.legend_min_label,
"legend_max_label": product.legend_max_label,
"coverage_scope": "municipality" if payload.area_id else "bounded_selection",
},
provenance_metadata={
"acquisition": "explicit_bounded_tiled_wcs_coverage",
"acquired_at": acquired_at.isoformat(),
"request_hash": prepared["request_hash"],
"tile_count": transfer["tile_count"],
"tile_request_urls": transfer["request_urls"],
"response_content_types": transfer["response_content_types"],
"coverage_sha256": transfer["coverage_sha256"],
"normalized_sha256": hashlib.sha256(normalized).hexdigest(),
"bbox_epsg4326": prepared["bbox_epsg4326"],
"bbox_epsg31370": prepared["bbox_epsg31370"],
"clipped_to_area_id": str(payload.area_id) if payload.area_id else None,
"validation": validation,
"limitation_message": product.limitation_message,
},
)
return ThematicRasterAcquisitionResult(
output_dataset_id=dataset.id,
reused=False,
provider=ThematicRasterAcquisitionService.PROVIDER,
product_key=product.key,
display_name=product.display_name,
theme=product.theme,
metric_kind=product.metric_kind,
coverage_id=product.coverage_id,
resolution_m=product.native_resolution_m,
width=validation["width"],
height=validation["height"],
valid_pixel_count=validation["valid_pixel_count"],
bbox_epsg4326=prepared["bbox_epsg4326"],
bbox_epsg31370=prepared["bbox_epsg31370"],
observation_year=product.observation_year,
source_value_unit=product.source_value_unit,
attribution=ThematicRasterAcquisitionService.ATTRIBUTION,
limitation_message=product.limitation_message,
).model_dump(mode="json")
@@ -0,0 +1,260 @@
from __future__ import annotations
import io
import math
from datetime import UTC, datetime
from pathlib import Path
from uuid import UUID
from geoalchemy2.shape import to_shape
from pyproj import Transformer
from shapely.geometry import box, mapping
from shapely.ops import transform as shapely_transform
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import Area, Dataset
from app.schemas.thematic_raster import (
ThematicRasterMetric,
ThematicRasterSelectionRequest,
ThematicRasterSelectionResponse,
ThematicRasterSelectionSummary,
)
from app.services.thematic_raster_acquisition_service import (
ThematicRasterAcquisitionService,
ThematicRasterProduct,
)
class ThematicRasterAnalysisService:
@staticmethod
def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> Dataset:
dataset = db.get(Dataset, dataset_id)
if not dataset or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
if dataset.dataset_type != "raster" or dataset.source_name != ThematicRasterAcquisitionService.PROVIDER:
raise AppError(
code="INVALID_THEMATIC_RASTER_DATASET",
message="Thematic analysis requires a governed Departement Omgeving raster dataset",
status_code=400,
)
if dataset.status != "ready" or not dataset.storage_path or not Path(dataset.storage_path).is_file():
raise AppError(code="DATASET_FILE_MISSING", message="Persisted thematic raster file is unavailable", status_code=404)
return dataset
@staticmethod
def _product(dataset: Dataset) -> ThematicRasterProduct:
source_metadata = dataset.source_metadata or {}
product = ThematicRasterAcquisitionService._products().get(str(source_metadata.get("product_key") or ""))
if product is None or source_metadata.get("coverage_id") != product.coverage_id:
raise AppError(code="INVALID_THEMATIC_RASTER_METADATA", message="Thematic raster provenance is incomplete", status_code=409)
return product
@staticmethod
def _selection_geometry(db, project_id: UUID, payload: ThematicRasterSelectionRequest):
selection = box(payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
if payload.area_id is None:
return selection
area = db.get(Area, payload.area_id)
if not area:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
if area.project_id != project_id:
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
selection = selection.intersection(to_shape(area.geometry))
if selection.is_empty or selection.area <= 0:
raise AppError(code="THEMATIC_RASTER_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422)
return selection
@staticmethod
def _unsupported_metrics(product: ThematicRasterProduct) -> list[str]:
if product.metric_kind == "binary_area":
return ["object_count", "parcel_area", "current_land_use"]
if product.metric_kind == "population_density":
return ["current_population", "household_count", "address_level_population"]
if product.metric_kind == "index_score":
return ["travel_time_minutes", "current_timetable", "stop_count"]
return ["facility_count", "opening_hours", "current_service_availability"]
@staticmethod
def analyze(
db,
project_id: UUID,
dataset_id: UUID,
payload: ThematicRasterSelectionRequest,
*,
settings: Settings | None = None,
) -> dict:
resolved_settings = settings or get_settings()
dataset = ThematicRasterAnalysisService._load_dataset(db, project_id, dataset_id)
product = ThematicRasterAnalysisService._product(dataset)
selection_4326 = ThematicRasterAnalysisService._selection_geometry(db, project_id, payload)
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.mask import mask
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for thematic raster analysis", status_code=503) from exc
try:
with rasterio.open(dataset.storage_path) as source:
if source.crs is None or source.crs.to_epsg() != 31370:
raise AppError(code="INVALID_DATASET_CRS", message="Thematic raster CRS must be EPSG:31370", status_code=409)
transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True)
selection_metric = shapely_transform(transformer.transform, selection_4326)
analysis_geometry = selection_metric.intersection(box(*source.bounds))
if analysis_geometry.is_empty or analysis_geometry.area <= 0:
raise AppError(code="THEMATIC_RASTER_SELECTION_OUTSIDE_DATASET", message="Selection does not overlap the persisted thematic raster", status_code=422)
min_x, min_y, max_x, max_y = analysis_geometry.bounds
expected_cells = math.ceil((max_x - min_x) / abs(source.res[0])) * math.ceil((max_y - min_y) / abs(source.res[1]))
if expected_cells > resolved_settings.thematic_raster_max_pixels:
raise AppError(
code="THEMATIC_RASTER_SELECTION_TOO_LARGE",
message="Thematic raster analysis exceeds the configured cell limit",
details={"pixel_count": expected_cells, "max_pixels": resolved_settings.thematic_raster_max_pixels},
status_code=422,
)
clipped, clipped_transform = mask(source, [mapping(analysis_geometry)], crop=True, filled=False, indexes=[1])
band = np.ma.asarray(clipped[0], dtype="float64")
raw = band.filled(np.nan)
selected = geometry_mask([mapping(analysis_geometry)], out_shape=band.shape, transform=clipped_transform, invert=True)
valid = selected & ~np.ma.getmaskarray(band) & np.isfinite(raw)
if source.nodata is not None:
valid &= ~np.isclose(raw, float(source.nodata))
values = raw[valid]
ThematicRasterAcquisitionService._validate_values(values, product)
selected_cell_count = int(selected.sum())
valid_cell_count = int(values.size)
resolution_x = abs(float(source.res[0]))
resolution_y = abs(float(source.res[1]))
cell_area_m2 = resolution_x * resolution_y
except AppError:
raise
except Exception as exc:
raise AppError(
code="THEMATIC_RASTER_ANALYSIS_FAILED",
message="The persisted thematic raster could not be analysed",
details={"reason": str(exc)},
status_code=500,
) from exc
def metric(key: str, label: str, value: float, unit: str, method: str, *, estimate: bool = True) -> ThematicRasterMetric:
return ThematicRasterMetric(
metric_key=key,
metric_label=label,
metric_value=round(float(value), 4),
metric_unit=unit,
aggregation_method=method,
is_estimate=estimate,
)
if product.metric_kind == "binary_area":
positive_count = int(np.count_nonzero(values >= 0.5))
positive_area_ha = positive_count * cell_area_m2 / 10_000.0
positive_share = positive_count / max(1, valid_cell_count) * 100.0
label = "Ruimtebeslag" if product.theme == "space_occupation" else "Open ruimte"
metrics = [
metric(f"{product.theme}_area_ha", f"{label} in selectie", positive_area_ha, "ha", "positive_source_cells_times_cell_area"),
metric(f"{product.theme}_share_pct", f"Aandeel {label.lower()}", positive_share, "%", "positive_source_cells_divided_by_valid_selected_cells"),
metric("valid_raster_area_ha", "Rasteroppervlakte met bronwaarde", valid_cell_count * cell_area_m2 / 10_000.0, "ha", "valid_selected_cells_times_cell_area"),
]
elif product.metric_kind == "population_density":
estimated_population = float(values.sum() * (cell_area_m2 / 10_000.0))
metrics = [
metric("estimated_inhabitants", "Geraamd aantal inwoners (2019)", estimated_population, "inwoners", "sum_density_times_selected_cell_area_hectares"),
metric("population_density_mean_per_ha", "Gemiddelde inwonersdichtheid", values.mean(), "inwoners/ha", "mean_valid_one_hectare_source_cells"),
metric("population_density_p90_per_ha", "90e percentiel inwonersdichtheid", np.percentile(values, 90), "inwoners/ha", "percentile_90_valid_source_cells"),
]
else:
unit = "score" if product.metric_kind == "index_score" else "score (0-1)"
label = "Knooppuntwaarde" if product.metric_kind == "index_score" else "Voorzieningenniveau"
metrics = [
metric(f"{product.theme}_mean", f"Gemiddelde {label.lower()}", values.mean(), unit, "mean_valid_source_cells"),
metric(f"{product.theme}_p10", f"10e percentiel {label.lower()}", np.percentile(values, 10), unit, "percentile_10_valid_source_cells"),
metric(f"{product.theme}_median", f"Mediaan {label.lower()}", np.percentile(values, 50), unit, "median_valid_source_cells"),
metric(f"{product.theme}_p90", f"90e percentiel {label.lower()}", np.percentile(values, 90), unit, "percentile_90_valid_source_cells"),
]
primary = metrics[0]
response = ThematicRasterSelectionResponse(
dataset_id=dataset.id,
product_key=product.key,
theme=product.theme,
metric_kind=product.metric_kind,
selection_bbox=payload.bbox,
selection_area_id=payload.area_id,
selected_cell_count=selected_cell_count,
valid_cell_count=valid_cell_count,
coverage_ratio=round(valid_cell_count / max(1, selected_cell_count), 6),
resolution_m=round(max(resolution_x, resolution_y), 4),
observation_year=product.observation_year,
summary=ThematicRasterSelectionSummary(
metric_label=primary.metric_label,
metric_value=primary.metric_value,
metric_unit=primary.metric_unit,
aggregation_method=primary.aggregation_method,
primary_metric_key=primary.metric_key,
metrics=metrics,
),
unsupported_metrics=ThematicRasterAnalysisService._unsupported_metrics(product),
limitation_message=product.limitation_message,
generated_at=datetime.now(UTC).isoformat(),
)
return response.model_dump(mode="json")
@staticmethod
def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes:
dataset = ThematicRasterAnalysisService._load_dataset(db, project_id, dataset_id)
product = ThematicRasterAnalysisService._product(dataset)
try:
import numpy as np
import rasterio
from PIL import Image
from rasterio.enums import Resampling
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio, numpy and Pillow are required for thematic raster rendering", status_code=503) from exc
palettes = {
"space_occupation": np.asarray([[251, 231, 211], [190, 62, 51]], dtype="float64"),
"open_space": np.asarray([[221, 238, 219], [38, 122, 70]], dtype="float64"),
"population": np.asarray([[238, 231, 246], [103, 58, 151]], dtype="float64"),
"accessibility": np.asarray([[233, 241, 244], [15, 118, 110]], dtype="float64"),
"services": np.asarray([[255, 244, 191], [182, 109, 22]], dtype="float64"),
}
try:
with rasterio.open(dataset.storage_path) as source:
scale = min(1.0, max_dimension / max(source.width, source.height))
width = max(1, round(source.width * scale))
height = max(1, round(source.height * scale))
resampling = Resampling.nearest if product.metric_kind == "binary_area" else Resampling.bilinear
data = source.read(1, out_shape=(height, width), masked=True, resampling=resampling)
values = np.asarray(data.filled(np.nan), dtype="float64")
valid = np.isfinite(values) & ~np.ma.getmaskarray(data)
if source.nodata is not None:
valid &= ~np.isclose(values, float(source.nodata))
if product.metric_kind == "binary_area":
valid &= values >= 0.5
normalized = np.where(valid, 1.0, 0.0)
else:
source_metadata = dataset.source_metadata or {}
lower = float(source_metadata.get("render_min_value", np.nanpercentile(values[valid], 2) if valid.any() else 0.0))
upper = float(source_metadata.get("render_max_value", np.nanpercentile(values[valid], 98) if valid.any() else 1.0))
if upper <= lower:
upper = lower + 1.0
normalized = np.clip((values - lower) / (upper - lower), 0.0, 1.0)
colors = palettes[product.theme]
rgba = np.zeros((height, width, 4), dtype="uint8")
for channel in range(3):
rgba[:, :, channel] = (colors[0, channel] + normalized * (colors[1, channel] - colors[0, channel])).astype("uint8")
rgba[:, :, 3] = np.where(valid, 205, 0).astype("uint8")
output = io.BytesIO()
Image.fromarray(rgba).save(output, format="PNG", optimize=True)
return output.getvalue()
except AppError:
raise
except Exception as exc:
raise AppError(
code="THEMATIC_RASTER_PREVIEW_FAILED",
message="The persisted thematic raster could not be rendered",
details={"reason": str(exc)},
status_code=500,
) from exc
@@ -29,6 +29,7 @@ FULL_AREA_CLIPPED_OPERATOR_TOOLS = {
"provision_regional_bwk_natura2000.py",
"provision_agricultural_parcel_history.py",
"provision_buildings_addresses_register.py",
"provision_mol_soil_map.py",
}
SEMANTIC_METRICS_DISABLED_OPERATOR_TOOLS = {
@@ -101,6 +102,16 @@ SEMANTIC_SELECTION_METRICS: dict[str, tuple[dict[str, Any], ...]] = {
),
"nature_value": (),
"agriculture": (),
"soil": (
{
"metric_key": "soil_mapped_area",
"method": "intersection_area",
"label": "Bodemkaartoppervlakte",
"unit": "ha",
"geometry_dimension": 2,
"warning": "Historische bodemkartering op schaal 1:20.000; actuele lokale bodem- en drainagetoestand kan afwijken.",
},
),
}
SEMANTIC_COUNT_LABELS = {
@@ -112,6 +123,7 @@ SEMANTIC_COUNT_LABELS = {
"parcels": "Percelen",
"nature_value": "BWK-kaartvlakken",
"agriculture": "Landbouwgebruikspercelen",
"soil": "Bodemkaartvlakken",
}
# Sprint 205 initially normalized two official comma-separated ALZ group labels
@@ -158,6 +170,8 @@ class VectorFeatureService:
"landbouw": "agriculture",
"landbouwgebruik": "agriculture",
"building_registry": "buildings",
"soil_map": "soil",
"bodem": "soil",
}
for candidate in candidates:
if not isinstance(candidate, str) or not candidate.strip():
@@ -0,0 +1,339 @@
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
import numpy as np
import pytest
import rasterio
from fastapi.testclient import TestClient
from pyproj import Transformer
from rasterio.io import MemoryFile
from rasterio.transform import from_origin
from app.core.config import Settings
from app.core.errors import AppError
from app.db.session import get_db
from app.main import app
from app.models import Dataset, Job, Project
from app.schemas.thematic_raster import ThematicRasterAcquireRequest, ThematicRasterSelectionRequest
from app.schemas.assistant import AssistantQueryRequest
from app.services.geo_assistant_service import GeoAssistantService
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
from app.services.dataset_service import DatasetService
ROOT = Path(__file__).resolve().parents[2]
class FakeQuery:
def __init__(self, result=None):
self.result = result
def filter(self, *_args):
return self
def order_by(self, *_args):
return self
def first(self):
return self.result
def all(self):
return self.result if isinstance(self.result, list) else []
class FakeSession:
def __init__(self, rows=None, query_result=None):
self.rows = rows or {}
self.query_result = query_result
self.added = []
def get(self, model, row_id):
row = self.rows.get((model, row_id))
if row is not None:
return row
return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
def add(self, row):
self.added.append(row)
def commit(self):
return None
def rollback(self):
return None
def refresh(self, row):
return row
def query(self, _model):
return FakeQuery(self.query_result)
class FakeResponse:
def __init__(self, content: bytes):
self.content = content
self.headers = {"Content-Type": "image/tiff", "Content-Length": str(len(content))}
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def read(self, limit: int):
return self.content[:limit]
def payload(product_key: str = "space_occupation_2025", *, side_m: float = 1000.0) -> ThematicRasterAcquireRequest:
transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
min_x, min_y = transformer.transform(200_000, 210_000)
max_x, max_y = transformer.transform(200_000 + side_m, 210_000 + side_m)
return ThematicRasterAcquireRequest(
bbox={"min_x": min_x, "min_y": min_y, "max_x": max_x, "max_y": max_y, "crs": "EPSG:4326"},
product_key=product_key,
force_refresh=True,
)
def raster_bytes(values: np.ndarray, resolution: float, *, nodata: float = -9999.0) -> bytes:
with MemoryFile() as memory:
with memory.open(
driver="GTiff",
width=values.shape[1],
height=values.shape[0],
count=1,
dtype=str(values.dtype),
crs="EPSG:31370",
transform=from_origin(200_000, 210_000 + values.shape[0] * resolution, resolution, resolution),
nodata=nodata,
) as output:
output.write(values, 1)
return memory.read()
def test_registry_contains_five_governed_non_water_policy_products() -> None:
products = ThematicRasterAcquisitionService.list_products()
assert [item["key"] for item in products] == [
"space_occupation_2025",
"open_space_2022",
"population_density_2019",
"node_value_2022",
"service_level_2022",
]
assert {item["theme"] for item in products} == {"space_occupation", "open_space", "population", "accessibility", "services"}
assert {item["native_resolution_m"] for item in products} == {10.0, 100.0}
assert all(item["coverage_id"].startswith(("lu:", "ni:")) for item in products)
assert all(item["source_crs"] == "EPSG:31370" for item in products)
assert all(item["attribution"] and item["license_note"] and item["limitation_message"] for item in products)
def test_request_is_bounded_allowlisted_and_uses_native_wcs_resolution() -> None:
settings = Settings(_env_file=None)
prepared = ThematicRasterAcquisitionService._prepared_request(payload("population_density_2019"), settings)
url = ThematicRasterAcquisitionService._wcs_request_url(settings, prepared["product"], tuple(prepared["bbox_epsg31370"]))
assert "VERSION=1.0.0" in url
assert "COVERAGE=ni%3Ani_inw_ha_vlaa_2019" in url
assert "RESX=100" in url and "RESY=100" in url
assert prepared["width"] * prepared["height"] <= settings.thematic_raster_max_pixels
with pytest.raises(AppError) as exc_info:
ThematicRasterAcquisitionService._prepared_request(payload("arbitrary_remote_layer"), settings)
assert exc_info.value.code == "THEMATIC_RASTER_PRODUCT_NOT_SUPPORTED"
def test_binary_and_normalized_products_fail_closed_on_invalid_values() -> None:
binary = ThematicRasterAcquisitionService._product("space_occupation_2025")
score = ThematicRasterAcquisitionService._product("service_level_2022")
with pytest.raises(AppError, match="Binary"):
ThematicRasterAcquisitionService._validate_values(np.asarray([0.0, 2.0]), binary)
with pytest.raises(AppError, match="0-1"):
ThematicRasterAcquisitionService._validate_values(np.asarray([0.2, 1.2]), score)
def test_acquisition_clips_validates_and_delegates_persistence(monkeypatch) -> None:
project_id, output_dataset_id = uuid4(), uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
content = raster_bytes(np.ones((100, 100), dtype="float32"), 10.0)
captured: dict = {}
def fake_import(_db, **kwargs):
captured.update(kwargs)
return SimpleNamespace(id=output_dataset_id)
monkeypatch.setattr(DatasetService, "import_raster_bytes", fake_import)
result = ThematicRasterAcquisitionService.acquire(
db,
project_id,
payload(side_m=1000.0),
settings=Settings(_env_file=None),
opener=lambda *_args, **_kwargs: FakeResponse(content),
)
assert result["output_dataset_id"] == str(output_dataset_id)
assert captured["source_name"] == ThematicRasterAcquisitionService.PROVIDER
assert captured["source_metadata"]["product_key"] == "space_occupation_2025"
assert captured["source_metadata"]["metric_kind"] == "binary_area"
assert captured["source_metadata"]["valid_pixel_count"] > 9_800
assert captured["provenance_metadata"]["acquisition"] == "explicit_bounded_tiled_wcs_coverage"
assert len(captured["provenance_metadata"]["normalized_sha256"]) == 64
def test_binary_area_analysis_returns_hectares_and_share(tmp_path) -> None:
project_id, dataset_id = uuid4(), uuid4()
values = np.zeros((10, 10), dtype="float32")
values[:, :5] = 1.0
path = tmp_path / "space.tif"
path.write_bytes(raster_bytes(values, 10.0))
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="space.tif",
dataset_type="raster",
source="official",
source_name=ThematicRasterAcquisitionService.PROVIDER,
source_metadata={"product_key": "space_occupation_2025", "coverage_id": "lu:lu_ruibes_vlaa_2025_v3"},
status="ready",
storage_path=str(path),
)
db = FakeSession({(Dataset, dataset_id): dataset})
result = ThematicRasterAnalysisService.analyze(
db,
project_id,
dataset_id,
ThematicRasterSelectionRequest(bbox=payload(side_m=100.0).bbox),
)
metrics = {item["metric_key"]: item for item in result["summary"]["metrics"]}
assert result["valid_cell_count"] == 100
assert metrics["space_occupation_area_ha"]["metric_value"] == pytest.approx(0.5)
assert metrics["space_occupation_share_pct"]["metric_value"] == pytest.approx(50.0)
assert "object_count" in result["unsupported_metrics"]
def test_population_analysis_sums_one_hectare_density_cells_without_claiming_current_counts(tmp_path) -> None:
project_id, dataset_id = uuid4(), uuid4()
values = np.asarray([[10.0, 20.0], [30.0, 40.0]], dtype="float32")
path = tmp_path / "population.tif"
path.write_bytes(raster_bytes(values, 100.0))
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="population.tif",
dataset_type="raster",
source="official",
source_name=ThematicRasterAcquisitionService.PROVIDER,
source_metadata={"product_key": "population_density_2019", "coverage_id": "ni:ni_inw_ha_vlaa_2019"},
status="ready",
storage_path=str(path),
)
db = FakeSession({(Dataset, dataset_id): dataset})
result = ThematicRasterAnalysisService.analyze(
db,
project_id,
dataset_id,
ThematicRasterSelectionRequest(bbox=payload("population_density_2019", side_m=200.0).bbox),
)
metrics = {item["metric_key"]: item for item in result["summary"]["metrics"]}
assert metrics["estimated_inhabitants"]["metric_value"] == pytest.approx(100.0)
assert metrics["population_density_mean_per_ha"]["metric_value"] == pytest.approx(25.0)
assert metrics["estimated_inhabitants"]["is_estimate"] is True
assert "current_population" in result["unsupported_metrics"]
def test_assistant_context_receives_persisted_thematic_metrics(tmp_path) -> None:
project_id, dataset_id = uuid4(), uuid4()
values = np.asarray([[10.0, 20.0], [30.0, 40.0]], dtype="float32")
path = tmp_path / "assistant-population.tif"
path.write_bytes(raster_bytes(values, 100.0))
project = Project(id=project_id, name="Mol", region="Mol")
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="population.tif",
dataset_type="raster",
source="official",
source_name=ThematicRasterAcquisitionService.PROVIDER,
source_metadata={"product_key": "population_density_2019", "coverage_id": "ni:ni_inw_ha_vlaa_2019"},
status="ready",
storage_path=str(path),
)
db = FakeSession({(Project, project_id): project, (Dataset, dataset_id): dataset}, query_result=[dataset])
context, metrics, _series, dataset_ids, _warnings, _scope = GeoAssistantService(Settings(_env_file=None))._build_context(
db,
project_id=project_id,
payload=AssistantQueryRequest(question="Hoeveel inwoners?", bbox=payload("population_density_2019", side_m=200.0).bbox),
)
assert any(metric.theme == "population" and metric.label.startswith("Geraamd aantal") for metric in metrics)
assert dataset_id in dataset_ids
assert context["rules"]["thematic_policy_rasters_available"] is True
def test_index_renderer_returns_browser_png(tmp_path) -> None:
project_id, dataset_id = uuid4(), uuid4()
values = np.linspace(0.1, 4.0, 100, dtype="float32").reshape((10, 10))
path = tmp_path / "node.tif"
path.write_bytes(raster_bytes(values, 100.0))
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="node.tif",
dataset_type="raster",
source="official",
source_name=ThematicRasterAcquisitionService.PROVIDER,
source_metadata={
"product_key": "node_value_2022",
"coverage_id": "lu:lu_knptw_ha_2022_v3",
"render_min_value": 0.1,
"render_max_value": 4.0,
},
status="ready",
storage_path=str(path),
)
db = FakeSession({(Dataset, dataset_id): dataset})
assert ThematicRasterAnalysisService.render_png(db, project_id, dataset_id).startswith(b"\x89PNG\r\n\x1a\n")
def test_api_uses_canonical_envelopes(monkeypatch) -> None:
project_id, dataset_id = uuid4(), uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
monkeypatch.setattr(
ThematicRasterAcquisitionService,
"acquire",
lambda *_args, **_kwargs: {"output_dataset_id": str(dataset_id), "provider": ThematicRasterAcquisitionService.PROVIDER},
)
monkeypatch.setattr(
ThematicRasterAnalysisService,
"analyze",
lambda *_args, **_kwargs: {"dataset_id": str(dataset_id), "theme": "population", "summary": {"metric_value": 10.0}},
)
app.dependency_overrides[get_db] = lambda: db
try:
client = TestClient(app)
products = client.get(f"/api/v1/projects/{project_id}/datasets/thematic-raster/products")
acquisition = client.post(
f"/api/v1/projects/{project_id}/datasets/thematic-raster/acquire",
json=payload().model_dump(mode="json"),
)
selection = client.post(
f"/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/thematic/select",
json={"bbox": payload().bbox.model_dump()},
)
finally:
app.dependency_overrides.clear()
assert products.status_code == 200 and set(products.json()) == {"data"}
assert products.json()["data"]["total"] == 5
assert acquisition.status_code == 200 and set(acquisition.json()) == {"data"}
assert acquisition.json()["data"]["job_type"] == "raster.thematic.acquire"
assert selection.status_code == 200 and selection.json()["data"]["theme"] == "population"
assert any(isinstance(item, Job) for item in db.added)
@@ -0,0 +1,183 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
from uuid import uuid4
import pytest
from shapely.geometry import box, mapping, shape
from shapely.ops import transform as transform_geometry
from app.models import Dataset
from app.services.vector_feature_service import VectorFeatureService
ROOT = Path(__file__).resolve().parents[2]
def load_operator():
path = ROOT / "scripts" / "provision_mol_soil_map.py"
spec = importlib.util.spec_from_file_location("dov_soil_map_operator", path)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
class FakeResponse:
def __init__(self, payload: dict, url: str):
self._payload = payload
self.url = url
self.content = b'{"type":"FeatureCollection"}'
def raise_for_status(self) -> None:
return None
def json(self) -> dict:
return self._payload
class FakeSession:
def __init__(self, pages: list[dict]):
self.pages = pages
self.calls: list[dict] = []
def get(self, _url: str, *, params: dict, timeout: int):
self.calls.append({"params": dict(params), "timeout": timeout})
return FakeResponse(self.pages[len(self.calls) - 1], f"https://example.test/page/{len(self.calls)}")
def soil_feature(module, feature_id: str = "bodemtypes.1") -> dict:
return {
"type": "Feature",
"id": feature_id,
"geometry": mapping(box(5.0, 51.0, 5.02, 51.02)),
"properties": {
"gid": 1,
"id_kaartvlak": 10,
"Bodemtype": "Zeg",
"Unibodemtype": "Zeg",
"Bodemserie": "Zeg",
"Beknopte_omschrijving_bodemserie": "Natte zandbodem",
"Gegeneraliseerde_legende": "Nat zand",
"Textuurklasse_code": "Z",
"Textuurklasse": "zand",
"Drainageklasse_code": "e",
"Drainageklasse": "nat",
"Profielontwikkelingsgroep_code": "g",
"Profielontwikkelingsgroep": "humus B horizont",
"Eenduidige_legende_titel": "bodemserie Zeg",
},
}
def test_wfs_pagination_is_bounded_complete_and_deterministic() -> None:
module = load_operator()
feature = soil_feature(module)
pages = [
{
"type": "FeatureCollection",
"numberMatched": 3,
"numberReturned": 2,
"features": [feature, {**feature, "id": "bodemtypes.2"}],
},
{
"type": "FeatureCollection",
"numberMatched": 3,
"numberReturned": 1,
"features": [{**feature, "id": "bodemtypes.3"}],
},
]
session = FakeSession(pages)
result = list(
module.iter_wfs_pages(
session,
(196000.0, 205000.0, 211000.0, 224000.0),
page_limit=2,
timeout=30,
)
)
assert len(result) == 2
assert [call["params"]["startIndex"] for call in session.calls] == ["0", "2"]
assert all(call["params"]["typeNames"] == "bodemkaart:bodemtypes" for call in session.calls)
assert all(call["params"]["bbox"].endswith("EPSG:31370") for call in session.calls)
assert all(call["params"]["sortBy"] == "gid" for call in session.calls)
def test_soil_feature_is_exactly_clipped_and_keeps_governed_properties() -> None:
module = load_operator()
boundary_wgs84 = box(5.005, 51.005, 5.015, 51.015)
boundary_lambert72 = transform_geometry(module.TO_LAMBERT72.transform, boundary_wgs84)
normalized, was_clipped = module.normalize_feature(soil_feature(module), boundary_lambert72)
assert normalized is not None and was_clipped is True
persisted_geometry = shape(normalized["geometry"])
assert persisted_geometry.within(boundary_wgs84.buffer(1e-7))
properties = normalized["properties"]
assert properties["source_name"] == "dov_soil_map"
assert properties["soil_texture_class"] == "zand"
assert properties["soil_drainage_class"] == "nat"
assert properties["survey_period"] == "1949-1971"
assert properties["clipped_area_ha"] > 0
assert "may differ today" in properties["historical_drainage_limitation"]
def test_soil_map_uses_existing_semantic_selection_architecture() -> None:
dataset = Dataset(
id=uuid4(),
project_id=uuid4(),
name="dov_soil_map_mol.geojson",
dataset_type="vector",
source="operator_official_import",
source_name="dov_soil_map",
reference_layer_name="soil",
source_metadata={
"theme": "soil",
"selection_aggregation": {
"method": "intersection_area",
"label": "Bodemkaartoppervlakte",
"unit": "ha",
},
},
status="ready",
)
assert VectorFeatureService._dataset_theme(dataset) == "soil"
assert VectorFeatureService.supports_selection_summary(dataset) is True
assert VectorFeatureService.can_use_full_area_fast_path(dataset, None) is False
def test_soil_operator_contract_has_no_direct_persistence_and_is_packaged() -> None:
operator = (ROOT / "scripts" / "provision_mol_soil_map.py").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
map_workspace = (ROOT / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8")
assert "/datasets/upload" in operator
assert "vector_features" in operator
assert "does not write directly" in " ".join(operator.split())
assert "SessionLocal" not in operator and "INSERT INTO" not in operator
assert "COPY scripts/provision_mol_soil_map.py" in dockerfile
assert "py_compile scripts/provision_mol_soil_map.py" in readiness
assert "id: 'soil'" in map_workspace
assert "dataset.source_name === 'dov_soil_map'" in map_workspace
def test_incomplete_wfs_pagination_fails_closed() -> None:
module = load_operator()
session = FakeSession(
[
{
"type": "FeatureCollection",
"numberMatched": 2,
"numberReturned": 0,
"features": [],
}
]
)
with pytest.raises(RuntimeError, match="returned 0 of 2"):
list(module.iter_wfs_pages(session, (0.0, 0.0, 1.0, 1.0), page_limit=100, timeout=30))