feat: add cross-domain Mol data profile
This commit is contained in:
@@ -1289,6 +1289,43 @@ Settings: `FLOOD_HAZARD_ENABLED`, `FLOOD_HAZARD_WCS_URL`,
|
||||
`FLOOD_HAZARD_MAX_SIDE_M`, `FLOOD_HAZARD_MAX_PIXELS`,
|
||||
`FLOOD_HAZARD_TIMEOUT_SECONDS` and `FLOOD_HAZARD_MAX_RESPONSE_MB`.
|
||||
|
||||
## Cross-domain thematic rasters and DOV soil
|
||||
|
||||
The governed thematic registry exposes five fixed MercatorNet products through
|
||||
`GET .../datasets/thematic-raster/products`. Acquisition uses
|
||||
`POST .../datasets/thematic-raster/acquire`; selection and PNG rendering use
|
||||
`POST .../raster/thematic/select` and `GET .../raster/thematic/image`.
|
||||
|
||||
Provision every product for the exact persisted Mol Area:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_thematic_rasters.py
|
||||
```
|
||||
|
||||
Inspect the complete 28-municipality matrix without writes, then run it after
|
||||
the Mol source/runtime gate passes:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_thematic_rasters.py \
|
||||
--project-name "Kempen Regional Workbench" --all-municipalities --dry-run
|
||||
```
|
||||
|
||||
Settings: `THEMATIC_RASTER_ENABLED`, `THEMATIC_RASTER_WCS_URL`,
|
||||
`THEMATIC_RASTER_MIN_SIDE_M`, `THEMATIC_RASTER_MAX_SIDE_M`,
|
||||
`THEMATIC_RASTER_MAX_PIXELS`, `THEMATIC_RASTER_TIMEOUT_SECONDS` and
|
||||
`THEMATIC_RASTER_MAX_RESPONSE_MB`.
|
||||
|
||||
Provision the official DOV soil polygons for Mol through the existing vector
|
||||
upload path:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_mol_soil_map.py
|
||||
```
|
||||
|
||||
Use `--fetch-only` to retain and validate source evidence without importing.
|
||||
The operator never writes directly to PostGIS. Soil drainage and related map
|
||||
classes represent the 1949-1971 survey and are not current observations.
|
||||
|
||||
## Waterinfo station histories
|
||||
|
||||
Run the explicit operator after the regional workspace and Mol Area exist:
|
||||
|
||||
@@ -26,6 +26,8 @@ from app.schemas import (
|
||||
TerrainSelectionRequest,
|
||||
FloodHazardAcquireRequest,
|
||||
FloodHazardSelectionRequest,
|
||||
ThematicRasterAcquireRequest,
|
||||
ThematicRasterSelectionRequest,
|
||||
VectorBBoxResponse,
|
||||
VectorBufferRequest,
|
||||
VectorClipRequest,
|
||||
@@ -48,6 +50,8 @@ from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||
from app.services.terrain_analysis_service import TerrainAnalysisService
|
||||
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
||||
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
||||
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
||||
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}", tags=["datasets"])
|
||||
@@ -207,6 +211,30 @@ def list_flood_hazard_products(project_id: UUID, db: Session = Depends(get_db)):
|
||||
return envelope({"items": items, "total": len(items)})
|
||||
|
||||
|
||||
@router.post("/datasets/thematic-raster/acquire", response_model=dict)
|
||||
def acquire_bounded_thematic_raster(
|
||||
project_id: UUID,
|
||||
payload: ThematicRasterAcquireRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
job = JobService.run_sync_job(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
job_type="raster.thematic.acquire",
|
||||
parameters=payload.model_dump(mode="json"),
|
||||
operation=lambda: ThematicRasterAcquisitionService.acquire(db, project_id, payload),
|
||||
)
|
||||
return envelope(job)
|
||||
|
||||
|
||||
@router.get("/datasets/thematic-raster/products", response_model=dict)
|
||||
def list_thematic_raster_products(project_id: UUID, db: Session = Depends(get_db)):
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
items = ThematicRasterAcquisitionService.list_products()
|
||||
return envelope({"items": items, "total": len(items)})
|
||||
|
||||
|
||||
@router.get("/datasets", response_model=dict)
|
||||
def list_datasets(
|
||||
project_id: UUID,
|
||||
@@ -560,6 +588,30 @@ def raster_flood_hazard_image(
|
||||
)
|
||||
|
||||
|
||||
@router.post("/datasets/{dataset_id}/raster/thematic/select", response_model=dict)
|
||||
def raster_thematic_selection(
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
payload: ThematicRasterSelectionRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
return envelope(ThematicRasterAnalysisService.analyze(db, project_id, dataset_id, payload))
|
||||
|
||||
|
||||
@router.get("/datasets/{dataset_id}/raster/thematic/image")
|
||||
def raster_thematic_image(
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
content = ThematicRasterAnalysisService.render_png(db, project_id, dataset_id)
|
||||
return Response(
|
||||
content=content,
|
||||
media_type="image/png",
|
||||
headers={"Cache-Control": "private, max-age=86400"},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/datasets/{dataset_id}/raster/stats", response_model=dict)
|
||||
def raster_stats(
|
||||
project_id: UUID,
|
||||
|
||||
@@ -53,6 +53,16 @@ class Settings(BaseSettings):
|
||||
flood_hazard_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="FLOOD_HAZARD_MAX_PIXELS")
|
||||
flood_hazard_timeout_seconds: int = Field(default=300, ge=1, validation_alias="FLOOD_HAZARD_TIMEOUT_SECONDS")
|
||||
flood_hazard_max_response_mb: int = Field(default=160, ge=1, validation_alias="FLOOD_HAZARD_MAX_RESPONSE_MB")
|
||||
thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED")
|
||||
thematic_raster_wcs_url: str = Field(
|
||||
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
|
||||
validation_alias="THEMATIC_RASTER_WCS_URL",
|
||||
)
|
||||
thematic_raster_min_side_m: float = Field(default=100.0, gt=0, validation_alias="THEMATIC_RASTER_MIN_SIDE_M")
|
||||
thematic_raster_max_side_m: float = Field(default=20_000.0, gt=0, validation_alias="THEMATIC_RASTER_MAX_SIDE_M")
|
||||
thematic_raster_max_pixels: int = Field(default=12_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS")
|
||||
thematic_raster_timeout_seconds: int = Field(default=300, ge=1, validation_alias="THEMATIC_RASTER_TIMEOUT_SECONDS")
|
||||
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
|
||||
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
|
||||
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
|
||||
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
|
||||
|
||||
@@ -51,6 +51,15 @@ from .flood_hazard import (
|
||||
FloodHazardSelectionResponse,
|
||||
FloodHazardSelectionSummary,
|
||||
)
|
||||
from .thematic_raster import (
|
||||
ThematicRasterAcquireRequest,
|
||||
ThematicRasterAcquisitionResult,
|
||||
ThematicRasterMetric,
|
||||
ThematicRasterProductRead,
|
||||
ThematicRasterSelectionRequest,
|
||||
ThematicRasterSelectionResponse,
|
||||
ThematicRasterSelectionSummary,
|
||||
)
|
||||
from .external import (
|
||||
ExternalFetchRequest,
|
||||
ExternalFetchResponse,
|
||||
@@ -167,6 +176,13 @@ __all__ = [
|
||||
"FloodHazardSelectionRequest",
|
||||
"FloodHazardSelectionResponse",
|
||||
"FloodHazardSelectionSummary",
|
||||
"ThematicRasterAcquireRequest",
|
||||
"ThematicRasterAcquisitionResult",
|
||||
"ThematicRasterMetric",
|
||||
"ThematicRasterProductRead",
|
||||
"ThematicRasterSelectionRequest",
|
||||
"ThematicRasterSelectionResponse",
|
||||
"ThematicRasterSelectionSummary",
|
||||
"VectorBBoxResponse",
|
||||
"VectorClipRequest",
|
||||
"VectorBufferRequest",
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class ThematicRasterAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
product_key: str
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class ThematicRasterProductRead(BaseModel):
|
||||
key: str
|
||||
display_name: str
|
||||
theme: str
|
||||
metric_kind: str
|
||||
coverage_id: str
|
||||
native_resolution_m: float
|
||||
source_crs: str
|
||||
source_value_unit: str
|
||||
observation_year: int
|
||||
source_version: str
|
||||
catalog_url: str
|
||||
attribution: str
|
||||
license_note: str
|
||||
legend_min_label: str
|
||||
legend_max_label: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class ThematicRasterAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
product_key: str
|
||||
display_name: str
|
||||
theme: str
|
||||
metric_kind: str
|
||||
coverage_id: str
|
||||
resolution_m: float
|
||||
width: int
|
||||
height: int
|
||||
valid_pixel_count: int
|
||||
bbox_epsg4326: list[float]
|
||||
bbox_epsg31370: list[float]
|
||||
observation_year: int
|
||||
source_value_unit: str
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class ThematicRasterSelectionRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
|
||||
|
||||
class ThematicRasterMetric(BaseModel):
|
||||
metric_key: str
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
derived: bool = True
|
||||
is_estimate: bool = True
|
||||
|
||||
|
||||
class ThematicRasterSelectionSummary(BaseModel):
|
||||
metric_label: str
|
||||
metric_value: float
|
||||
metric_unit: str
|
||||
aggregation_method: str
|
||||
primary_metric_key: str
|
||||
metrics: list[ThematicRasterMetric]
|
||||
|
||||
|
||||
class ThematicRasterSelectionResponse(BaseModel):
|
||||
dataset_id: UUID
|
||||
product_key: str
|
||||
theme: str
|
||||
metric_kind: str
|
||||
selection_bbox: VectorSelectionBBox
|
||||
selection_area_id: UUID | None = None
|
||||
selected_cell_count: int
|
||||
valid_cell_count: int
|
||||
coverage_ratio: float
|
||||
resolution_m: float
|
||||
observation_year: int
|
||||
summary: ThematicRasterSelectionSummary
|
||||
unsupported_metrics: list[str]
|
||||
limitation_message: str
|
||||
generated_at: str
|
||||
@@ -22,8 +22,11 @@ from app.schemas.assistant import (
|
||||
AssistantTemporalSeries,
|
||||
)
|
||||
from app.schemas.flood_hazard import FloodHazardSelectionRequest
|
||||
from app.schemas.thematic_raster import ThematicRasterSelectionRequest
|
||||
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
||||
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
||||
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
||||
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
@@ -42,9 +45,17 @@ class GeoAssistantService:
|
||||
)
|
||||
ESTIMATE_TOPIC_TERMS = {
|
||||
"population": ("bevolk", "inwoner"),
|
||||
"space_occupation": ("ruimtebeslag",),
|
||||
"open_space": ("open ruimte",),
|
||||
"accessibility": ("bereikbaar", "knooppunt"),
|
||||
"services": ("voorziening",),
|
||||
}
|
||||
ESTIMATE_TOPIC_LABELS = {
|
||||
"population": "bevolkingswaarden",
|
||||
"space_occupation": "ruimtebeslagoppervlakten",
|
||||
"open_space": "openruimte-oppervlakten",
|
||||
"accessibility": "bereikbaarheidsscores",
|
||||
"services": "voorzieningenscores",
|
||||
}
|
||||
|
||||
@classmethod
|
||||
@@ -264,6 +275,19 @@ class GeoAssistantService:
|
||||
if dataset.dataset_type == "raster" and dataset.source_name == FloodHazardAcquisitionService.PROVIDER
|
||||
and (area is None or dataset.area_id is None or dataset.area_id == area.id)
|
||||
]
|
||||
thematic_candidates = [
|
||||
dataset
|
||||
for dataset in datasets
|
||||
if dataset.dataset_type == "raster" and dataset.source_name == ThematicRasterAcquisitionService.PROVIDER
|
||||
and (area is None or dataset.area_id is None or dataset.area_id == area.id)
|
||||
]
|
||||
thematic_by_product: dict[str, Dataset] = {}
|
||||
for dataset in thematic_candidates:
|
||||
product_key = str((dataset.source_metadata or {}).get("product_key") or "")
|
||||
current = thematic_by_product.get(product_key)
|
||||
if product_key and (current is None or (dataset.imported_at or datetime.min.replace(tzinfo=timezone.utc)) > (current.imported_at or datetime.min.replace(tzinfo=timezone.utc))):
|
||||
thematic_by_product[product_key] = dataset
|
||||
thematic_datasets = list(thematic_by_product.values())
|
||||
warnings: list[str] = []
|
||||
context_metrics: list[AssistantContextMetric] = []
|
||||
source_dataset_ids: list[UUID] = []
|
||||
@@ -323,6 +347,47 @@ class GeoAssistantService:
|
||||
}
|
||||
)
|
||||
|
||||
for dataset in sorted(thematic_datasets, key=lambda item: str((item.source_metadata or {}).get("product_key") or item.name)):
|
||||
try:
|
||||
result = ThematicRasterAnalysisService.analyze(
|
||||
db,
|
||||
project_id,
|
||||
dataset.id,
|
||||
ThematicRasterSelectionRequest(bbox=bbox, area_id=area.id if area is not None else None),
|
||||
settings=self.settings,
|
||||
)
|
||||
except AppError as exc:
|
||||
warnings.append(f"{dataset.name}: {exc.message}")
|
||||
continue
|
||||
serialized_metrics: list[dict[str, Any]] = []
|
||||
for metric in result["summary"]["metrics"]:
|
||||
item = AssistantContextMetric(
|
||||
theme=result["theme"],
|
||||
label=str(metric["metric_label"]),
|
||||
value=float(metric["metric_value"]),
|
||||
unit=str(metric["metric_unit"]),
|
||||
source=ThematicRasterAcquisitionService.ATTRIBUTION,
|
||||
dataset_id=dataset.id,
|
||||
observed_at=dataset.observed_at,
|
||||
is_estimate=bool(metric.get("is_estimate", True)),
|
||||
)
|
||||
context_metrics.append(item)
|
||||
serialized_metrics.append(item.model_dump(mode="json"))
|
||||
serialized_metrics[-1]["measurement_quality"] = "resolutiegebonden_bronmeting"
|
||||
source_dataset_ids.append(dataset.id)
|
||||
current_context.append(
|
||||
{
|
||||
"dataset_name": dataset.name,
|
||||
"dataset_id": str(dataset.id),
|
||||
"theme": result["theme"],
|
||||
"source": ThematicRasterAcquisitionService.ATTRIBUTION,
|
||||
"observed_at": dataset.observed_at.isoformat() if dataset.observed_at else None,
|
||||
"metrics": serialized_metrics,
|
||||
"unsupported_metrics": result["unsupported_metrics"],
|
||||
"warning": result["limitation_message"],
|
||||
}
|
||||
)
|
||||
|
||||
for dataset in sorted(
|
||||
flood_hazard_datasets,
|
||||
key=lambda item: str((item.source_metadata or {}).get("product_key") or item.name),
|
||||
@@ -425,6 +490,7 @@ class GeoAssistantService:
|
||||
"water_volume_available": False,
|
||||
"water_volume_reason": "Geen bathymetrie gekoppeld voor de permanente inhoud van waterlichamen.",
|
||||
"flood_hazard_scenarios_available": bool(flood_hazard_datasets),
|
||||
"thematic_policy_rasters_available": bool(thematic_datasets),
|
||||
"flood_depth_area_integral_is_concurrent_volume": False,
|
||||
"object_counts_are_supporting_metrics": True,
|
||||
"causal_explanations_available": False,
|
||||
|
||||
@@ -0,0 +1,597 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import box, mapping
|
||||
from shapely.ops import transform as shapely_transform
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.schemas.thematic_raster import (
|
||||
ThematicRasterAcquireRequest,
|
||||
ThematicRasterAcquisitionResult,
|
||||
ThematicRasterProductRead,
|
||||
)
|
||||
from app.services.dataset_service import DatasetService
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ThematicRasterProduct:
|
||||
key: str
|
||||
display_name: str
|
||||
theme: str
|
||||
metric_kind: str
|
||||
coverage_id: str
|
||||
native_resolution_m: float
|
||||
source_value_unit: str
|
||||
observation_year: int
|
||||
source_version: str
|
||||
catalog_url: str
|
||||
legend_min_label: str
|
||||
legend_max_label: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class ThematicRasterAcquisitionService:
|
||||
"""Acquire bounded, allowlisted policy rasters from MercatorNet WCS."""
|
||||
|
||||
PROVIDER = "department_omgeving_thematic_raster"
|
||||
SOURCE_CRS = "EPSG:31370"
|
||||
WCS_VERSION = "1.0.0"
|
||||
NODATA = -9999.0
|
||||
WCS_TILE_SIDE_M = 10_000.0
|
||||
WCS_REQUEST_INTERVAL_SECONDS = 0.5
|
||||
ATTRIBUTION = "Bron: Departement Omgeving, MercatorNet"
|
||||
LICENSE_NOTE = "Publieke GDI-Vlaanderen bron; bronvermelding en productspecifieke gebruiksvoorwaarden blijven van toepassing."
|
||||
|
||||
@staticmethod
|
||||
def _products() -> dict[str, ThematicRasterProduct]:
|
||||
products = (
|
||||
ThematicRasterProduct(
|
||||
key="space_occupation_2025",
|
||||
display_name="Ruimtebeslag Vlaanderen 2025",
|
||||
theme="space_occupation",
|
||||
metric_kind="binary_area",
|
||||
coverage_id="lu:lu_ruibes_vlaa_2025_v3",
|
||||
native_resolution_m=10.0,
|
||||
source_value_unit="class_0_1",
|
||||
observation_year=2025,
|
||||
source_version="Toestand 2025 versie 3",
|
||||
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/ruimtebeslag-vlaanderen-toestand-2025",
|
||||
legend_min_label="Geen ruimtebeslag",
|
||||
legend_max_label="Ruimtebeslag",
|
||||
limitation_message=(
|
||||
"Binaire 10 m-kaart volgens de beleidsdefinitie van ruimtebeslag. Celgebaseerde oppervlakte is een "
|
||||
"resolutiegebonden schatting en is niet gelijk aan uitsluitend bebouwde oppervlakte of verharding."
|
||||
),
|
||||
),
|
||||
ThematicRasterProduct(
|
||||
key="open_space_2022",
|
||||
display_name="Open ruimte Vlaanderen 2022",
|
||||
theme="open_space",
|
||||
metric_kind="binary_area",
|
||||
coverage_id="lu:lu_openruimte_vlaa_2022_v3",
|
||||
native_resolution_m=10.0,
|
||||
source_value_unit="class_0_1",
|
||||
observation_year=2022,
|
||||
source_version="Toestand 2022 versie 3",
|
||||
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/open-ruimte-vlaanderen-toestand-2022",
|
||||
legend_min_label="Geen open ruimte",
|
||||
legend_max_label="Open ruimte",
|
||||
limitation_message=(
|
||||
"Binaire 10 m-beleidskaart afgeleid uit landgebruik, ruimtebeslag en kernen. Open ruimte is niet "
|
||||
"synoniem met natuur, bos, publieke toegankelijkheid of planologische bestemming."
|
||||
),
|
||||
),
|
||||
ThematicRasterProduct(
|
||||
key="population_density_2019",
|
||||
display_name="Inwonersdichtheid per hectare 2019",
|
||||
theme="population",
|
||||
metric_kind="population_density",
|
||||
coverage_id="ni:ni_inw_ha_vlaa_2019",
|
||||
native_resolution_m=100.0,
|
||||
source_value_unit="inhabitants_per_hectare",
|
||||
observation_year=2019,
|
||||
source_version="Toestand 2019",
|
||||
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/inwonersdichtheid-per-ha-vlaanderen-toestand-2019",
|
||||
legend_min_label="0 inwoners/ha",
|
||||
legend_max_label="Hogere dichtheid",
|
||||
limitation_message=(
|
||||
"Statistische 1 ha-rasterinschatting voor 2019, gecorrigeerd op statistische-sectorbasis. De som "
|
||||
"binnen een getekende grens is een rasterraming en geen actuele registertelling."
|
||||
),
|
||||
),
|
||||
ThematicRasterProduct(
|
||||
key="node_value_2022",
|
||||
display_name="Knooppuntwaarde collectief vervoer 2022",
|
||||
theme="accessibility",
|
||||
metric_kind="index_score",
|
||||
coverage_id="lu:lu_knptw_ha_2022_v3",
|
||||
native_resolution_m=100.0,
|
||||
source_value_unit="source_index_score",
|
||||
observation_year=2022,
|
||||
source_version="Toestand 2022 versie 3",
|
||||
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/knooppuntwaarde-per-ha-toestand-2022",
|
||||
legend_min_label="Lagere knooppuntwaarde",
|
||||
legend_max_label="Hogere knooppuntwaarde",
|
||||
limitation_message=(
|
||||
"Bronindex per hectare op basis van collectief-vervoerknooppunten en afstandsverval. De score is "
|
||||
"geen percentage, reistijd, dienstregeling van vandaag of garantie op bereikbaarheid."
|
||||
),
|
||||
),
|
||||
ThematicRasterProduct(
|
||||
key="service_level_2022",
|
||||
display_name="Totaal voorzieningenniveau 2022",
|
||||
theme="services",
|
||||
metric_kind="normalized_score",
|
||||
coverage_id="lu:lu_totvznv_ha_2022_v3",
|
||||
native_resolution_m=100.0,
|
||||
source_value_unit="score_0_1",
|
||||
observation_year=2022,
|
||||
source_version="Toestand 2022 versie 3",
|
||||
catalog_url="https://www.vlaanderen.be/datavindplaats/catalogus/totaal-voorzieningenniveau-toestand-2022",
|
||||
legend_min_label="Lager voorzieningenniveau",
|
||||
legend_max_label="Hoger voorzieningenniveau",
|
||||
limitation_message=(
|
||||
"Genormaliseerde 0-1 nabijheidsscore voor basis-, regionale en metropolitane voorzieningen in "
|
||||
"referentiejaar 2022. Dit is geen objecttelling, openingsurencontrole of actuele reistijd."
|
||||
),
|
||||
),
|
||||
)
|
||||
return {product.key: product for product in products}
|
||||
|
||||
@staticmethod
|
||||
def list_products() -> list[dict[str, Any]]:
|
||||
return [
|
||||
ThematicRasterProductRead(
|
||||
key=product.key,
|
||||
display_name=product.display_name,
|
||||
theme=product.theme,
|
||||
metric_kind=product.metric_kind,
|
||||
coverage_id=product.coverage_id,
|
||||
native_resolution_m=product.native_resolution_m,
|
||||
source_crs=ThematicRasterAcquisitionService.SOURCE_CRS,
|
||||
source_value_unit=product.source_value_unit,
|
||||
observation_year=product.observation_year,
|
||||
source_version=product.source_version,
|
||||
catalog_url=product.catalog_url,
|
||||
attribution=ThematicRasterAcquisitionService.ATTRIBUTION,
|
||||
license_note=ThematicRasterAcquisitionService.LICENSE_NOTE,
|
||||
legend_min_label=product.legend_min_label,
|
||||
legend_max_label=product.legend_max_label,
|
||||
limitation_message=product.limitation_message,
|
||||
).model_dump()
|
||||
for product in ThematicRasterAcquisitionService._products().values()
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _product(product_key: str) -> ThematicRasterProduct:
|
||||
product = ThematicRasterAcquisitionService._products().get(product_key.strip().lower())
|
||||
if product is None:
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_PRODUCT_NOT_SUPPORTED",
|
||||
message="Select a product from the governed Flemish thematic raster registry",
|
||||
details={"product_key": product_key},
|
||||
status_code=422,
|
||||
)
|
||||
return product
|
||||
|
||||
@staticmethod
|
||||
def _prepared_request(payload: ThematicRasterAcquireRequest, settings: Settings) -> dict[str, Any]:
|
||||
if not settings.thematic_raster_enabled:
|
||||
raise AppError(code="THEMATIC_RASTER_NOT_CONFIGURED", message="Official thematic raster acquisition is disabled", status_code=503)
|
||||
product = ThematicRasterAcquisitionService._product(payload.product_key)
|
||||
values = (payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
|
||||
if payload.bbox.crs.upper() != "EPSG:4326":
|
||||
raise AppError(code="INVALID_BBOX_CRS", message="Thematic raster selection requires EPSG:4326", status_code=400)
|
||||
if not all(math.isfinite(value) for value in values) or values[0] >= values[2] or values[1] >= values[3]:
|
||||
raise AppError(code="INVALID_BBOX", message="Thematic raster selection must be a finite non-empty rectangle", status_code=400)
|
||||
|
||||
transformer = Transformer.from_crs("EPSG:4326", ThematicRasterAcquisitionService.SOURCE_CRS, always_xy=True)
|
||||
raw_bounds = transformer.transform_bounds(*values, densify_pts=21)
|
||||
resolution = product.native_resolution_m
|
||||
lambert_bounds = (
|
||||
math.floor(raw_bounds[0] / resolution) * resolution,
|
||||
math.floor(raw_bounds[1] / resolution) * resolution,
|
||||
math.ceil(raw_bounds[2] / resolution) * resolution,
|
||||
math.ceil(raw_bounds[3] / resolution) * resolution,
|
||||
)
|
||||
width_m = lambert_bounds[2] - lambert_bounds[0]
|
||||
height_m = lambert_bounds[3] - lambert_bounds[1]
|
||||
if width_m < settings.thematic_raster_min_side_m or height_m < settings.thematic_raster_min_side_m:
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_SELECTION_TOO_SMALL",
|
||||
message=f"Select an area of at least {settings.thematic_raster_min_side_m:g} by {settings.thematic_raster_min_side_m:g} metres",
|
||||
status_code=422,
|
||||
)
|
||||
if width_m > settings.thematic_raster_max_side_m or height_m > settings.thematic_raster_max_side_m:
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_SELECTION_TOO_LARGE",
|
||||
message=f"Select an area no larger than {settings.thematic_raster_max_side_m:g} by {settings.thematic_raster_max_side_m:g} metres",
|
||||
details={"width_m": width_m, "height_m": height_m},
|
||||
status_code=422,
|
||||
)
|
||||
width = max(1, round(width_m / resolution))
|
||||
height = max(1, round(height_m / resolution))
|
||||
if width * height > settings.thematic_raster_max_pixels:
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_SELECTION_TOO_LARGE",
|
||||
message="Thematic raster selection exceeds the configured cell limit",
|
||||
details={"pixel_count": width * height, "max_pixels": settings.thematic_raster_max_pixels},
|
||||
status_code=422,
|
||||
)
|
||||
request_identity = {
|
||||
"provider": ThematicRasterAcquisitionService.PROVIDER,
|
||||
"coverage_id": product.coverage_id,
|
||||
"bbox_epsg4326": [round(float(value), 8) for value in values],
|
||||
"bbox_epsg31370": [round(float(value), 3) for value in lambert_bounds],
|
||||
"resolution_m": resolution,
|
||||
"area_id": str(payload.area_id) if payload.area_id else None,
|
||||
}
|
||||
request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode()).hexdigest()
|
||||
return {
|
||||
**request_identity,
|
||||
"product": product,
|
||||
"request_hash": request_hash,
|
||||
"width": width,
|
||||
"height": height,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _wcs_request_url(settings: Settings, product: ThematicRasterProduct, bounds: tuple[float, float, float, float]) -> str:
|
||||
query = {
|
||||
"SERVICE": "WCS",
|
||||
"VERSION": ThematicRasterAcquisitionService.WCS_VERSION,
|
||||
"REQUEST": "GetCoverage",
|
||||
"COVERAGE": product.coverage_id,
|
||||
"CRS": ThematicRasterAcquisitionService.SOURCE_CRS,
|
||||
"BBOX": ",".join(f"{value:.3f}" for value in bounds),
|
||||
"RESX": f"{product.native_resolution_m:g}",
|
||||
"RESY": f"{product.native_resolution_m:g}",
|
||||
"FORMAT": "image/tiff",
|
||||
"RESPONSE_CRS": ThematicRasterAcquisitionService.SOURCE_CRS,
|
||||
}
|
||||
return f"{settings.thematic_raster_wcs_url}?{urlencode(query)}"
|
||||
|
||||
@staticmethod
|
||||
def _tile_bounds(prepared: dict[str, Any]) -> list[tuple[float, float, float, float]]:
|
||||
min_x, min_y, max_x, max_y = prepared["bbox_epsg31370"]
|
||||
resolution = prepared["product"].native_resolution_m
|
||||
side = max(resolution, math.floor(ThematicRasterAcquisitionService.WCS_TILE_SIDE_M / resolution) * resolution)
|
||||
tiles: list[tuple[float, float, float, float]] = []
|
||||
y = min_y
|
||||
while y < max_y:
|
||||
tile_max_y = min(y + side, max_y)
|
||||
x = min_x
|
||||
while x < max_x:
|
||||
tile_max_x = min(x + side, max_x)
|
||||
tiles.append((x, y, tile_max_x, tile_max_y))
|
||||
x = tile_max_x
|
||||
y = tile_max_y
|
||||
return tiles
|
||||
|
||||
@staticmethod
|
||||
def _scope_geometry(db, project_id: UUID, area_id: UUID | None, bbox_epsg4326: list[float]):
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
selection = box(*bbox_epsg4326)
|
||||
if area_id is None:
|
||||
return selection
|
||||
area = db.get(Area, area_id)
|
||||
if not area:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
if area.project_id != project_id:
|
||||
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
|
||||
intersection = to_shape(area.geometry).intersection(selection)
|
||||
if intersection.is_empty or intersection.area <= 0:
|
||||
raise AppError(code="THEMATIC_RASTER_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422)
|
||||
return intersection
|
||||
|
||||
@staticmethod
|
||||
def _fetch(request_url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, str]:
|
||||
request = Request(request_url, headers={"Accept": "image/tiff,*/*", "User-Agent": "GeoIntel/0.1 bounded-thematic-raster"})
|
||||
max_bytes = settings.thematic_raster_max_response_mb * 1024 * 1024
|
||||
try:
|
||||
with (opener or urlopen)(request, timeout=settings.thematic_raster_timeout_seconds) as response:
|
||||
content_type = str(response.headers.get("Content-Type", ""))
|
||||
content_length = response.headers.get("Content-Length")
|
||||
if content_length and int(content_length) > max_bytes:
|
||||
raise AppError(code="THEMATIC_RASTER_RESPONSE_TOO_LARGE", message="Official raster response exceeds the configured size limit", status_code=502)
|
||||
content = response.read(max_bytes + 1)
|
||||
except AppError:
|
||||
raise
|
||||
except HTTPError as exc:
|
||||
preview = exc.read(300).decode("utf-8", errors="replace")
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_PROVIDER_UNAVAILABLE",
|
||||
message="The official MercatorNet WCS could not complete the bounded request",
|
||||
details={"reason": str(exc), "provider_status_code": int(exc.code), "response_preview": preview},
|
||||
status_code=502,
|
||||
) from exc
|
||||
except (URLError, TimeoutError, OSError) as exc:
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_PROVIDER_UNAVAILABLE",
|
||||
message="The official MercatorNet WCS could not complete the bounded request",
|
||||
details={"reason": str(exc)},
|
||||
status_code=502,
|
||||
) from exc
|
||||
if len(content) > max_bytes:
|
||||
raise AppError(code="THEMATIC_RASTER_RESPONSE_TOO_LARGE", message="Official raster response exceeds the configured size limit", status_code=502)
|
||||
if not content.startswith((b"II*\x00", b"MM\x00*")):
|
||||
preview = content[:300].decode("utf-8", errors="replace")
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_PROVIDER_INVALID_RESPONSE",
|
||||
message="The official MercatorNet service did not return a GeoTIFF coverage",
|
||||
details={"content_type": content_type, "response_preview": preview},
|
||||
status_code=502,
|
||||
)
|
||||
return content, content_type
|
||||
|
||||
@staticmethod
|
||||
def _mosaic(coverages: list[bytes], product: ThematicRasterProduct) -> bytes:
|
||||
if len(coverages) == 1:
|
||||
return coverages[0]
|
||||
try:
|
||||
import rasterio
|
||||
from rasterio.io import MemoryFile
|
||||
from rasterio.merge import merge
|
||||
except ImportError as exc:
|
||||
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio is required to assemble thematic raster tiles", status_code=503) from exc
|
||||
memories = [MemoryFile(content) for content in coverages]
|
||||
sources = []
|
||||
try:
|
||||
sources = [memory.open() for memory in memories]
|
||||
for source in sources:
|
||||
if source.crs is None or source.crs.to_epsg() != 31370 or source.count != 1:
|
||||
raise AppError(code="THEMATIC_RASTER_TILE_MISMATCH", message="Thematic raster tiles have incompatible CRS or bands", status_code=502)
|
||||
if not all(math.isclose(abs(float(value)), product.native_resolution_m, abs_tol=0.05) for value in source.res):
|
||||
raise AppError(code="THEMATIC_RASTER_TILE_MISMATCH", message="Thematic raster tile resolution differs from the registry", status_code=502)
|
||||
mosaic, transform = merge(sources, res=(product.native_resolution_m, product.native_resolution_m), nodata=ThematicRasterAcquisitionService.NODATA, dtype="float32")
|
||||
profile = sources[0].profile.copy()
|
||||
profile.pop("blockxsize", None)
|
||||
profile.pop("blockysize", None)
|
||||
profile.update(driver="GTiff", width=mosaic.shape[2], height=mosaic.shape[1], count=1, dtype="float32", crs=ThematicRasterAcquisitionService.SOURCE_CRS, transform=transform, nodata=ThematicRasterAcquisitionService.NODATA, compress="deflate", predictor=3)
|
||||
with MemoryFile() as output_memory:
|
||||
with output_memory.open(**profile) as output:
|
||||
output.write(mosaic)
|
||||
return output_memory.read()
|
||||
except AppError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AppError(code="THEMATIC_RASTER_TILE_MOSAIC_FAILED", message="Thematic raster tiles could not be assembled", details={"reason": str(exc)}, status_code=502) from exc
|
||||
finally:
|
||||
for source in sources:
|
||||
source.close()
|
||||
for memory in memories:
|
||||
memory.close()
|
||||
|
||||
@staticmethod
|
||||
def _fetch_coverage(prepared: dict[str, Any], settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, dict[str, Any]]:
|
||||
product: ThematicRasterProduct = prepared["product"]
|
||||
request_urls = [ThematicRasterAcquisitionService._wcs_request_url(settings, product, bounds) for bounds in ThematicRasterAcquisitionService._tile_bounds(prepared)]
|
||||
coverages: list[bytes] = []
|
||||
digest = hashlib.sha256()
|
||||
content_types: list[str] = []
|
||||
for index, request_url in enumerate(request_urls):
|
||||
if index and opener is None:
|
||||
time.sleep(ThematicRasterAcquisitionService.WCS_REQUEST_INTERVAL_SECONDS)
|
||||
content, content_type = ThematicRasterAcquisitionService._fetch(request_url, settings, opener)
|
||||
digest.update(len(content).to_bytes(8, "big"))
|
||||
digest.update(content)
|
||||
coverages.append(content)
|
||||
content_types.append(content_type)
|
||||
return ThematicRasterAcquisitionService._mosaic(coverages, product), {
|
||||
"tile_count": len(request_urls),
|
||||
"request_urls": request_urls,
|
||||
"response_content_types": content_types,
|
||||
"coverage_sha256": digest.hexdigest(),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _validate_values(values, product: ThematicRasterProduct) -> None:
|
||||
import numpy as np
|
||||
|
||||
if values.size == 0:
|
||||
raise AppError(code="THEMATIC_RASTER_NO_VALID_DATA", message="The official product contains no valid cells in this selection", status_code=422)
|
||||
minimum = float(values.min())
|
||||
maximum = float(values.max())
|
||||
if minimum < 0:
|
||||
raise AppError(code="THEMATIC_RASTER_INVALID_VALUES", message="Official thematic raster contains unexpected negative values", details={"minimum": minimum}, status_code=502)
|
||||
if product.metric_kind == "binary_area" and not set(np.unique(values).tolist()).issubset({0.0, 1.0}):
|
||||
raise AppError(code="THEMATIC_RASTER_INVALID_VALUES", message="Binary thematic raster contains classes outside 0 and 1", status_code=502)
|
||||
if product.metric_kind == "normalized_score" and maximum > 1.0001:
|
||||
raise AppError(code="THEMATIC_RASTER_INVALID_VALUES", message="Normalized thematic score falls outside the documented 0-1 range", details={"maximum": maximum}, status_code=502)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_raster(content: bytes, scope_geometry_4326, prepared: dict[str, Any]) -> tuple[bytes, dict[str, Any]]:
|
||||
try:
|
||||
import numpy as np
|
||||
from rasterio.io import MemoryFile
|
||||
from rasterio.mask import mask
|
||||
except ImportError as exc:
|
||||
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for thematic raster validation", status_code=503) from exc
|
||||
product: ThematicRasterProduct = prepared["product"]
|
||||
try:
|
||||
with MemoryFile(content) as source_memory, source_memory.open() as source:
|
||||
if source.crs is None or source.crs.to_epsg() != 31370:
|
||||
raise AppError(code="THEMATIC_RASTER_INVALID_CRS", message="Official thematic raster must use EPSG:31370", status_code=502)
|
||||
if source.count != 1:
|
||||
raise AppError(code="THEMATIC_RASTER_INVALID_BANDS", message="Official thematic raster must contain one band", status_code=502)
|
||||
if not all(math.isclose(abs(float(value)), product.native_resolution_m, abs_tol=0.05) for value in source.res):
|
||||
raise AppError(code="THEMATIC_RASTER_INVALID_RESOLUTION", message="Official thematic raster resolution differs from the registry", status_code=502)
|
||||
transformer = Transformer.from_crs("EPSG:4326", ThematicRasterAcquisitionService.SOURCE_CRS, always_xy=True)
|
||||
scope_metric = shapely_transform(transformer.transform, scope_geometry_4326)
|
||||
clipped, transform = mask(source, [mapping(scope_metric)], crop=True, filled=False, indexes=[1])
|
||||
band = np.ma.asarray(clipped[0], dtype="float32")
|
||||
raw = np.asarray(band.filled(np.nan), dtype="float32")
|
||||
invalid = np.ma.getmaskarray(band) | ~np.isfinite(raw)
|
||||
if source.nodata is not None:
|
||||
invalid |= np.isclose(raw, float(source.nodata))
|
||||
normalized = np.ma.array(raw, mask=invalid)
|
||||
values = normalized.compressed().astype("float64")
|
||||
ThematicRasterAcquisitionService._validate_values(values, product)
|
||||
profile = source.profile.copy()
|
||||
profile.pop("blockxsize", None)
|
||||
profile.pop("blockysize", None)
|
||||
profile.update(driver="GTiff", width=normalized.shape[1], height=normalized.shape[0], count=1, dtype="float32", crs=ThematicRasterAcquisitionService.SOURCE_CRS, transform=transform, nodata=ThematicRasterAcquisitionService.NODATA, compress="deflate", predictor=3)
|
||||
with MemoryFile() as output_memory:
|
||||
with output_memory.open(**profile) as output:
|
||||
output.write(normalized.filled(ThematicRasterAcquisitionService.NODATA), 1)
|
||||
normalized_content = output_memory.read()
|
||||
return normalized_content, {
|
||||
"width": int(normalized.shape[1]),
|
||||
"height": int(normalized.shape[0]),
|
||||
"valid_pixel_count": int(values.size),
|
||||
"nodata_value": ThematicRasterAcquisitionService.NODATA,
|
||||
"resolution_m": product.native_resolution_m,
|
||||
"minimum_value": float(values.min()),
|
||||
"maximum_value": float(values.max()),
|
||||
"p02_value": float(np.percentile(values, 2)),
|
||||
"p98_value": float(np.percentile(values, 98)),
|
||||
}
|
||||
except AppError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AppError(code="THEMATIC_RASTER_INVALID", message="The official thematic raster could not be validated", details={"reason": str(exc)}, status_code=502) from exc
|
||||
|
||||
@staticmethod
|
||||
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
|
||||
candidate = (
|
||||
db.query(Dataset)
|
||||
.filter(Dataset.project_id == project_id, Dataset.name == filename, Dataset.source_name == ThematicRasterAcquisitionService.PROVIDER, Dataset.status == "ready")
|
||||
.order_by(Dataset.imported_at.desc())
|
||||
.first()
|
||||
)
|
||||
return candidate if candidate and candidate.storage_path and Path(candidate.storage_path).is_file() else None
|
||||
|
||||
@staticmethod
|
||||
def acquire(db, project_id: UUID, payload: ThematicRasterAcquireRequest, *, settings: Settings | None = None, opener: Callable[..., Any] | None = None) -> dict[str, Any]:
|
||||
resolved_settings = settings or get_settings()
|
||||
prepared = ThematicRasterAcquisitionService._prepared_request(payload, resolved_settings)
|
||||
product: ThematicRasterProduct = prepared["product"]
|
||||
scope_geometry = ThematicRasterAcquisitionService._scope_geometry(db, project_id, payload.area_id, prepared["bbox_epsg4326"])
|
||||
filename = f"thematic_{product.key}_{prepared['request_hash'][:12]}.tif"
|
||||
if not payload.force_refresh:
|
||||
cached = ThematicRasterAcquisitionService._cached_dataset(db, project_id, filename)
|
||||
if cached is not None:
|
||||
metadata = cached.source_metadata or {}
|
||||
raster_metadata = cached.metadata_json or {}
|
||||
return ThematicRasterAcquisitionResult(
|
||||
output_dataset_id=cached.id,
|
||||
reused=True,
|
||||
provider=ThematicRasterAcquisitionService.PROVIDER,
|
||||
product_key=product.key,
|
||||
display_name=product.display_name,
|
||||
theme=product.theme,
|
||||
metric_kind=product.metric_kind,
|
||||
coverage_id=product.coverage_id,
|
||||
resolution_m=product.native_resolution_m,
|
||||
width=int(raster_metadata.get("width", prepared["width"])),
|
||||
height=int(raster_metadata.get("height", prepared["height"])),
|
||||
valid_pixel_count=int(metadata.get("valid_pixel_count", 0)),
|
||||
bbox_epsg4326=prepared["bbox_epsg4326"],
|
||||
bbox_epsg31370=prepared["bbox_epsg31370"],
|
||||
observation_year=product.observation_year,
|
||||
source_value_unit=product.source_value_unit,
|
||||
attribution=ThematicRasterAcquisitionService.ATTRIBUTION,
|
||||
limitation_message=product.limitation_message,
|
||||
).model_dump(mode="json")
|
||||
|
||||
coverage, transfer = ThematicRasterAcquisitionService._fetch_coverage(prepared, resolved_settings, opener)
|
||||
normalized, validation = ThematicRasterAcquisitionService._normalize_raster(coverage, scope_geometry, prepared)
|
||||
acquired_at = datetime.now(UTC)
|
||||
observed_at = datetime(product.observation_year, 12, 31, 23, 59, 59, tzinfo=UTC)
|
||||
scope_key = str(payload.area_id) if payload.area_id else prepared["request_hash"][:24]
|
||||
dataset = DatasetService.import_raster_bytes(
|
||||
db,
|
||||
project_id=project_id,
|
||||
area_id=payload.area_id,
|
||||
filename=filename,
|
||||
content=normalized,
|
||||
source=f"Departement Omgeving MercatorNet WCS {product.coverage_id}",
|
||||
source_name=ThematicRasterAcquisitionService.PROVIDER,
|
||||
temporal_series_key=f"department-omgeving:thematic-raster:{product.key}:{scope_key}",
|
||||
observed_at=observed_at,
|
||||
valid_from=datetime(product.observation_year, 1, 1, tzinfo=UTC),
|
||||
valid_to=observed_at,
|
||||
temporal_granularity="year",
|
||||
source_version=product.source_version,
|
||||
source_metadata={
|
||||
"provider": ThematicRasterAcquisitionService.PROVIDER,
|
||||
"service": "WCS",
|
||||
"service_version": ThematicRasterAcquisitionService.WCS_VERSION,
|
||||
"product_key": product.key,
|
||||
"product_display_name": product.display_name,
|
||||
"theme": product.theme,
|
||||
"metric_kind": product.metric_kind,
|
||||
"coverage_id": product.coverage_id,
|
||||
"native_resolution_m": product.native_resolution_m,
|
||||
"analysis_resolution_m": product.native_resolution_m,
|
||||
"source_crs": ThematicRasterAcquisitionService.SOURCE_CRS,
|
||||
"source_value_unit": product.source_value_unit,
|
||||
"observation_year": product.observation_year,
|
||||
"observation_date_precision": "year",
|
||||
"valid_pixel_count": validation["valid_pixel_count"],
|
||||
"minimum_value": validation["minimum_value"],
|
||||
"maximum_value": validation["maximum_value"],
|
||||
"render_min_value": validation["p02_value"],
|
||||
"render_max_value": validation["p98_value"],
|
||||
"bbox_epsg4326": prepared["bbox_epsg4326"],
|
||||
"bbox_epsg31370": prepared["bbox_epsg31370"],
|
||||
"catalog_url": product.catalog_url,
|
||||
"attribution": ThematicRasterAcquisitionService.ATTRIBUTION,
|
||||
"license_note": ThematicRasterAcquisitionService.LICENSE_NOTE,
|
||||
"legend_min_label": product.legend_min_label,
|
||||
"legend_max_label": product.legend_max_label,
|
||||
"coverage_scope": "municipality" if payload.area_id else "bounded_selection",
|
||||
},
|
||||
provenance_metadata={
|
||||
"acquisition": "explicit_bounded_tiled_wcs_coverage",
|
||||
"acquired_at": acquired_at.isoformat(),
|
||||
"request_hash": prepared["request_hash"],
|
||||
"tile_count": transfer["tile_count"],
|
||||
"tile_request_urls": transfer["request_urls"],
|
||||
"response_content_types": transfer["response_content_types"],
|
||||
"coverage_sha256": transfer["coverage_sha256"],
|
||||
"normalized_sha256": hashlib.sha256(normalized).hexdigest(),
|
||||
"bbox_epsg4326": prepared["bbox_epsg4326"],
|
||||
"bbox_epsg31370": prepared["bbox_epsg31370"],
|
||||
"clipped_to_area_id": str(payload.area_id) if payload.area_id else None,
|
||||
"validation": validation,
|
||||
"limitation_message": product.limitation_message,
|
||||
},
|
||||
)
|
||||
return ThematicRasterAcquisitionResult(
|
||||
output_dataset_id=dataset.id,
|
||||
reused=False,
|
||||
provider=ThematicRasterAcquisitionService.PROVIDER,
|
||||
product_key=product.key,
|
||||
display_name=product.display_name,
|
||||
theme=product.theme,
|
||||
metric_kind=product.metric_kind,
|
||||
coverage_id=product.coverage_id,
|
||||
resolution_m=product.native_resolution_m,
|
||||
width=validation["width"],
|
||||
height=validation["height"],
|
||||
valid_pixel_count=validation["valid_pixel_count"],
|
||||
bbox_epsg4326=prepared["bbox_epsg4326"],
|
||||
bbox_epsg31370=prepared["bbox_epsg31370"],
|
||||
observation_year=product.observation_year,
|
||||
source_value_unit=product.source_value_unit,
|
||||
attribution=ThematicRasterAcquisitionService.ATTRIBUTION,
|
||||
limitation_message=product.limitation_message,
|
||||
).model_dump(mode="json")
|
||||
@@ -0,0 +1,260 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import math
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import box, mapping
|
||||
from shapely.ops import transform as shapely_transform
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset
|
||||
from app.schemas.thematic_raster import (
|
||||
ThematicRasterMetric,
|
||||
ThematicRasterSelectionRequest,
|
||||
ThematicRasterSelectionResponse,
|
||||
ThematicRasterSelectionSummary,
|
||||
)
|
||||
from app.services.thematic_raster_acquisition_service import (
|
||||
ThematicRasterAcquisitionService,
|
||||
ThematicRasterProduct,
|
||||
)
|
||||
|
||||
|
||||
class ThematicRasterAnalysisService:
|
||||
@staticmethod
|
||||
def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> Dataset:
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if not dataset or dataset.project_id != project_id:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||
if dataset.dataset_type != "raster" or dataset.source_name != ThematicRasterAcquisitionService.PROVIDER:
|
||||
raise AppError(
|
||||
code="INVALID_THEMATIC_RASTER_DATASET",
|
||||
message="Thematic analysis requires a governed Departement Omgeving raster dataset",
|
||||
status_code=400,
|
||||
)
|
||||
if dataset.status != "ready" or not dataset.storage_path or not Path(dataset.storage_path).is_file():
|
||||
raise AppError(code="DATASET_FILE_MISSING", message="Persisted thematic raster file is unavailable", status_code=404)
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def _product(dataset: Dataset) -> ThematicRasterProduct:
|
||||
source_metadata = dataset.source_metadata or {}
|
||||
product = ThematicRasterAcquisitionService._products().get(str(source_metadata.get("product_key") or ""))
|
||||
if product is None or source_metadata.get("coverage_id") != product.coverage_id:
|
||||
raise AppError(code="INVALID_THEMATIC_RASTER_METADATA", message="Thematic raster provenance is incomplete", status_code=409)
|
||||
return product
|
||||
|
||||
@staticmethod
|
||||
def _selection_geometry(db, project_id: UUID, payload: ThematicRasterSelectionRequest):
|
||||
selection = box(payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
|
||||
if payload.area_id is None:
|
||||
return selection
|
||||
area = db.get(Area, payload.area_id)
|
||||
if not area:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
if area.project_id != project_id:
|
||||
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
|
||||
selection = selection.intersection(to_shape(area.geometry))
|
||||
if selection.is_empty or selection.area <= 0:
|
||||
raise AppError(code="THEMATIC_RASTER_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422)
|
||||
return selection
|
||||
|
||||
@staticmethod
|
||||
def _unsupported_metrics(product: ThematicRasterProduct) -> list[str]:
|
||||
if product.metric_kind == "binary_area":
|
||||
return ["object_count", "parcel_area", "current_land_use"]
|
||||
if product.metric_kind == "population_density":
|
||||
return ["current_population", "household_count", "address_level_population"]
|
||||
if product.metric_kind == "index_score":
|
||||
return ["travel_time_minutes", "current_timetable", "stop_count"]
|
||||
return ["facility_count", "opening_hours", "current_service_availability"]
|
||||
|
||||
@staticmethod
|
||||
def analyze(
|
||||
db,
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
payload: ThematicRasterSelectionRequest,
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
) -> dict:
|
||||
resolved_settings = settings or get_settings()
|
||||
dataset = ThematicRasterAnalysisService._load_dataset(db, project_id, dataset_id)
|
||||
product = ThematicRasterAnalysisService._product(dataset)
|
||||
selection_4326 = ThematicRasterAnalysisService._selection_geometry(db, project_id, payload)
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from rasterio.features import geometry_mask
|
||||
from rasterio.mask import mask
|
||||
except ImportError as exc:
|
||||
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for thematic raster analysis", status_code=503) from exc
|
||||
|
||||
try:
|
||||
with rasterio.open(dataset.storage_path) as source:
|
||||
if source.crs is None or source.crs.to_epsg() != 31370:
|
||||
raise AppError(code="INVALID_DATASET_CRS", message="Thematic raster CRS must be EPSG:31370", status_code=409)
|
||||
transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True)
|
||||
selection_metric = shapely_transform(transformer.transform, selection_4326)
|
||||
analysis_geometry = selection_metric.intersection(box(*source.bounds))
|
||||
if analysis_geometry.is_empty or analysis_geometry.area <= 0:
|
||||
raise AppError(code="THEMATIC_RASTER_SELECTION_OUTSIDE_DATASET", message="Selection does not overlap the persisted thematic raster", status_code=422)
|
||||
min_x, min_y, max_x, max_y = analysis_geometry.bounds
|
||||
expected_cells = math.ceil((max_x - min_x) / abs(source.res[0])) * math.ceil((max_y - min_y) / abs(source.res[1]))
|
||||
if expected_cells > resolved_settings.thematic_raster_max_pixels:
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_SELECTION_TOO_LARGE",
|
||||
message="Thematic raster analysis exceeds the configured cell limit",
|
||||
details={"pixel_count": expected_cells, "max_pixels": resolved_settings.thematic_raster_max_pixels},
|
||||
status_code=422,
|
||||
)
|
||||
clipped, clipped_transform = mask(source, [mapping(analysis_geometry)], crop=True, filled=False, indexes=[1])
|
||||
band = np.ma.asarray(clipped[0], dtype="float64")
|
||||
raw = band.filled(np.nan)
|
||||
selected = geometry_mask([mapping(analysis_geometry)], out_shape=band.shape, transform=clipped_transform, invert=True)
|
||||
valid = selected & ~np.ma.getmaskarray(band) & np.isfinite(raw)
|
||||
if source.nodata is not None:
|
||||
valid &= ~np.isclose(raw, float(source.nodata))
|
||||
values = raw[valid]
|
||||
ThematicRasterAcquisitionService._validate_values(values, product)
|
||||
selected_cell_count = int(selected.sum())
|
||||
valid_cell_count = int(values.size)
|
||||
resolution_x = abs(float(source.res[0]))
|
||||
resolution_y = abs(float(source.res[1]))
|
||||
cell_area_m2 = resolution_x * resolution_y
|
||||
except AppError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_ANALYSIS_FAILED",
|
||||
message="The persisted thematic raster could not be analysed",
|
||||
details={"reason": str(exc)},
|
||||
status_code=500,
|
||||
) from exc
|
||||
|
||||
def metric(key: str, label: str, value: float, unit: str, method: str, *, estimate: bool = True) -> ThematicRasterMetric:
|
||||
return ThematicRasterMetric(
|
||||
metric_key=key,
|
||||
metric_label=label,
|
||||
metric_value=round(float(value), 4),
|
||||
metric_unit=unit,
|
||||
aggregation_method=method,
|
||||
is_estimate=estimate,
|
||||
)
|
||||
|
||||
if product.metric_kind == "binary_area":
|
||||
positive_count = int(np.count_nonzero(values >= 0.5))
|
||||
positive_area_ha = positive_count * cell_area_m2 / 10_000.0
|
||||
positive_share = positive_count / max(1, valid_cell_count) * 100.0
|
||||
label = "Ruimtebeslag" if product.theme == "space_occupation" else "Open ruimte"
|
||||
metrics = [
|
||||
metric(f"{product.theme}_area_ha", f"{label} in selectie", positive_area_ha, "ha", "positive_source_cells_times_cell_area"),
|
||||
metric(f"{product.theme}_share_pct", f"Aandeel {label.lower()}", positive_share, "%", "positive_source_cells_divided_by_valid_selected_cells"),
|
||||
metric("valid_raster_area_ha", "Rasteroppervlakte met bronwaarde", valid_cell_count * cell_area_m2 / 10_000.0, "ha", "valid_selected_cells_times_cell_area"),
|
||||
]
|
||||
elif product.metric_kind == "population_density":
|
||||
estimated_population = float(values.sum() * (cell_area_m2 / 10_000.0))
|
||||
metrics = [
|
||||
metric("estimated_inhabitants", "Geraamd aantal inwoners (2019)", estimated_population, "inwoners", "sum_density_times_selected_cell_area_hectares"),
|
||||
metric("population_density_mean_per_ha", "Gemiddelde inwonersdichtheid", values.mean(), "inwoners/ha", "mean_valid_one_hectare_source_cells"),
|
||||
metric("population_density_p90_per_ha", "90e percentiel inwonersdichtheid", np.percentile(values, 90), "inwoners/ha", "percentile_90_valid_source_cells"),
|
||||
]
|
||||
else:
|
||||
unit = "score" if product.metric_kind == "index_score" else "score (0-1)"
|
||||
label = "Knooppuntwaarde" if product.metric_kind == "index_score" else "Voorzieningenniveau"
|
||||
metrics = [
|
||||
metric(f"{product.theme}_mean", f"Gemiddelde {label.lower()}", values.mean(), unit, "mean_valid_source_cells"),
|
||||
metric(f"{product.theme}_p10", f"10e percentiel {label.lower()}", np.percentile(values, 10), unit, "percentile_10_valid_source_cells"),
|
||||
metric(f"{product.theme}_median", f"Mediaan {label.lower()}", np.percentile(values, 50), unit, "median_valid_source_cells"),
|
||||
metric(f"{product.theme}_p90", f"90e percentiel {label.lower()}", np.percentile(values, 90), unit, "percentile_90_valid_source_cells"),
|
||||
]
|
||||
|
||||
primary = metrics[0]
|
||||
response = ThematicRasterSelectionResponse(
|
||||
dataset_id=dataset.id,
|
||||
product_key=product.key,
|
||||
theme=product.theme,
|
||||
metric_kind=product.metric_kind,
|
||||
selection_bbox=payload.bbox,
|
||||
selection_area_id=payload.area_id,
|
||||
selected_cell_count=selected_cell_count,
|
||||
valid_cell_count=valid_cell_count,
|
||||
coverage_ratio=round(valid_cell_count / max(1, selected_cell_count), 6),
|
||||
resolution_m=round(max(resolution_x, resolution_y), 4),
|
||||
observation_year=product.observation_year,
|
||||
summary=ThematicRasterSelectionSummary(
|
||||
metric_label=primary.metric_label,
|
||||
metric_value=primary.metric_value,
|
||||
metric_unit=primary.metric_unit,
|
||||
aggregation_method=primary.aggregation_method,
|
||||
primary_metric_key=primary.metric_key,
|
||||
metrics=metrics,
|
||||
),
|
||||
unsupported_metrics=ThematicRasterAnalysisService._unsupported_metrics(product),
|
||||
limitation_message=product.limitation_message,
|
||||
generated_at=datetime.now(UTC).isoformat(),
|
||||
)
|
||||
return response.model_dump(mode="json")
|
||||
|
||||
@staticmethod
|
||||
def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes:
|
||||
dataset = ThematicRasterAnalysisService._load_dataset(db, project_id, dataset_id)
|
||||
product = ThematicRasterAnalysisService._product(dataset)
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from PIL import Image
|
||||
from rasterio.enums import Resampling
|
||||
except ImportError as exc:
|
||||
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio, numpy and Pillow are required for thematic raster rendering", status_code=503) from exc
|
||||
palettes = {
|
||||
"space_occupation": np.asarray([[251, 231, 211], [190, 62, 51]], dtype="float64"),
|
||||
"open_space": np.asarray([[221, 238, 219], [38, 122, 70]], dtype="float64"),
|
||||
"population": np.asarray([[238, 231, 246], [103, 58, 151]], dtype="float64"),
|
||||
"accessibility": np.asarray([[233, 241, 244], [15, 118, 110]], dtype="float64"),
|
||||
"services": np.asarray([[255, 244, 191], [182, 109, 22]], dtype="float64"),
|
||||
}
|
||||
try:
|
||||
with rasterio.open(dataset.storage_path) as source:
|
||||
scale = min(1.0, max_dimension / max(source.width, source.height))
|
||||
width = max(1, round(source.width * scale))
|
||||
height = max(1, round(source.height * scale))
|
||||
resampling = Resampling.nearest if product.metric_kind == "binary_area" else Resampling.bilinear
|
||||
data = source.read(1, out_shape=(height, width), masked=True, resampling=resampling)
|
||||
values = np.asarray(data.filled(np.nan), dtype="float64")
|
||||
valid = np.isfinite(values) & ~np.ma.getmaskarray(data)
|
||||
if source.nodata is not None:
|
||||
valid &= ~np.isclose(values, float(source.nodata))
|
||||
if product.metric_kind == "binary_area":
|
||||
valid &= values >= 0.5
|
||||
normalized = np.where(valid, 1.0, 0.0)
|
||||
else:
|
||||
source_metadata = dataset.source_metadata or {}
|
||||
lower = float(source_metadata.get("render_min_value", np.nanpercentile(values[valid], 2) if valid.any() else 0.0))
|
||||
upper = float(source_metadata.get("render_max_value", np.nanpercentile(values[valid], 98) if valid.any() else 1.0))
|
||||
if upper <= lower:
|
||||
upper = lower + 1.0
|
||||
normalized = np.clip((values - lower) / (upper - lower), 0.0, 1.0)
|
||||
colors = palettes[product.theme]
|
||||
rgba = np.zeros((height, width, 4), dtype="uint8")
|
||||
for channel in range(3):
|
||||
rgba[:, :, channel] = (colors[0, channel] + normalized * (colors[1, channel] - colors[0, channel])).astype("uint8")
|
||||
rgba[:, :, 3] = np.where(valid, 205, 0).astype("uint8")
|
||||
output = io.BytesIO()
|
||||
Image.fromarray(rgba).save(output, format="PNG", optimize=True)
|
||||
return output.getvalue()
|
||||
except AppError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AppError(
|
||||
code="THEMATIC_RASTER_PREVIEW_FAILED",
|
||||
message="The persisted thematic raster could not be rendered",
|
||||
details={"reason": str(exc)},
|
||||
status_code=500,
|
||||
) from exc
|
||||
@@ -29,6 +29,7 @@ FULL_AREA_CLIPPED_OPERATOR_TOOLS = {
|
||||
"provision_regional_bwk_natura2000.py",
|
||||
"provision_agricultural_parcel_history.py",
|
||||
"provision_buildings_addresses_register.py",
|
||||
"provision_mol_soil_map.py",
|
||||
}
|
||||
|
||||
SEMANTIC_METRICS_DISABLED_OPERATOR_TOOLS = {
|
||||
@@ -101,6 +102,16 @@ SEMANTIC_SELECTION_METRICS: dict[str, tuple[dict[str, Any], ...]] = {
|
||||
),
|
||||
"nature_value": (),
|
||||
"agriculture": (),
|
||||
"soil": (
|
||||
{
|
||||
"metric_key": "soil_mapped_area",
|
||||
"method": "intersection_area",
|
||||
"label": "Bodemkaartoppervlakte",
|
||||
"unit": "ha",
|
||||
"geometry_dimension": 2,
|
||||
"warning": "Historische bodemkartering op schaal 1:20.000; actuele lokale bodem- en drainagetoestand kan afwijken.",
|
||||
},
|
||||
),
|
||||
}
|
||||
|
||||
SEMANTIC_COUNT_LABELS = {
|
||||
@@ -112,6 +123,7 @@ SEMANTIC_COUNT_LABELS = {
|
||||
"parcels": "Percelen",
|
||||
"nature_value": "BWK-kaartvlakken",
|
||||
"agriculture": "Landbouwgebruikspercelen",
|
||||
"soil": "Bodemkaartvlakken",
|
||||
}
|
||||
|
||||
# Sprint 205 initially normalized two official comma-separated ALZ group labels
|
||||
@@ -158,6 +170,8 @@ class VectorFeatureService:
|
||||
"landbouw": "agriculture",
|
||||
"landbouwgebruik": "agriculture",
|
||||
"building_registry": "buildings",
|
||||
"soil_map": "soil",
|
||||
"bodem": "soil",
|
||||
}
|
||||
for candidate in candidates:
|
||||
if not isinstance(candidate, str) or not candidate.strip():
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import rasterio
|
||||
from fastapi.testclient import TestClient
|
||||
from pyproj import Transformer
|
||||
from rasterio.io import MemoryFile
|
||||
from rasterio.transform import from_origin
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models import Dataset, Job, Project
|
||||
from app.schemas.thematic_raster import ThematicRasterAcquireRequest, ThematicRasterSelectionRequest
|
||||
from app.schemas.assistant import AssistantQueryRequest
|
||||
from app.services.geo_assistant_service import GeoAssistantService
|
||||
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
||||
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
||||
from app.services.dataset_service import DatasetService
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, result=None):
|
||||
self.result = result
|
||||
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
|
||||
def order_by(self, *_args):
|
||||
return self
|
||||
|
||||
def first(self):
|
||||
return self.result
|
||||
|
||||
def all(self):
|
||||
return self.result if isinstance(self.result, list) else []
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, rows=None, query_result=None):
|
||||
self.rows = rows or {}
|
||||
self.query_result = query_result
|
||||
self.added = []
|
||||
|
||||
def get(self, model, row_id):
|
||||
row = self.rows.get((model, row_id))
|
||||
if row is not None:
|
||||
return row
|
||||
return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
|
||||
|
||||
def add(self, row):
|
||||
self.added.append(row)
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
def rollback(self):
|
||||
return None
|
||||
|
||||
def refresh(self, row):
|
||||
return row
|
||||
|
||||
def query(self, _model):
|
||||
return FakeQuery(self.query_result)
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, content: bytes):
|
||||
self.content = content
|
||||
self.headers = {"Content-Type": "image/tiff", "Content-Length": str(len(content))}
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def read(self, limit: int):
|
||||
return self.content[:limit]
|
||||
|
||||
|
||||
def payload(product_key: str = "space_occupation_2025", *, side_m: float = 1000.0) -> ThematicRasterAcquireRequest:
|
||||
transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
||||
min_x, min_y = transformer.transform(200_000, 210_000)
|
||||
max_x, max_y = transformer.transform(200_000 + side_m, 210_000 + side_m)
|
||||
return ThematicRasterAcquireRequest(
|
||||
bbox={"min_x": min_x, "min_y": min_y, "max_x": max_x, "max_y": max_y, "crs": "EPSG:4326"},
|
||||
product_key=product_key,
|
||||
force_refresh=True,
|
||||
)
|
||||
|
||||
|
||||
def raster_bytes(values: np.ndarray, resolution: float, *, nodata: float = -9999.0) -> bytes:
|
||||
with MemoryFile() as memory:
|
||||
with memory.open(
|
||||
driver="GTiff",
|
||||
width=values.shape[1],
|
||||
height=values.shape[0],
|
||||
count=1,
|
||||
dtype=str(values.dtype),
|
||||
crs="EPSG:31370",
|
||||
transform=from_origin(200_000, 210_000 + values.shape[0] * resolution, resolution, resolution),
|
||||
nodata=nodata,
|
||||
) as output:
|
||||
output.write(values, 1)
|
||||
return memory.read()
|
||||
|
||||
|
||||
def test_registry_contains_five_governed_non_water_policy_products() -> None:
|
||||
products = ThematicRasterAcquisitionService.list_products()
|
||||
|
||||
assert [item["key"] for item in products] == [
|
||||
"space_occupation_2025",
|
||||
"open_space_2022",
|
||||
"population_density_2019",
|
||||
"node_value_2022",
|
||||
"service_level_2022",
|
||||
]
|
||||
assert {item["theme"] for item in products} == {"space_occupation", "open_space", "population", "accessibility", "services"}
|
||||
assert {item["native_resolution_m"] for item in products} == {10.0, 100.0}
|
||||
assert all(item["coverage_id"].startswith(("lu:", "ni:")) for item in products)
|
||||
assert all(item["source_crs"] == "EPSG:31370" for item in products)
|
||||
assert all(item["attribution"] and item["license_note"] and item["limitation_message"] for item in products)
|
||||
|
||||
|
||||
def test_request_is_bounded_allowlisted_and_uses_native_wcs_resolution() -> None:
|
||||
settings = Settings(_env_file=None)
|
||||
prepared = ThematicRasterAcquisitionService._prepared_request(payload("population_density_2019"), settings)
|
||||
url = ThematicRasterAcquisitionService._wcs_request_url(settings, prepared["product"], tuple(prepared["bbox_epsg31370"]))
|
||||
|
||||
assert "VERSION=1.0.0" in url
|
||||
assert "COVERAGE=ni%3Ani_inw_ha_vlaa_2019" in url
|
||||
assert "RESX=100" in url and "RESY=100" in url
|
||||
assert prepared["width"] * prepared["height"] <= settings.thematic_raster_max_pixels
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
ThematicRasterAcquisitionService._prepared_request(payload("arbitrary_remote_layer"), settings)
|
||||
assert exc_info.value.code == "THEMATIC_RASTER_PRODUCT_NOT_SUPPORTED"
|
||||
|
||||
|
||||
def test_binary_and_normalized_products_fail_closed_on_invalid_values() -> None:
|
||||
binary = ThematicRasterAcquisitionService._product("space_occupation_2025")
|
||||
score = ThematicRasterAcquisitionService._product("service_level_2022")
|
||||
|
||||
with pytest.raises(AppError, match="Binary"):
|
||||
ThematicRasterAcquisitionService._validate_values(np.asarray([0.0, 2.0]), binary)
|
||||
with pytest.raises(AppError, match="0-1"):
|
||||
ThematicRasterAcquisitionService._validate_values(np.asarray([0.2, 1.2]), score)
|
||||
|
||||
|
||||
def test_acquisition_clips_validates_and_delegates_persistence(monkeypatch) -> None:
|
||||
project_id, output_dataset_id = uuid4(), uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
|
||||
content = raster_bytes(np.ones((100, 100), dtype="float32"), 10.0)
|
||||
captured: dict = {}
|
||||
|
||||
def fake_import(_db, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return SimpleNamespace(id=output_dataset_id)
|
||||
|
||||
monkeypatch.setattr(DatasetService, "import_raster_bytes", fake_import)
|
||||
result = ThematicRasterAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
payload(side_m=1000.0),
|
||||
settings=Settings(_env_file=None),
|
||||
opener=lambda *_args, **_kwargs: FakeResponse(content),
|
||||
)
|
||||
|
||||
assert result["output_dataset_id"] == str(output_dataset_id)
|
||||
assert captured["source_name"] == ThematicRasterAcquisitionService.PROVIDER
|
||||
assert captured["source_metadata"]["product_key"] == "space_occupation_2025"
|
||||
assert captured["source_metadata"]["metric_kind"] == "binary_area"
|
||||
assert captured["source_metadata"]["valid_pixel_count"] > 9_800
|
||||
assert captured["provenance_metadata"]["acquisition"] == "explicit_bounded_tiled_wcs_coverage"
|
||||
assert len(captured["provenance_metadata"]["normalized_sha256"]) == 64
|
||||
|
||||
|
||||
def test_binary_area_analysis_returns_hectares_and_share(tmp_path) -> None:
|
||||
project_id, dataset_id = uuid4(), uuid4()
|
||||
values = np.zeros((10, 10), dtype="float32")
|
||||
values[:, :5] = 1.0
|
||||
path = tmp_path / "space.tif"
|
||||
path.write_bytes(raster_bytes(values, 10.0))
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="space.tif",
|
||||
dataset_type="raster",
|
||||
source="official",
|
||||
source_name=ThematicRasterAcquisitionService.PROVIDER,
|
||||
source_metadata={"product_key": "space_occupation_2025", "coverage_id": "lu:lu_ruibes_vlaa_2025_v3"},
|
||||
status="ready",
|
||||
storage_path=str(path),
|
||||
)
|
||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||
result = ThematicRasterAnalysisService.analyze(
|
||||
db,
|
||||
project_id,
|
||||
dataset_id,
|
||||
ThematicRasterSelectionRequest(bbox=payload(side_m=100.0).bbox),
|
||||
)
|
||||
metrics = {item["metric_key"]: item for item in result["summary"]["metrics"]}
|
||||
|
||||
assert result["valid_cell_count"] == 100
|
||||
assert metrics["space_occupation_area_ha"]["metric_value"] == pytest.approx(0.5)
|
||||
assert metrics["space_occupation_share_pct"]["metric_value"] == pytest.approx(50.0)
|
||||
assert "object_count" in result["unsupported_metrics"]
|
||||
|
||||
|
||||
def test_population_analysis_sums_one_hectare_density_cells_without_claiming_current_counts(tmp_path) -> None:
|
||||
project_id, dataset_id = uuid4(), uuid4()
|
||||
values = np.asarray([[10.0, 20.0], [30.0, 40.0]], dtype="float32")
|
||||
path = tmp_path / "population.tif"
|
||||
path.write_bytes(raster_bytes(values, 100.0))
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="population.tif",
|
||||
dataset_type="raster",
|
||||
source="official",
|
||||
source_name=ThematicRasterAcquisitionService.PROVIDER,
|
||||
source_metadata={"product_key": "population_density_2019", "coverage_id": "ni:ni_inw_ha_vlaa_2019"},
|
||||
status="ready",
|
||||
storage_path=str(path),
|
||||
)
|
||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||
result = ThematicRasterAnalysisService.analyze(
|
||||
db,
|
||||
project_id,
|
||||
dataset_id,
|
||||
ThematicRasterSelectionRequest(bbox=payload("population_density_2019", side_m=200.0).bbox),
|
||||
)
|
||||
metrics = {item["metric_key"]: item for item in result["summary"]["metrics"]}
|
||||
|
||||
assert metrics["estimated_inhabitants"]["metric_value"] == pytest.approx(100.0)
|
||||
assert metrics["population_density_mean_per_ha"]["metric_value"] == pytest.approx(25.0)
|
||||
assert metrics["estimated_inhabitants"]["is_estimate"] is True
|
||||
assert "current_population" in result["unsupported_metrics"]
|
||||
|
||||
|
||||
def test_assistant_context_receives_persisted_thematic_metrics(tmp_path) -> None:
|
||||
project_id, dataset_id = uuid4(), uuid4()
|
||||
values = np.asarray([[10.0, 20.0], [30.0, 40.0]], dtype="float32")
|
||||
path = tmp_path / "assistant-population.tif"
|
||||
path.write_bytes(raster_bytes(values, 100.0))
|
||||
project = Project(id=project_id, name="Mol", region="Mol")
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="population.tif",
|
||||
dataset_type="raster",
|
||||
source="official",
|
||||
source_name=ThematicRasterAcquisitionService.PROVIDER,
|
||||
source_metadata={"product_key": "population_density_2019", "coverage_id": "ni:ni_inw_ha_vlaa_2019"},
|
||||
status="ready",
|
||||
storage_path=str(path),
|
||||
)
|
||||
db = FakeSession({(Project, project_id): project, (Dataset, dataset_id): dataset}, query_result=[dataset])
|
||||
context, metrics, _series, dataset_ids, _warnings, _scope = GeoAssistantService(Settings(_env_file=None))._build_context(
|
||||
db,
|
||||
project_id=project_id,
|
||||
payload=AssistantQueryRequest(question="Hoeveel inwoners?", bbox=payload("population_density_2019", side_m=200.0).bbox),
|
||||
)
|
||||
|
||||
assert any(metric.theme == "population" and metric.label.startswith("Geraamd aantal") for metric in metrics)
|
||||
assert dataset_id in dataset_ids
|
||||
assert context["rules"]["thematic_policy_rasters_available"] is True
|
||||
|
||||
|
||||
def test_index_renderer_returns_browser_png(tmp_path) -> None:
|
||||
project_id, dataset_id = uuid4(), uuid4()
|
||||
values = np.linspace(0.1, 4.0, 100, dtype="float32").reshape((10, 10))
|
||||
path = tmp_path / "node.tif"
|
||||
path.write_bytes(raster_bytes(values, 100.0))
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="node.tif",
|
||||
dataset_type="raster",
|
||||
source="official",
|
||||
source_name=ThematicRasterAcquisitionService.PROVIDER,
|
||||
source_metadata={
|
||||
"product_key": "node_value_2022",
|
||||
"coverage_id": "lu:lu_knptw_ha_2022_v3",
|
||||
"render_min_value": 0.1,
|
||||
"render_max_value": 4.0,
|
||||
},
|
||||
status="ready",
|
||||
storage_path=str(path),
|
||||
)
|
||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||
|
||||
assert ThematicRasterAnalysisService.render_png(db, project_id, dataset_id).startswith(b"\x89PNG\r\n\x1a\n")
|
||||
|
||||
|
||||
def test_api_uses_canonical_envelopes(monkeypatch) -> None:
|
||||
project_id, dataset_id = uuid4(), uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
|
||||
monkeypatch.setattr(
|
||||
ThematicRasterAcquisitionService,
|
||||
"acquire",
|
||||
lambda *_args, **_kwargs: {"output_dataset_id": str(dataset_id), "provider": ThematicRasterAcquisitionService.PROVIDER},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
ThematicRasterAnalysisService,
|
||||
"analyze",
|
||||
lambda *_args, **_kwargs: {"dataset_id": str(dataset_id), "theme": "population", "summary": {"metric_value": 10.0}},
|
||||
)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
try:
|
||||
client = TestClient(app)
|
||||
products = client.get(f"/api/v1/projects/{project_id}/datasets/thematic-raster/products")
|
||||
acquisition = client.post(
|
||||
f"/api/v1/projects/{project_id}/datasets/thematic-raster/acquire",
|
||||
json=payload().model_dump(mode="json"),
|
||||
)
|
||||
selection = client.post(
|
||||
f"/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/thematic/select",
|
||||
json={"bbox": payload().bbox.model_dump()},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert products.status_code == 200 and set(products.json()) == {"data"}
|
||||
assert products.json()["data"]["total"] == 5
|
||||
assert acquisition.status_code == 200 and set(acquisition.json()) == {"data"}
|
||||
assert acquisition.json()["data"]["job_type"] == "raster.thematic.acquire"
|
||||
assert selection.status_code == 200 and selection.json()["data"]["theme"] == "population"
|
||||
assert any(isinstance(item, Job) for item in db.added)
|
||||
@@ -0,0 +1,183 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box, mapping, shape
|
||||
from shapely.ops import transform as transform_geometry
|
||||
|
||||
from app.models import Dataset
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def load_operator():
|
||||
path = ROOT / "scripts" / "provision_mol_soil_map.py"
|
||||
spec = importlib.util.spec_from_file_location("dov_soil_map_operator", path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload: dict, url: str):
|
||||
self._payload = payload
|
||||
self.url = url
|
||||
self.content = b'{"type":"FeatureCollection"}'
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict:
|
||||
return self._payload
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, pages: list[dict]):
|
||||
self.pages = pages
|
||||
self.calls: list[dict] = []
|
||||
|
||||
def get(self, _url: str, *, params: dict, timeout: int):
|
||||
self.calls.append({"params": dict(params), "timeout": timeout})
|
||||
return FakeResponse(self.pages[len(self.calls) - 1], f"https://example.test/page/{len(self.calls)}")
|
||||
|
||||
|
||||
def soil_feature(module, feature_id: str = "bodemtypes.1") -> dict:
|
||||
return {
|
||||
"type": "Feature",
|
||||
"id": feature_id,
|
||||
"geometry": mapping(box(5.0, 51.0, 5.02, 51.02)),
|
||||
"properties": {
|
||||
"gid": 1,
|
||||
"id_kaartvlak": 10,
|
||||
"Bodemtype": "Zeg",
|
||||
"Unibodemtype": "Zeg",
|
||||
"Bodemserie": "Zeg",
|
||||
"Beknopte_omschrijving_bodemserie": "Natte zandbodem",
|
||||
"Gegeneraliseerde_legende": "Nat zand",
|
||||
"Textuurklasse_code": "Z",
|
||||
"Textuurklasse": "zand",
|
||||
"Drainageklasse_code": "e",
|
||||
"Drainageklasse": "nat",
|
||||
"Profielontwikkelingsgroep_code": "g",
|
||||
"Profielontwikkelingsgroep": "humus B horizont",
|
||||
"Eenduidige_legende_titel": "bodemserie Zeg",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_wfs_pagination_is_bounded_complete_and_deterministic() -> None:
|
||||
module = load_operator()
|
||||
feature = soil_feature(module)
|
||||
pages = [
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"numberMatched": 3,
|
||||
"numberReturned": 2,
|
||||
"features": [feature, {**feature, "id": "bodemtypes.2"}],
|
||||
},
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"numberMatched": 3,
|
||||
"numberReturned": 1,
|
||||
"features": [{**feature, "id": "bodemtypes.3"}],
|
||||
},
|
||||
]
|
||||
session = FakeSession(pages)
|
||||
|
||||
result = list(
|
||||
module.iter_wfs_pages(
|
||||
session,
|
||||
(196000.0, 205000.0, 211000.0, 224000.0),
|
||||
page_limit=2,
|
||||
timeout=30,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(result) == 2
|
||||
assert [call["params"]["startIndex"] for call in session.calls] == ["0", "2"]
|
||||
assert all(call["params"]["typeNames"] == "bodemkaart:bodemtypes" for call in session.calls)
|
||||
assert all(call["params"]["bbox"].endswith("EPSG:31370") for call in session.calls)
|
||||
assert all(call["params"]["sortBy"] == "gid" for call in session.calls)
|
||||
|
||||
|
||||
def test_soil_feature_is_exactly_clipped_and_keeps_governed_properties() -> None:
|
||||
module = load_operator()
|
||||
boundary_wgs84 = box(5.005, 51.005, 5.015, 51.015)
|
||||
boundary_lambert72 = transform_geometry(module.TO_LAMBERT72.transform, boundary_wgs84)
|
||||
|
||||
normalized, was_clipped = module.normalize_feature(soil_feature(module), boundary_lambert72)
|
||||
|
||||
assert normalized is not None and was_clipped is True
|
||||
persisted_geometry = shape(normalized["geometry"])
|
||||
assert persisted_geometry.within(boundary_wgs84.buffer(1e-7))
|
||||
properties = normalized["properties"]
|
||||
assert properties["source_name"] == "dov_soil_map"
|
||||
assert properties["soil_texture_class"] == "zand"
|
||||
assert properties["soil_drainage_class"] == "nat"
|
||||
assert properties["survey_period"] == "1949-1971"
|
||||
assert properties["clipped_area_ha"] > 0
|
||||
assert "may differ today" in properties["historical_drainage_limitation"]
|
||||
|
||||
|
||||
def test_soil_map_uses_existing_semantic_selection_architecture() -> None:
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="dov_soil_map_mol.geojson",
|
||||
dataset_type="vector",
|
||||
source="operator_official_import",
|
||||
source_name="dov_soil_map",
|
||||
reference_layer_name="soil",
|
||||
source_metadata={
|
||||
"theme": "soil",
|
||||
"selection_aggregation": {
|
||||
"method": "intersection_area",
|
||||
"label": "Bodemkaartoppervlakte",
|
||||
"unit": "ha",
|
||||
},
|
||||
},
|
||||
status="ready",
|
||||
)
|
||||
|
||||
assert VectorFeatureService._dataset_theme(dataset) == "soil"
|
||||
assert VectorFeatureService.supports_selection_summary(dataset) is True
|
||||
assert VectorFeatureService.can_use_full_area_fast_path(dataset, None) is False
|
||||
|
||||
|
||||
def test_soil_operator_contract_has_no_direct_persistence_and_is_packaged() -> None:
|
||||
operator = (ROOT / "scripts" / "provision_mol_soil_map.py").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
map_workspace = (ROOT / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "/datasets/upload" in operator
|
||||
assert "vector_features" in operator
|
||||
assert "does not write directly" in " ".join(operator.split())
|
||||
assert "SessionLocal" not in operator and "INSERT INTO" not in operator
|
||||
assert "COPY scripts/provision_mol_soil_map.py" in dockerfile
|
||||
assert "py_compile scripts/provision_mol_soil_map.py" in readiness
|
||||
assert "id: 'soil'" in map_workspace
|
||||
assert "dataset.source_name === 'dov_soil_map'" in map_workspace
|
||||
|
||||
|
||||
def test_incomplete_wfs_pagination_fails_closed() -> None:
|
||||
module = load_operator()
|
||||
session = FakeSession(
|
||||
[
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"numberMatched": 2,
|
||||
"numberReturned": 0,
|
||||
"features": [],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="returned 0 of 2"):
|
||||
list(module.iter_wfs_pages(session, (0.0, 0.0, 1.0, 1.0), page_limit=100, timeout=30))
|
||||
Reference in New Issue
Block a user