feat: complete governed Walloon coverage sources
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-22 03:39:08 +02:00
parent 17d4442e60
commit c2101ea8f9
37 changed files with 1813 additions and 45 deletions
+34
View File
@@ -1599,6 +1599,40 @@ Datasets. A full-Flanders raster request remains blocked by the same 60 km and
`Gemeente ...` as `coverage_scope=municipality`; regional Area clipping is
stored as `bounded_selection`.
## Walloon WALOUS land cover and flood hazard
The Wallonia map flow uses bounded PICC vector products, the queryable legal
SPW flood-hazard polygon layer and provisioned official WALOUS land-cover
rasters. Provision the 2020 and 2023 source editions once in the persistent
storage mount:
```bash
docker exec geointel python /app/scripts/provision_walous_sources.py \
--years 2020 2023 \
--destination /app/storage/source-cache/walous
```
The provisioner verifies advertised archive sizes, safe ZIP structure,
EPSG:3812, one band, 1 m cells, class values 1-11 and SHA-256 checksums. It
does not run at application startup. `GET .../datasets/walous/products`
therefore reports `source_not_provisioned` until both source files exist.
For a bounded Walloon selection the browser persists the latest edition and
all other configured comparable editions. `POST .../raster/walous/select`
returns cell-area hectares; the temporal API compares the same semantic metric
keys for 2020 and 2023. WALOUS is land cover, not legal land use, ownership,
tree count, timber volume or water volume.
Settings: `WALOUS_ENABLED`, `WALOUS_SOURCE_DIR`,
`WALOUS_ANALYSIS_RESOLUTION_M`, `WALOUS_MAX_SIDE_M` and
`WALOUS_MAX_PIXELS`. The SPW flood polygon adapter uses
`SPW_FLOOD_HAZARD_ENABLED` and `SPW_FLOOD_HAZARD_MAPSERVER_URL`.
The official Walloon 2021-2022 DTM is currently not an implicit runtime asset:
the published 1 m whole-region artifact is about 41 GB and the 0.5 m INSPIRE
artifact about 213 GB. A later operator capacity plan must define storage,
partitioning and refresh before it can be called operational.
Provision the official DOV soil polygons for Mol through the existing vector
upload path:
+55
View File
@@ -97,6 +97,7 @@ from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisi
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
from app.services.walous_land_cover_service import WalousLandCoverService
from app.utils.response import envelope
router = APIRouter(prefix="/projects/{project_id}", tags=["datasets"])
@@ -459,6 +460,33 @@ def list_thematic_raster_products(project_id: UUID, db: Session = Depends(get_db
return envelope({"items": items, "total": len(items)})
@router.post("/datasets/walous/acquire", response_model=Envelope[JobRead])
def acquire_bounded_walous_land_cover(
project_id: UUID,
payload: ThematicRasterAcquireRequest,
db: Session = Depends(get_db),
):
job = JobService.run_sync_job(
db=db,
project_id=project_id,
job_type="raster.walous.acquire",
parameters=payload.model_dump(mode="json"),
operation=lambda: WalousLandCoverService.acquire(db, project_id, payload),
)
return envelope(job)
@router.get(
"/datasets/walous/products",
response_model=Envelope[ItemList[ThematicRasterProductRead]],
)
def list_walous_products(project_id: UUID, db: Session = Depends(get_db)):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
items = WalousLandCoverService.list_products()
return envelope({"items": items, "total": len(items)})
@router.get("/datasets", response_model=Envelope[DatasetList])
def list_datasets(
project_id: UUID,
@@ -1006,6 +1034,33 @@ def raster_thematic_image(
)
@router.post(
"/datasets/{dataset_id}/raster/walous/select",
response_model=Envelope[ThematicRasterSelectionResponse],
)
def raster_walous_selection(
project_id: UUID,
dataset_id: UUID,
payload: ThematicRasterSelectionRequest,
db: Session = Depends(get_db),
):
return envelope(WalousLandCoverService.analyze(db, project_id, dataset_id, payload))
@router.get("/datasets/{dataset_id}/raster/walous/image")
def raster_walous_image(
project_id: UUID,
dataset_id: UUID,
db: Session = Depends(get_db),
):
content = WalousLandCoverService.render_png(db, project_id, dataset_id)
return Response(
content=content,
media_type="image/png",
headers={"Cache-Control": "private, max-age=86400"},
)
@router.get(
"/datasets/{dataset_id}/raster/stats",
response_model=Envelope[RasterStatsResponse],
+21
View File
@@ -160,6 +160,14 @@ class Settings(BaseSettings):
),
validation_alias="SPW_PICC_MAPSERVER_URL",
)
spw_flood_hazard_enabled: bool = Field(default=True, validation_alias="SPW_FLOOD_HAZARD_ENABLED")
spw_flood_hazard_mapserver_url: str = Field(
default=(
"https://geoservices.wallonie.be/arcgis/rest/services/"
"EAU/ALEA_INOND/MapServer"
),
validation_alias="SPW_FLOOD_HAZARD_MAPSERVER_URL",
)
urbis_enabled: bool = Field(default=True, validation_alias="URBIS_ENABLED")
urbis_wfs_url: str = Field(
default="https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows",
@@ -273,6 +281,19 @@ class Settings(BaseSettings):
thematic_raster_max_pixels: int = Field(default=30_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS")
thematic_raster_timeout_seconds: int = Field(default=300, ge=1, validation_alias="THEMATIC_RASTER_TIMEOUT_SECONDS")
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
walous_enabled: bool = Field(default=True, validation_alias="WALOUS_ENABLED")
walous_source_dir: str = Field(
default="/app/storage/source-cache/walous",
validation_alias="WALOUS_SOURCE_DIR",
)
walous_analysis_resolution_m: float = Field(
default=10.0,
ge=1.0,
le=100.0,
validation_alias="WALOUS_ANALYSIS_RESOLUTION_M",
)
walous_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="WALOUS_MAX_SIDE_M")
walous_max_pixels: int = Field(default=36_000_000, ge=1, validation_alias="WALOUS_MAX_PIXELS")
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL")
+25 -1
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
from uuid import UUID
from pydantic import BaseModel
from pydantic import BaseModel, Field
from .operations import VectorSelectionBBox
@@ -32,6 +32,30 @@ class ThematicRasterProductRead(BaseModel):
legend_max_label: str
included_source_values: list[int]
limitation_message: str
analysis_resolution_m: float | None = None
coverage_zones: list[str] = Field(default_factory=list)
configured: bool = True
status: str = "configured"
class WalousAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
product_key: str
display_name: str
theme: str
metric_kind: str
resolution_m: float
width: int
height: int
valid_pixel_count: int
bbox_epsg4326: list[float]
bbox_epsg3812: list[float]
observation_year: int
source_value_unit: str
attribution: str
limitation_message: str
class ThematicRasterAcquisitionResult(BaseModel):
@@ -252,11 +252,11 @@ SOURCE_DEFINITIONS = (
attribution="Service public de Wallonie",
license_note="Consult the license of each Geoportail Wallonie product.",
limitation_message=(
"Bounded PICC buildings, road axes, hydrography and operator-imported SPW bathymetry are operational; "
"other Walloon themes remain unavailable until separately governed."
"Bounded PICC buildings, road axes and hydrography, the legally current flood-hazard polygons, "
"and operator-imported SPW bathymetry are operational; other Walloon themes remain separately governed."
),
materialized_source_names=("spw_picc", "spw_bathymetry"),
operational_themes=("buildings", "roads", "surface_water", "bathymetry"),
materialized_source_names=("spw_picc", "spw_flood_hazard", "spw_walous_land_cover", "spw_bathymetry"),
operational_themes=("buildings", "roads", "surface_water", "land_cover_use", "flood_climate", "bathymetry"),
),
_contract(
source_name="urbis",
@@ -375,6 +375,8 @@ REGIONAL_THEME_DATASETS: dict[str, dict[str, dict[str, tuple[str, ...]]]] = {
"buildings": {"spw_picc": ("buildings",)},
"roads": {"spw_picc": ("roads",)},
"surface_water": {"spw_picc": ("water",)},
"land_cover_use": {"spw_walous_land_cover": ()},
"flood_climate": {"spw_flood_hazard": ("flood_hazard",)},
"bathymetry": {"spw_bathymetry": ()},
},
"urbis": {
@@ -460,6 +460,75 @@ class OfficialVectorAcquisitionService:
identity_field="GEOREF_ID",
requires_coverage_area=True,
),
OfficialVectorProduct(
key="spw_flood_hazard_2021",
display_name="Waalse overstromingsgevaarkaart 2021",
theme="flood_hazard",
provider="Service public de Wallonie",
source_name="spw_flood_hazard",
reference_layer_name="flood_hazard",
service_type="ArcGIS REST",
collection="2",
source_crs="EPSG:31370",
source_version="2021-03-04",
observation_label="Juridisch geldende toestand 2021",
authority_level="authoritative",
catalog_url=(
"https://geoportail.wallonie.be/catalogue/"
"14084108-2c7b-4091-b62d-ff0fc235213a.html"
),
attribution="Service public de Wallonie (SPW) - Cartographie de l'alea d'inondation",
license_note="CC BY 4.0; cite SPW and identify modifications.",
limitation_message=(
"Juridische gevarenkaart voor overstroming door waterloopoverloop en afstroming. "
"Dit is geen actuele overstroming, gemeten waterdiepte, voorspelling of bathymetrie."
),
source="SPW flood-hazard ArcGIS REST",
observed_at=datetime(2021, 3, 4, tzinfo=UTC),
valid_from=datetime(2021, 3, 4, tzinfo=UTC),
valid_to=None,
primary_metric={
"metric_key": "flood_hazard_area",
"method": "intersection_area",
"label": "Oppervlakte met overstromingsgevaar",
"unit": "ha",
"geometry_dimension": 2,
"is_estimate": False,
},
selection_metrics=(
{
"metric_key": "flood_hazard_high_area",
"method": "intersection_area",
"label": "Hoog overstromingsgevaar",
"unit": "ha",
"geometry_dimension": 2,
"filter_property": "CLASSEMENT",
"filter_values": [130, 230, 330, "130", "230", "330"],
},
{
"metric_key": "flood_hazard_medium_area",
"method": "intersection_area",
"label": "Middelgroot overstromingsgevaar",
"unit": "ha",
"geometry_dimension": 2,
"filter_property": "CLASSEMENT",
"filter_values": [120, 220, 320, "120", "220", "320"],
},
{
"metric_key": "flood_hazard_low_area",
"method": "intersection_area",
"label": "Laag overstromingsgevaar",
"unit": "ha",
"geometry_dimension": 2,
"filter_property": "CLASSEMENT",
"filter_values": [110, 210, 310, "110", "210", "310"],
},
),
coverage_zones=("wallonia",),
endpoint_kind="spw_flood_arcgis",
identity_field="LOCALID",
requires_coverage_area=True,
),
OfficialVectorProduct(
key="urbis_buildings",
display_name="UrbIS buildings",
@@ -869,6 +938,12 @@ class OfficialVectorAcquisitionService:
message="Bounded SPW PICC acquisition is disabled",
status_code=503,
)
if product.endpoint_kind == "spw_flood_arcgis" and not settings.spw_flood_hazard_enabled:
raise AppError(
code="SPW_FLOOD_HAZARD_NOT_CONFIGURED",
message="Bounded SPW flood-hazard acquisition is disabled",
status_code=503,
)
if product.endpoint_kind == "urbis_wfs" and not settings.urbis_enabled:
raise AppError(
code="URBIS_NOT_CONFIGURED",
@@ -1132,7 +1207,11 @@ class OfficialVectorAcquisitionService:
"f": "geojson",
}
)
base = settings.spw_picc_mapserver_url.rstrip("/")
base = (
settings.spw_flood_hazard_mapserver_url
if product.endpoint_kind == "spw_flood_arcgis"
else settings.spw_picc_mapserver_url
).rstrip("/")
return f"{base}/{product.collection}/query?{query}"
@staticmethod
@@ -1178,7 +1257,7 @@ class OfficialVectorAcquisitionService:
tuple(scope_metric.bounds),
start_index,
)
if product.endpoint_kind == "spw_arcgis":
if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis"}:
return OfficialVectorAcquisitionService._spw_url(
settings,
product,
@@ -1209,6 +1288,7 @@ class OfficialVectorAcquisitionService:
"bwk_wfs": settings.bwk_wfs_url,
"dov_wfs": settings.dov_soil_wfs_url,
"spw_arcgis": settings.spw_picc_mapserver_url,
"spw_flood_arcgis": settings.spw_flood_hazard_mapserver_url,
"urbis_wfs": settings.urbis_wfs_url,
}.get(product.endpoint_kind)
if configured_url is None:
@@ -1220,7 +1300,7 @@ class OfficialVectorAcquisitionService:
base = urlparse(configured_url)
expected_path = (
f"{base.path.rstrip('/')}/{product.collection}/query"
if product.endpoint_kind == "spw_arcgis"
if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis"}
else base.path
)
if (
@@ -1361,7 +1441,7 @@ class OfficialVectorAcquisitionService:
scope_metric: Any,
coverage_scope: str,
) -> dict[str, Any] | None:
if product.endpoint_kind in {"spw_arcgis", "urbis_wfs"}:
if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis", "urbis_wfs"}:
return OfficialVectorAcquisitionService._normalize_regional_feature(
product,
feature,
@@ -1601,7 +1681,7 @@ class OfficialVectorAcquisitionService:
)
start_index += returned_count
arcgis_has_more = payload.get("exceededTransferLimit") is True
if product.endpoint_kind == "spw_arcgis":
if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis"}:
if arcgis_has_more and returned_count == 0:
raise AppError(
code="OFFICIAL_VECTOR_PROVIDER_INCOMPLETE_RESPONSE",
@@ -22,7 +22,9 @@ from app.schemas.temporal import (
TemporalSeriesDataset,
TemporalSeriesRead,
)
from app.schemas.thematic_raster import ThematicRasterSelectionRequest
from app.services.vector_feature_service import VectorFeatureService
from app.services.walous_land_cover_service import WalousLandCoverService
class TemporalAnalysisService:
@@ -31,6 +33,7 @@ class TemporalAnalysisService:
"provision_regional_grb_buildings.py",
"provision_regional_grb_context.py",
}
SUPPORTED_RASTER_TEMPORAL_SOURCES = {WalousLandCoverService.PROVIDER}
@staticmethod
def _canonical_observation_snapshots(datasets: list[Dataset]) -> list[Dataset]:
@@ -141,6 +144,15 @@ class TemporalAnalysisService:
status_code=400,
)
if earlier.dataset_type == "raster" or later.dataset_type == "raster":
return TemporalAnalysisService._compare_walous_rasters(
db,
project_id=project_id,
payload=payload,
earlier=earlier,
later=later,
)
bbox = payload.bbox.model_dump()
selection_area = TemporalAnalysisService._get_selection_area(db, project_id, payload.area_id)
selection_geometry = None
@@ -259,6 +271,90 @@ class TemporalAnalysisService:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
return area
@staticmethod
def _compare_walous_rasters(
db: Session,
*,
project_id: UUID,
payload: TemporalComparisonRequest,
earlier: Dataset,
later: Dataset,
) -> TemporalComparisonResponse:
if {
earlier.dataset_type,
later.dataset_type,
} != {"raster"} or earlier.source_name != WalousLandCoverService.PROVIDER or later.source_name != WalousLandCoverService.PROVIDER:
raise AppError(
code="INCOMPATIBLE_TEMPORAL_DATASET_TYPES",
message="Raster evolution currently supports only two governed WALOUS land-cover snapshots",
status_code=400,
)
request = ThematicRasterSelectionRequest(bbox=payload.bbox, area_id=payload.area_id)
summaries: dict[UUID, dict[str, Any]] = {}
def summarize(dataset: Dataset) -> dict[str, Any]:
cached = summaries.get(dataset.id)
if cached is not None:
return cached
result = WalousLandCoverService.analyze(db, project_id, dataset.id, request)
summary = dict(result["summary"])
summary["warning"] = result.get("limitation_message")
summaries[dataset.id] = summary
return summary
earlier_summary = summarize(earlier)
later_summary = summarize(later)
metric_comparisons = TemporalAnalysisService._compare_summary_metrics(earlier_summary, later_summary)
if not metric_comparisons:
raise AppError(
code="INCOMPATIBLE_TEMPORAL_AGGREGATION",
message="WALOUS snapshots use incompatible aggregation semantics",
status_code=400,
)
primary_key = str(later_summary.get("primary_metric_key") or metric_comparisons[0].metric_key)
primary_metric = next(
(metric for metric in metric_comparisons if metric.metric_key == primary_key),
metric_comparisons[0],
)
timeline = TemporalAnalysisService._build_timeline(
db,
project_id=project_id,
series_key=str(earlier.temporal_series_key),
fallback_datasets=[earlier, later],
summarize=summarize,
)
warnings = [
"WALOUS-evolutie vergelijkt celgebaseerde landbedekkingsoppervlakten; individuele objectwijzigingen zijn niet beschikbaar.",
]
limitation = str(later_summary.get("warning") or earlier_summary.get("warning") or "").strip()
if limitation:
warnings.append(limitation)
return TemporalComparisonResponse(
temporal_series_key=str(earlier.temporal_series_key),
earlier=TemporalDatasetRef(
id=earlier.id,
name=earlier.name,
observed_at=earlier.observed_at,
source_version=earlier.source_version,
),
later=TemporalDatasetRef(
id=later.id,
name=later.name,
observed_at=later.observed_at,
source_version=later.source_version,
),
selection_bbox=payload.bbox,
selection_area_id=payload.area_id,
metric=primary_metric,
metrics=metric_comparisons,
timeline=timeline,
object_changes=TemporalObjectChanges(available=False),
geojson={"type": "FeatureCollection", "features": []},
warnings=warnings,
generated_at=datetime.now(timezone.utc),
)
@staticmethod
def _summary_metrics(summary: dict[str, Any]) -> list[dict[str, Any]]:
configured = summary.get("metrics")
@@ -370,10 +466,15 @@ class TemporalAnalysisService:
dataset = db.get(Dataset, dataset_id)
if not dataset or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message=f"{label} dataset not found", status_code=404)
if dataset.dataset_type not in {"vector", "geojson"}:
supported_vector = dataset.dataset_type in {"vector", "geojson"}
supported_raster = (
dataset.dataset_type == "raster"
and dataset.source_name in TemporalAnalysisService.SUPPORTED_RASTER_TEMPORAL_SOURCES
)
if not supported_vector and not supported_raster:
raise AppError(
code="DATASET_NOT_VECTOR",
message="Temporal selection comparison currently requires vector datasets",
code="TEMPORAL_DATASET_NOT_SUPPORTED",
message="Temporal comparison requires a vector series or a governed WALOUS raster series",
status_code=400,
)
if not dataset.temporal_series_key or not dataset.observed_at:
@@ -0,0 +1,551 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
import hashlib
import io
import json
import math
from pathlib import Path
from typing import Any
from uuid import UUID
from geoalchemy2.shape import to_shape
from pyproj import Transformer
from shapely.geometry import box, mapping
from shapely.ops import transform as shapely_transform
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import Area, Dataset, Project
from app.schemas.thematic_raster import (
ThematicRasterAcquireRequest,
ThematicRasterMetric,
ThematicRasterProductRead,
ThematicRasterSelectionRequest,
ThematicRasterSelectionResponse,
ThematicRasterSelectionSummary,
WalousAcquisitionResult,
)
from app.services.dataset_service import DatasetService
@dataclass(frozen=True)
class WalousProduct:
key: str
display_name: str
observation_year: int
source_filename: str
source_version: str
catalog_url: str
download_url: str
source_sha256_filename: str
accuracy_label: str
class WalousLandCoverService:
PROVIDER = "spw_walous_land_cover"
SOURCE_CRS = "EPSG:3812"
SOURCE_RESOLUTION_M = 1.0
SOURCE_VALUE_UNIT = "class_1_11"
THEME = "land_cover_use"
METRIC_KIND = "categorical_area"
NODATA = 255
ATTRIBUTION = "Service public de Wallonie (SPW), Aerospacelab S.A."
LICENSE_NOTE = "CC BY 4.0; cite the official SPW WALOUS edition and identify modifications."
LIMITATION = (
"GeoIntel analyseert een nearest-neighbour afgeleide van het officiele 1 m WALOUS-raster op de "
"geconfigureerde analyseresolutie. Oppervlakten zijn celgebaseerde schattingen; de kaart is landbedekking, "
"geen juridisch landgebruik, eigendom, boomtelling of actuele terreinwaarneming."
)
CLASS_LABELS = {
1: "Jaarlijks wisselende kruidlaag",
2: "Jaarronde kruidlaag",
3: "Naaldbomen hoger dan 3 m",
4: "Loofbomen hoger dan 3 m",
5: "Naaldbomen tot 3 m",
6: "Loofbomen tot 3 m",
7: "Kale bodem",
8: "Oppervlaktewater",
9: "Kunstmatige bodembedekking",
10: "Spoorweg",
11: "Kunstmatige constructies boven maaiveld",
}
CLASS_COLORS = {
1: (236, 202, 73),
2: (161, 201, 78),
3: (28, 89, 51),
4: (52, 132, 72),
5: (78, 125, 70),
6: (107, 164, 87),
7: (194, 165, 119),
8: (44, 129, 185),
9: (155, 155, 155),
10: (68, 68, 68),
11: (183, 72, 67),
}
@staticmethod
def _products() -> dict[str, WalousProduct]:
products = (
WalousProduct(
key="walous_land_cover_2020",
display_name="WALOUS landbedekking 2020",
observation_year=2020,
source_filename="walous_land_cover_2020_3812.tif",
source_version="WAL_OCS_IA__2020",
catalog_url="https://geoportail.wallonie.be/catalogue/47b348f1-6e7a-4baa-963c-0232a43c0cff.html",
download_url=(
"https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
"47b348f1-6e7a-4baa-963c-0232a43c0cff/WAL_OCS_IA__2020_GEOTIFF_3812.zip"
),
source_sha256_filename="walous_land_cover_2020_3812.sha256",
accuracy_label="Officiele globale nauwkeurigheid 83,30%",
),
WalousProduct(
key="walous_land_cover_2023",
display_name="WALOUS landbedekking 2023",
observation_year=2023,
source_filename="walous_land_cover_2023_3812.tif",
source_version="WAL_OCS_IA__2023",
catalog_url="https://geoportail.wallonie.be/catalogue/4e780ba1-463c-478e-95df-d2f1963a150d.html",
download_url=(
"https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
"4e780ba1-463c-478e-95df-d2f1963a150d/WAL_OCS_IA__2023_GEOTIFF_3812.zip"
),
source_sha256_filename="walous_land_cover_2023_3812.sha256",
accuracy_label="Officiele globale nauwkeurigheid 87,10%",
),
)
return {product.key: product for product in products}
@staticmethod
def _source_path(settings: Settings, product: WalousProduct) -> Path:
return Path(settings.walous_source_dir) / product.source_filename
@staticmethod
def list_products(*, settings: Settings | None = None) -> list[dict[str, Any]]:
resolved = settings or get_settings()
result: list[dict[str, Any]] = []
for product in WalousLandCoverService._products().values():
configured = resolved.walous_enabled and WalousLandCoverService._source_path(resolved, product).is_file()
result.append(
ThematicRasterProductRead(
key=product.key,
display_name=product.display_name,
theme=WalousLandCoverService.THEME,
metric_kind=WalousLandCoverService.METRIC_KIND,
coverage_id=product.source_version,
native_resolution_m=WalousLandCoverService.SOURCE_RESOLUTION_M,
analysis_resolution_m=resolved.walous_analysis_resolution_m,
source_crs=WalousLandCoverService.SOURCE_CRS,
source_value_unit=WalousLandCoverService.SOURCE_VALUE_UNIT,
observation_year=product.observation_year,
source_version=product.source_version,
catalog_url=product.catalog_url,
attribution=WalousLandCoverService.ATTRIBUTION,
license_note=WalousLandCoverService.LICENSE_NOTE,
legend_min_label="WALOUS klasse 1",
legend_max_label="WALOUS klasse 11",
included_source_values=list(WalousLandCoverService.CLASS_LABELS),
limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.",
coverage_zones=["wallonia"],
configured=configured,
status="configured" if configured else "source_not_provisioned",
).model_dump()
)
return result
@staticmethod
def _product(product_key: str) -> WalousProduct:
product = WalousLandCoverService._products().get(product_key.strip().lower())
if product is None:
raise AppError(
code="WALOUS_PRODUCT_NOT_SUPPORTED",
message="Select a product from the governed WALOUS registry",
details={"product_key": product_key},
status_code=422,
)
return product
@staticmethod
def _scope_geometry(db, project_id: UUID, payload: ThematicRasterAcquireRequest):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
if payload.bbox.crs.upper() != "EPSG:4326":
raise AppError(code="INVALID_BBOX_CRS", message="WALOUS acquisition requires EPSG:4326", status_code=400)
values = [payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y]
if not all(math.isfinite(value) for value in values) or values[0] >= values[2] or values[1] >= values[3]:
raise AppError(code="INVALID_BBOX", message="WALOUS selection must be a finite non-empty rectangle", status_code=400)
selection = box(*values)
if payload.area_id is None:
return selection, values
area = db.get(Area, payload.area_id)
if area is None or area.project_id != project_id:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
selection = selection.intersection(to_shape(area.geometry))
if selection.is_empty or selection.area <= 0:
raise AppError(code="WALOUS_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422)
return selection, values
@staticmethod
def _read_source_window(
source_path: Path,
scope_4326,
settings: Settings,
) -> tuple[bytes, dict[str, Any]]:
try:
import numpy as np
import rasterio
from rasterio.enums import Resampling
from rasterio.features import geometry_mask
from rasterio.io import MemoryFile
from rasterio.transform import from_bounds
from rasterio.windows import from_bounds as window_from_bounds
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for WALOUS", status_code=503) from exc
resolution = float(settings.walous_analysis_resolution_m)
transformer = Transformer.from_crs("EPSG:4326", WalousLandCoverService.SOURCE_CRS, always_xy=True)
scope_metric = shapely_transform(transformer.transform, scope_4326)
try:
with rasterio.open(source_path) as source:
if source.crs is None or source.crs.to_epsg() != 3812 or source.count != 1:
raise AppError(code="WALOUS_SOURCE_INVALID", message="WALOUS source must be a one-band EPSG:3812 raster", status_code=409)
if not all(math.isclose(abs(float(value)), 1.0, abs_tol=0.05) for value in source.res):
raise AppError(code="WALOUS_SOURCE_INVALID", message="WALOUS source must retain the official 1 m resolution", status_code=409)
clipped_geometry = scope_metric.intersection(box(*source.bounds))
if clipped_geometry.is_empty or clipped_geometry.area <= 0:
raise AppError(code="WALOUS_SELECTION_OUTSIDE_COVERAGE", message="Selection does not overlap WALOUS coverage", status_code=422)
min_x, min_y, max_x, max_y = clipped_geometry.bounds
bounds = (
math.floor(min_x / resolution) * resolution,
math.floor(min_y / resolution) * resolution,
math.ceil(max_x / resolution) * resolution,
math.ceil(max_y / resolution) * resolution,
)
width_m, height_m = bounds[2] - bounds[0], bounds[3] - bounds[1]
if width_m > settings.walous_max_side_m or height_m > settings.walous_max_side_m:
raise AppError(
code="WALOUS_SELECTION_TOO_LARGE",
message=f"Select no more than {settings.walous_max_side_m:g} by {settings.walous_max_side_m:g} metres",
details={"width_m": width_m, "height_m": height_m},
status_code=422,
)
width, height = max(1, round(width_m / resolution)), max(1, round(height_m / resolution))
if width * height > settings.walous_max_pixels:
raise AppError(code="WALOUS_SELECTION_TOO_LARGE", message="WALOUS selection exceeds the configured cell limit", details={"pixel_count": width * height, "max_pixels": settings.walous_max_pixels}, status_code=422)
window = window_from_bounds(*bounds, transform=source.transform)
band = source.read(1, window=window, out_shape=(height, width), masked=True, resampling=Resampling.nearest)
output_transform = from_bounds(*bounds, width, height)
outside_scope = geometry_mask([mapping(clipped_geometry)], out_shape=(height, width), transform=output_transform, invert=False)
raw = np.asarray(band.filled(WalousLandCoverService.NODATA), dtype="uint8")
invalid = np.ma.getmaskarray(band) | outside_scope
if source.nodata is not None:
invalid |= np.isclose(raw.astype("float64"), float(source.nodata))
raw[invalid] = WalousLandCoverService.NODATA
valid = raw[raw != WalousLandCoverService.NODATA]
if valid.size == 0:
raise AppError(code="WALOUS_NO_VALID_DATA", message="WALOUS contains no valid cells in this selection", status_code=422)
classes = set(np.unique(valid).astype(int).tolist())
unexpected = sorted(classes - set(WalousLandCoverService.CLASS_LABELS))
if unexpected:
raise AppError(code="WALOUS_SOURCE_INVALID_VALUES", message="WALOUS contains classes outside the governed 1-11 legend", details={"unexpected_classes": unexpected}, status_code=409)
profile = {
"driver": "GTiff",
"width": width,
"height": height,
"count": 1,
"dtype": "uint8",
"crs": WalousLandCoverService.SOURCE_CRS,
"transform": output_transform,
"nodata": WalousLandCoverService.NODATA,
"compress": "deflate",
"predictor": 2,
}
with MemoryFile() as memory:
with memory.open(**profile) as output:
output.write(raw, 1)
content = memory.read()
return content, {
"width": width,
"height": height,
"valid_pixel_count": int(valid.size),
"classes_present": sorted(classes),
"bbox_epsg3812": list(bounds),
"source_width": int(source.width),
"source_height": int(source.height),
"source_nodata": None if source.nodata is None else float(source.nodata),
"source_resolution_m": 1.0,
"analysis_resolution_m": resolution,
}
except AppError:
raise
except Exception as exc:
raise AppError(code="WALOUS_SOURCE_READ_FAILED", message="The provisioned WALOUS source could not be read", details={"reason": str(exc)}, status_code=500) from exc
@staticmethod
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
candidate = (
db.query(Dataset)
.filter(Dataset.project_id == project_id, Dataset.name == filename, Dataset.source_name == WalousLandCoverService.PROVIDER, Dataset.status == "ready")
.order_by(Dataset.imported_at.desc())
.first()
)
return candidate if candidate and candidate.storage_path and Path(candidate.storage_path).is_file() else None
@staticmethod
def acquire(db, project_id: UUID, payload: ThematicRasterAcquireRequest, *, settings: Settings | None = None) -> dict[str, Any]:
resolved = settings or get_settings()
if not resolved.walous_enabled:
raise AppError(code="WALOUS_NOT_CONFIGURED", message="WALOUS bounded analysis is disabled", status_code=503)
product = WalousLandCoverService._product(payload.product_key)
source_path = WalousLandCoverService._source_path(resolved, product)
if not source_path.is_file():
raise AppError(
code="WALOUS_SOURCE_NOT_PROVISIONED",
message="The official WALOUS source archive has not been provisioned on this runtime",
details={"expected_path": str(source_path), "operator_command": "python scripts/provision_walous_sources.py --years 2020 2023"},
status_code=503,
)
scope, bbox_4326 = WalousLandCoverService._scope_geometry(db, project_id, payload)
identity = {
"product_key": product.key,
"bbox_epsg4326": [round(float(value), 8) for value in bbox_4326],
"area_id": str(payload.area_id) if payload.area_id else None,
"analysis_resolution_m": resolved.walous_analysis_resolution_m,
}
request_hash = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
filename = f"walous_{product.observation_year}_{request_hash[:12]}_3812.tif"
if not payload.force_refresh:
cached = WalousLandCoverService._cached_dataset(db, project_id, filename)
if cached is not None:
metadata = cached.source_metadata or {}
return WalousAcquisitionResult(
output_dataset_id=cached.id,
reused=True,
provider=WalousLandCoverService.PROVIDER,
product_key=product.key,
display_name=product.display_name,
theme=WalousLandCoverService.THEME,
metric_kind=WalousLandCoverService.METRIC_KIND,
resolution_m=float(metadata.get("analysis_resolution_m", resolved.walous_analysis_resolution_m)),
width=int((cached.metadata_json or {}).get("width", 0)),
height=int((cached.metadata_json or {}).get("height", 0)),
valid_pixel_count=int(metadata.get("valid_pixel_count", 0)),
bbox_epsg4326=bbox_4326,
bbox_epsg3812=list(metadata.get("bbox_epsg3812") or []),
observation_year=product.observation_year,
source_value_unit=WalousLandCoverService.SOURCE_VALUE_UNIT,
attribution=WalousLandCoverService.ATTRIBUTION,
limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.",
).model_dump(mode="json")
content, validation = WalousLandCoverService._read_source_window(source_path, scope, resolved)
source_sha256_path = source_path.with_name(product.source_sha256_filename)
source_sha256 = source_sha256_path.read_text(encoding="ascii").strip().split()[0] if source_sha256_path.is_file() else None
acquired_at = datetime.now(UTC)
observed_at = datetime(product.observation_year, 12, 31, 23, 59, 59, tzinfo=UTC)
spatial_series_hash = hashlib.sha256(json.dumps({"bbox": identity["bbox_epsg4326"], "area_id": identity["area_id"], "resolution": identity["analysis_resolution_m"]}, sort_keys=True).encode()).hexdigest()[:24]
dataset = DatasetService.import_raster_bytes(
db,
project_id=project_id,
area_id=payload.area_id,
filename=filename,
content=content,
source=f"SPW WALOUS {product.source_version} operator-provisioned GeoTIFF",
source_name=WalousLandCoverService.PROVIDER,
temporal_series_key=f"spw:walous:land-cover:{spatial_series_hash}",
observed_at=observed_at,
valid_from=datetime(product.observation_year, 1, 1, tzinfo=UTC),
valid_to=observed_at,
temporal_granularity="year",
source_version=product.source_version,
source_metadata={
"provider": WalousLandCoverService.PROVIDER,
"service": "official_predefined_dataset_atom",
"product_key": product.key,
"product_display_name": product.display_name,
"theme": WalousLandCoverService.THEME,
"metric_kind": WalousLandCoverService.METRIC_KIND,
"source_crs": WalousLandCoverService.SOURCE_CRS,
"source_resolution_m": WalousLandCoverService.SOURCE_RESOLUTION_M,
"analysis_resolution_m": validation["analysis_resolution_m"],
"source_value_unit": WalousLandCoverService.SOURCE_VALUE_UNIT,
"class_labels": WalousLandCoverService.CLASS_LABELS,
"observation_year": product.observation_year,
"valid_pixel_count": validation["valid_pixel_count"],
"classes_present": validation["classes_present"],
"bbox_epsg4326": bbox_4326,
"bbox_epsg3812": validation["bbox_epsg3812"],
"coverage_zones": ["wallonia"],
"catalog_url": product.catalog_url,
"download_url": product.download_url,
"attribution": WalousLandCoverService.ATTRIBUTION,
"license_note": WalousLandCoverService.LICENSE_NOTE,
"limitation_message": f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.",
},
provenance_metadata={
"acquisition": "operator_provisioned_official_archive_bounded_window",
"acquired_at": acquired_at.isoformat(),
"request_hash": request_hash,
"source_filename": product.source_filename,
"source_sha256": source_sha256,
"derived_sha256": hashlib.sha256(content).hexdigest(),
"resampling": "nearest",
"validation": validation,
},
)
return WalousAcquisitionResult(
output_dataset_id=dataset.id,
reused=False,
provider=WalousLandCoverService.PROVIDER,
product_key=product.key,
display_name=product.display_name,
theme=WalousLandCoverService.THEME,
metric_kind=WalousLandCoverService.METRIC_KIND,
resolution_m=validation["analysis_resolution_m"],
width=validation["width"],
height=validation["height"],
valid_pixel_count=validation["valid_pixel_count"],
bbox_epsg4326=bbox_4326,
bbox_epsg3812=validation["bbox_epsg3812"],
observation_year=product.observation_year,
source_value_unit=WalousLandCoverService.SOURCE_VALUE_UNIT,
attribution=WalousLandCoverService.ATTRIBUTION,
limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.",
).model_dump(mode="json")
@staticmethod
def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> tuple[Dataset, WalousProduct]:
dataset = db.get(Dataset, dataset_id)
if not dataset or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
if dataset.dataset_type != "raster" or dataset.source_name != WalousLandCoverService.PROVIDER:
raise AppError(code="INVALID_WALOUS_DATASET", message="WALOUS analysis requires a governed WALOUS raster", status_code=400)
if dataset.status != "ready" or not dataset.storage_path or not Path(dataset.storage_path).is_file():
raise AppError(code="DATASET_FILE_MISSING", message="Persisted WALOUS raster is unavailable", status_code=404)
product = WalousLandCoverService._product(str((dataset.source_metadata or {}).get("product_key") or ""))
return dataset, product
@staticmethod
def _analysis_geometry(db, project_id: UUID, payload: ThematicRasterSelectionRequest):
selection = box(payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
if payload.area_id is None:
return selection
area = db.get(Area, payload.area_id)
if area is None or area.project_id != project_id:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
selection = selection.intersection(to_shape(area.geometry))
if selection.is_empty or selection.area <= 0:
raise AppError(code="WALOUS_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422)
return selection
@staticmethod
def analyze(db, project_id: UUID, dataset_id: UUID, payload: ThematicRasterSelectionRequest) -> dict[str, Any]:
dataset, product = WalousLandCoverService._load_dataset(db, project_id, dataset_id)
selection_4326 = WalousLandCoverService._analysis_geometry(db, project_id, payload)
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.mask import mask
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for WALOUS analysis", status_code=503) from exc
try:
with rasterio.open(dataset.storage_path) as source:
transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True)
selection_metric = shapely_transform(transformer.transform, selection_4326)
geometry = selection_metric.intersection(box(*source.bounds))
if geometry.is_empty or geometry.area <= 0:
raise AppError(code="WALOUS_SELECTION_OUTSIDE_DATASET", message="Selection does not overlap the persisted WALOUS raster", status_code=422)
clipped, transform = mask(source, [mapping(geometry)], crop=True, filled=False, indexes=[1])
band = np.ma.asarray(clipped[0])
raw = np.asarray(band.filled(WalousLandCoverService.NODATA), dtype="uint8")
selected = geometry_mask([mapping(geometry)], out_shape=raw.shape, transform=transform, invert=True)
valid = selected & ~np.ma.getmaskarray(band) & (raw != WalousLandCoverService.NODATA)
values = raw[valid]
selected_count = int(selected.sum())
valid_count = int(values.size)
if not valid_count:
raise AppError(code="WALOUS_NO_VALID_DATA", message="WALOUS contains no valid cells in this selection", status_code=422)
cell_area_m2 = abs(float(source.res[0]) * float(source.res[1]))
except AppError:
raise
except Exception as exc:
raise AppError(code="WALOUS_ANALYSIS_FAILED", message="The persisted WALOUS raster could not be analysed", details={"reason": str(exc)}, status_code=500) from exc
def area_for(classes: set[int]) -> float:
return float(np.count_nonzero(np.isin(values, list(classes))) * cell_area_m2 / 10_000.0)
metric_specs = [
("land_cover_observed_area_ha", "Gekarteerde landbedekking", set(WalousLandCoverService.CLASS_LABELS)),
("forest_cover_area_ha", "Boom- en bosbedekking", {3, 4, 5, 6}),
("surface_water_area_ha", "Oppervlaktewater", {8}),
("artificial_cover_area_ha", "Kunstmatige bedekking en constructies", {9, 10, 11}),
("annual_herbaceous_cover_area_ha", "Jaarlijks wisselende kruidlaag", {1}),
("permanent_herbaceous_cover_area_ha", "Jaarronde kruidlaag", {2}),
("bare_soil_area_ha", "Kale bodem", {7}),
]
metrics = [
ThematicRasterMetric(
metric_key=key,
metric_label=label,
metric_value=round(area_for(classes), 4),
metric_unit="ha",
aggregation_method="nearest_resampled_cells_times_cell_area",
is_estimate=True,
)
for key, label, classes in metric_specs
]
primary = metrics[0]
return ThematicRasterSelectionResponse(
dataset_id=dataset.id,
product_key=product.key,
theme=WalousLandCoverService.THEME,
metric_kind=WalousLandCoverService.METRIC_KIND,
selection_bbox=payload.bbox,
selection_area_id=payload.area_id,
selected_cell_count=selected_count,
valid_cell_count=valid_count,
coverage_ratio=round(valid_count / max(1, selected_count), 6),
resolution_m=round(math.sqrt(cell_area_m2), 4),
observation_year=product.observation_year,
summary=ThematicRasterSelectionSummary(
metric_label=primary.metric_label,
metric_value=primary.metric_value,
metric_unit=primary.metric_unit,
aggregation_method=primary.aggregation_method,
primary_metric_key=primary.metric_key,
metrics=metrics,
),
unsupported_metrics=["legal_land_use", "ownership", "tree_count", "timber_volume", "water_volume"],
limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.",
generated_at=datetime.now(UTC).isoformat(),
).model_dump(mode="json")
@staticmethod
def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes:
dataset, _product = WalousLandCoverService._load_dataset(db, project_id, dataset_id)
try:
import numpy as np
import rasterio
from PIL import Image
from rasterio.enums import Resampling
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio, numpy and Pillow are required for WALOUS rendering", status_code=503) from exc
with rasterio.open(dataset.storage_path) as source:
scale = min(1.0, max_dimension / max(source.width, source.height))
width, height = max(1, round(source.width * scale)), max(1, round(source.height * scale))
values = source.read(1, out_shape=(height, width), masked=True, resampling=Resampling.nearest)
raw = np.asarray(values.filled(WalousLandCoverService.NODATA), dtype="uint8")
rgba = np.zeros((height, width, 4), dtype="uint8")
for value, color in WalousLandCoverService.CLASS_COLORS.items():
selected = raw == value
rgba[:, :, 0][selected] = color[0]
rgba[:, :, 1][selected] = color[1]
rgba[:, :, 2][selected] = color[2]
rgba[:, :, 3][selected] = 205
output = io.BytesIO()
Image.fromarray(rgba).save(output, format="PNG", optimize=True)
return output.getvalue()
@@ -95,6 +95,7 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None
"activate_promoted_yolo_candidate.py",
"manage_grb_refresh.py",
"orthophoto_release_preflight.py",
"provision_walous_sources.py",
}
for script_name in required_runtime_scripts:
assert f"COPY scripts/{script_name} /app/scripts/{script_name}" in dockerfile
@@ -149,6 +150,24 @@ def test_env_example_uses_runtime_env_names_read_by_backend_and_frontend() -> No
assert "VITE_API_PROXY_TARGET=http://localhost:8000" in env_example
def test_walloon_runtime_settings_are_editable_in_compose_and_unraid() -> None:
files = [
(ROOT / "docker-compose.yml").read_text(encoding="utf-8"),
(ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8"),
(ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8"),
(ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8"),
(ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(encoding="utf-8"),
]
for content in files:
assert "SPW_FLOOD_HAZARD_ENABLED" in content
assert "SPW_FLOOD_HAZARD_MAPSERVER_URL" in content
assert "WALOUS_ENABLED" in content
assert "WALOUS_SOURCE_DIR" in content
assert "WALOUS_ANALYSIS_RESOLUTION_M" in content
assert "WALOUS_MAX_SIDE_M" in content
assert "WALOUS_MAX_PIXELS" in content
def test_frontend_uses_same_origin_api_proxy_by_default() -> None:
api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
@@ -116,6 +116,9 @@ def test_regional_product_registry_is_explicit_and_source_specific() -> None:
]
assert products["spw_picc_waterways"]["collection"] == "28"
assert products["spw_picc_water_surfaces"]["collection"] == "30"
assert products["spw_flood_hazard_2021"]["collection"] == "2"
assert products["spw_flood_hazard_2021"]["theme"] == "flood_hazard"
assert products["spw_flood_hazard_2021"]["coverage_zones"] == ["wallonia"]
assert products["urbis_buildings"]["coverage_zones"] == ["brussels"]
assert products["urbis_buildings"]["license_note"] == "Buildings are published under CC0."
assert "FPS Finance" in products["urbis_cadastral_parcels"]["license_note"]
@@ -252,6 +255,82 @@ def test_spw_arcgis_paging_is_bounded_stable_and_clipped() -> None:
assert all(item["properties"]["clipped_area_ha"] > 0 for item in features)
def test_spw_flood_hazard_uses_separate_governed_endpoint_and_persists_classification() -> None:
product = OfficialVectorAcquisitionService._product("spw_flood_hazard_2021")
scope = Polygon(
[(4.55, 50.58), (4.56, 50.58), (4.56, 50.59), (4.55, 50.59), (4.55, 50.58)]
)
scope_metric = Polygon(
[_TO_LAMBERT72.transform(x, y) for x, y in scope.exterior.coords]
)
def opener(raw_request, timeout):
assert timeout == 180
parsed = urlparse(raw_request.full_url)
assert parsed.path.endswith("/EAU/ALEA_INOND/MapServer/2/query")
query = parse_qs(parsed.query)
assert query["outSR"] == ["4326"]
return JsonResponse(
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": 7,
"geometry": {
"type": "Polygon",
"coordinates": [[
[4.551, 50.581],
[4.559, 50.581],
[4.559, 50.589],
[4.551, 50.589],
[4.551, 50.581],
]],
},
"properties": {
"OBJECTID": 7,
"LOCALID": "ALEA-7",
"TYPEALEA": "Debordement",
"CLASSEMENT": 130,
"MILLESIME": 2021,
},
}
],
"exceededTransferLimit": False,
}
)
features, transfer = OfficialVectorAcquisitionService._fetch_features(
product,
scope,
scope_metric,
"wallonia",
Settings(_env_file=None),
opener,
)
assert transfer["feature_count"] == 1
assert features[0]["id"] == "2:ALEA-7"
assert features[0]["properties"]["CLASSEMENT"] == 130
assert features[0]["properties"]["source_name"] == "spw_flood_hazard"
assert features[0]["properties"]["clipped_area_ha"] > 0
def test_spw_flood_hazard_can_be_disabled_independently() -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
with pytest.raises(AppError) as exc_info:
OfficialVectorAcquisitionService.acquire(
db,
project_id,
request("spw_flood_hazard_2021", (4.55, 50.58, 4.56, 50.59)),
settings=Settings(_env_file=None, SPW_FLOOD_HAZARD_ENABLED=False),
)
assert exc_info.value.code == "SPW_FLOOD_HAZARD_NOT_CONFIGURED"
def test_regional_products_require_the_persisted_authoritative_coverage_area() -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
@@ -130,8 +130,9 @@ def test_product_registries_expose_honest_forest_agriculture_nature_and_soil() -
"dov_soil_types",
"spw_picc_buildings",
"spw_picc_roads",
"spw_picc_waterways",
"spw_picc_water_surfaces",
"spw_picc_waterways",
"spw_picc_water_surfaces",
"spw_flood_hazard_2021",
"urbis_buildings",
"urbis_cadastral_parcels",
"urbis_street_axes",
@@ -405,7 +406,7 @@ def test_official_vector_routes_and_frontend_use_canonical_backend_path(monkeypa
assert products_response.status_code == 200
assert set(products_response.json()) == {"data"}
assert products_response.json()["data"]["total"] == 12
assert products_response.json()["data"]["total"] == 13
assert acquire_response.status_code == 200
assert set(acquire_response.json()) == {"data"}
assert acquire_response.json()["data"]["job_type"] == "vector.official.acquire"
@@ -0,0 +1,345 @@
from __future__ import annotations
from datetime import datetime, timezone
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
import numpy as np
from fastapi.testclient import TestClient
from pyproj import Transformer
import rasterio
from rasterio.transform import from_origin
from app.core.config import Settings
from app.db.session import get_db
from app.main import app
from app.models import Dataset, Job, Project
from app.schemas.thematic_raster import ThematicRasterAcquireRequest, ThematicRasterSelectionRequest
from app.schemas.temporal import TemporalComparisonRequest
from app.services.dataset_service import DatasetService
from app.services.temporal_analysis_service import TemporalAnalysisService
from app.services.walous_land_cover_service import WalousLandCoverService
class FakeQuery:
def filter(self, *_args):
return self
def order_by(self, *_args):
return self
def first(self):
return None
class FakeSession:
def __init__(self, project, dataset=None):
self.project = project
self.dataset = dataset
self.added = []
def get(self, model, row_id):
if model is Project and row_id == self.project.id:
return self.project
if model is Dataset and self.dataset is not None and row_id == self.dataset.id:
return self.dataset
match = next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
if match is not None:
return match
return None
def query(self, _model):
return FakeQuery()
def add(self, row):
self.added.append(row)
def commit(self):
return None
def rollback(self):
return None
def refresh(self, row):
return row
def make_source(path: Path) -> tuple[list[float], np.ndarray]:
to_3812 = Transformer.from_crs("EPSG:4326", "EPSG:3812", always_xy=True)
to_4326 = Transformer.from_crs("EPSG:3812", "EPSG:4326", always_xy=True)
x, y = to_3812.transform(4.85, 50.45)
transform = from_origin(x, y + 100, 1, 1)
values = np.ones((100, 100), dtype="uint8")
values[:, 20:40] = 4
values[:, 40:50] = 8
values[:, 50:70] = 9
values[:, 70:] = 2
with rasterio.open(
path,
"w",
driver="GTiff",
width=100,
height=100,
count=1,
dtype="uint8",
crs="EPSG:3812",
transform=transform,
nodata=255,
) as target:
target.write(values, 1)
min_lon, min_lat = to_4326.transform(x, y)
max_lon, max_lat = to_4326.transform(x + 100, y + 100)
return [min_lon, min_lat, max_lon, max_lat], values
def settings(source_dir: Path) -> Settings:
return Settings(
_env_file=None,
WALOUS_SOURCE_DIR=str(source_dir),
WALOUS_ANALYSIS_RESOLUTION_M=10,
WALOUS_MAX_SIDE_M=60_000,
WALOUS_MAX_PIXELS=1_000_000,
)
def test_walous_registry_reports_real_provisioning_state(tmp_path: Path) -> None:
before = {item["key"]: item for item in WalousLandCoverService.list_products(settings=settings(tmp_path))}
assert before["walous_land_cover_2023"]["status"] == "source_not_provisioned"
make_source(tmp_path / "walous_land_cover_2023_3812.tif")
after = {item["key"]: item for item in WalousLandCoverService.list_products(settings=settings(tmp_path))}
assert after["walous_land_cover_2023"]["configured"] is True
assert after["walous_land_cover_2023"]["source_crs"] == "EPSG:3812"
assert after["walous_land_cover_2023"]["native_resolution_m"] == 1.0
assert after["walous_land_cover_2023"]["analysis_resolution_m"] == 10.0
assert after["walous_land_cover_2023"]["coverage_zones"] == ["wallonia"]
def test_walous_acquisition_reads_real_classes_and_persists_provenance(tmp_path: Path, monkeypatch) -> None:
bbox, _values = make_source(tmp_path / "walous_land_cover_2023_3812.tif")
project = Project(id=uuid4(), name="Belgium")
db = FakeSession(project)
captured = {}
output_id = uuid4()
def persist(_db, **kwargs):
captured.update(kwargs)
return SimpleNamespace(id=output_id)
monkeypatch.setattr(DatasetService, "import_raster_bytes", persist)
result = WalousLandCoverService.acquire(
db,
project.id,
ThematicRasterAcquireRequest(
bbox={"min_x": bbox[0], "min_y": bbox[1], "max_x": bbox[2], "max_y": bbox[3], "crs": "EPSG:4326"},
product_key="walous_land_cover_2023",
force_refresh=True,
),
settings=settings(tmp_path),
)
assert result["output_dataset_id"] == str(output_id)
assert result["resolution_m"] == 10
assert captured["source_name"] == "spw_walous_land_cover"
assert captured["source_metadata"]["classes_present"] == [1, 2, 4, 8, 9]
assert captured["provenance_metadata"]["resampling"] == "nearest"
assert captured["temporal_series_key"].startswith("spw:walous:land-cover:")
assert captured["observed_at"].year == 2023
def test_walous_analysis_returns_semantic_area_metrics(tmp_path: Path, monkeypatch) -> None:
bbox, _values = make_source(tmp_path / "walous_land_cover_2023_3812.tif")
project = Project(id=uuid4(), name="Belgium")
output_id = uuid4()
captured = {}
def persist(_db, **kwargs):
captured.update(kwargs)
return SimpleNamespace(id=output_id)
monkeypatch.setattr(DatasetService, "import_raster_bytes", persist)
db = FakeSession(project)
payload = ThematicRasterAcquireRequest(
bbox={"min_x": bbox[0], "min_y": bbox[1], "max_x": bbox[2], "max_y": bbox[3], "crs": "EPSG:4326"},
product_key="walous_land_cover_2023",
force_refresh=True,
)
WalousLandCoverService.acquire(db, project.id, payload, settings=settings(tmp_path))
persisted_path = tmp_path / "derived.tif"
persisted_path.write_bytes(captured["content"])
dataset = Dataset(
id=output_id,
project_id=project.id,
name="derived.tif",
dataset_type="raster",
source="SPW WALOUS",
source_name="spw_walous_land_cover",
source_metadata=captured["source_metadata"],
provenance_metadata=captured["provenance_metadata"],
storage_path=str(persisted_path),
status="ready",
)
db.dataset = dataset
result = WalousLandCoverService.analyze(
db,
project.id,
output_id,
ThematicRasterSelectionRequest(bbox=payload.bbox),
)
metrics = {item["metric_key"]: item["metric_value"] for item in result["summary"]["metrics"]}
assert result["metric_kind"] == "categorical_area"
assert metrics["land_cover_observed_area_ha"] > 0
assert metrics["forest_cover_area_ha"] > 0
assert metrics["surface_water_area_ha"] > 0
assert metrics["artificial_cover_area_ha"] > 0
assert "water_volume" in result["unsupported_metrics"]
def test_walous_render_png_uses_governed_class_colours(tmp_path: Path) -> None:
bbox, _values = make_source(tmp_path / "walous_land_cover_2023_3812.tif")
project = Project(id=uuid4(), name="Belgium")
dataset = Dataset(
id=uuid4(),
project_id=project.id,
name="walous.tif",
dataset_type="raster",
source="SPW WALOUS",
source_name="spw_walous_land_cover",
source_metadata={"product_key": "walous_land_cover_2023", "bbox_epsg4326": bbox},
storage_path=str(tmp_path / "walous_land_cover_2023_3812.tif"),
status="ready",
)
db = FakeSession(project, dataset)
rendered = WalousLandCoverService.render_png(db, project.id, dataset.id)
assert rendered.startswith(b"\x89PNG\r\n\x1a\n")
def test_walous_temporal_comparison_reuses_persisted_raster_metrics(monkeypatch) -> None:
project_id = uuid4()
earlier = Dataset(
id=uuid4(),
project_id=project_id,
name="walous-2020.tif",
dataset_type="raster",
source="SPW WALOUS",
source_name="spw_walous_land_cover",
temporal_series_key="spw:walous:land-cover:selection",
observed_at=datetime(2020, 12, 31, 23, 59, 59, tzinfo=timezone.utc),
source_version="WAL_OCS_IA__2020",
)
later = Dataset(
id=uuid4(),
project_id=project_id,
name="walous-2023.tif",
dataset_type="raster",
source="SPW WALOUS",
source_name="spw_walous_land_cover",
temporal_series_key=earlier.temporal_series_key,
observed_at=datetime(2023, 12, 31, 23, 59, 59, tzinfo=timezone.utc),
source_version="WAL_OCS_IA__2023",
)
rows = {earlier.id: earlier, later.id: later}
class TemporalSession:
def get(self, model, row_id):
return rows.get(row_id) if model is Dataset else None
def analyze(_db, _project_id, dataset_id, _payload):
value = 4.0 if dataset_id == earlier.id else 5.5
return {
"summary": {
"metric_label": "Gekarteerde landbedekking",
"metric_value": value,
"metric_unit": "ha",
"aggregation_method": "nearest_resampled_cells_times_cell_area",
"primary_metric_key": "land_cover_observed_area_ha",
"metrics": [{
"metric_key": "land_cover_observed_area_ha",
"metric_label": "Gekarteerde landbedekking",
"metric_value": value,
"metric_unit": "ha",
"aggregation_method": "nearest_resampled_cells_times_cell_area",
"is_estimate": True,
}],
},
"limitation_message": "Cell-based estimate.",
}
monkeypatch.setattr(WalousLandCoverService, "analyze", analyze)
payload = TemporalComparisonRequest(
earlier_dataset_id=earlier.id,
later_dataset_id=later.id,
bbox={"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"},
)
result = TemporalAnalysisService.compare(TemporalSession(), project_id=project_id, payload=payload)
assert result.metric.earlier_value == 4.0
assert result.metric.later_value == 5.5
assert result.metric.absolute_change == 1.5
assert result.object_changes.available is False
def test_walous_api_routes_use_canonical_envelopes(monkeypatch) -> None:
project = Project(id=uuid4(), name="Belgium")
dataset_id = uuid4()
db = FakeSession(project)
monkeypatch.setattr(
WalousLandCoverService,
"acquire",
lambda *_args, **_kwargs: {"output_dataset_id": str(dataset_id), "provider": WalousLandCoverService.PROVIDER},
)
monkeypatch.setattr(
WalousLandCoverService,
"analyze",
lambda *_args, **_kwargs: {
"dataset_id": str(dataset_id),
"product_key": "walous_land_cover_2023",
"theme": "land_cover_use",
"metric_kind": "categorical_area",
"selection_bbox": {"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"},
"selected_cell_count": 100,
"valid_cell_count": 100,
"coverage_ratio": 1.0,
"resolution_m": 10.0,
"observation_year": 2023,
"summary": {
"metric_label": "Gekarteerde landbedekking",
"metric_value": 1.0,
"metric_unit": "ha",
"aggregation_method": "nearest_resampled_cells_times_cell_area",
"primary_metric_key": "land_cover_observed_area_ha",
"metrics": [],
},
"unsupported_metrics": ["water_volume"],
"limitation_message": "Cell-based estimate.",
"generated_at": "2026-07-22T00:00:00Z",
},
)
app.dependency_overrides[get_db] = lambda: db
try:
client = TestClient(app)
products = client.get(f"/api/v1/projects/{project.id}/datasets/walous/products")
acquisition = client.post(
f"/api/v1/projects/{project.id}/datasets/walous/acquire",
json={
"bbox": {"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"},
"product_key": "walous_land_cover_2023",
},
)
selection = client.post(
f"/api/v1/projects/{project.id}/datasets/{dataset_id}/raster/walous/select",
json={"bbox": {"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"}},
)
finally:
app.dependency_overrides.clear()
assert products.status_code == 200 and set(products.json()) == {"data"}
assert products.json()["data"]["total"] == 2
assert acquisition.status_code == 200 and set(acquisition.json()) == {"data"}
assert acquisition.json()["data"]["job_type"] == "raster.walous.acquire"
assert selection.status_code == 200 and selection.json()["data"]["theme"] == "land_cover_use"
assert any(isinstance(item, Job) for item in db.added)