diff --git a/.env.example b/.env.example index 562a7aa7..716e7ce8 100644 --- a/.env.example +++ b/.env.example @@ -30,6 +30,8 @@ BWK_WFS_URL=https://geo.api.vlaanderen.be/BWK/wfs DOV_SOIL_WFS_URL=https://www.dov.vlaanderen.be/geoserver/wfs SPW_PICC_ENABLED=true SPW_PICC_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer +SPW_FLOOD_HAZARD_ENABLED=true +SPW_FLOOD_HAZARD_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer URBIS_ENABLED=true URBIS_WFS_URL=https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows OFFICIAL_VECTOR_MIN_SIDE_M=10 @@ -89,6 +91,11 @@ THEMATIC_RASTER_MAX_SIDE_M=60000 THEMATIC_RASTER_MAX_PIXELS=30000000 THEMATIC_RASTER_TIMEOUT_SECONDS=300 THEMATIC_RASTER_MAX_RESPONSE_MB=160 +WALOUS_ENABLED=true +WALOUS_SOURCE_DIR=/app/storage/source-cache/walous +WALOUS_ANALYSIS_RESOLUTION_M=10 +WALOUS_MAX_SIDE_M=60000 +WALOUS_MAX_PIXELS=36000000 YOLO_ENABLED=false YOLO_MODELS_DIR=/app/models YOLO_MODEL_PATH= diff --git a/CHANGELOG.md b/CHANGELOG.md index dbe1468d..b9955207 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,20 @@ ## Unreleased - Post-V1 capability completion (2026-07-19) +- Added operational Walloon WALOUS 2020/2023 land-cover analysis: an + allowlisted checksum-validating provisioner, bounded EPSG:3812 window + persistence, semantic hectare metrics, governed map rendering and raster + temporal comparison. The map acquires comparable configured editions for + the same selection so 2020-2023 evolution becomes available without manual + dataset administration. +- Added the current legal SPW Walloon flood-hazard polygons as a bounded + authoritative vector product with class-aware hectare metrics and canonical + persistence. No WMS pixels or modeled depths are fabricated. +- Kept the Walloon DTM and MDK North Sea depth model honest: the published DTM + artifacts require an explicit large-storage partition plan, while the MDK + endpoint still fails strict hostname validation. Neither is presented as a + measured analytical layer. + - Made `Belgium and North Sea Workbench` the unconditional frontend startup context, moved the initial MapLibre viewport to national extent and removed Mol/Kempen defaults from project/area forms and end-user source copy. Mol and diff --git a/backend/README.md b/backend/README.md index 38d63d81..5e0e80ef 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1599,6 +1599,40 @@ Datasets. A full-Flanders raster request remains blocked by the same 60 km and `Gemeente ...` as `coverage_scope=municipality`; regional Area clipping is stored as `bounded_selection`. +## Walloon WALOUS land cover and flood hazard + +The Wallonia map flow uses bounded PICC vector products, the queryable legal +SPW flood-hazard polygon layer and provisioned official WALOUS land-cover +rasters. Provision the 2020 and 2023 source editions once in the persistent +storage mount: + +```bash +docker exec geointel python /app/scripts/provision_walous_sources.py \ + --years 2020 2023 \ + --destination /app/storage/source-cache/walous +``` + +The provisioner verifies advertised archive sizes, safe ZIP structure, +EPSG:3812, one band, 1 m cells, class values 1-11 and SHA-256 checksums. It +does not run at application startup. `GET .../datasets/walous/products` +therefore reports `source_not_provisioned` until both source files exist. + +For a bounded Walloon selection the browser persists the latest edition and +all other configured comparable editions. `POST .../raster/walous/select` +returns cell-area hectares; the temporal API compares the same semantic metric +keys for 2020 and 2023. WALOUS is land cover, not legal land use, ownership, +tree count, timber volume or water volume. + +Settings: `WALOUS_ENABLED`, `WALOUS_SOURCE_DIR`, +`WALOUS_ANALYSIS_RESOLUTION_M`, `WALOUS_MAX_SIDE_M` and +`WALOUS_MAX_PIXELS`. The SPW flood polygon adapter uses +`SPW_FLOOD_HAZARD_ENABLED` and `SPW_FLOOD_HAZARD_MAPSERVER_URL`. + +The official Walloon 2021-2022 DTM is currently not an implicit runtime asset: +the published 1 m whole-region artifact is about 41 GB and the 0.5 m INSPIRE +artifact about 213 GB. A later operator capacity plan must define storage, +partitioning and refresh before it can be called operational. + Provision the official DOV soil polygons for Mol through the existing vector upload path: diff --git a/backend/app/api/routes/datasets.py b/backend/app/api/routes/datasets.py index cf51dce6..b3ea7691 100644 --- a/backend/app/api/routes/datasets.py +++ b/backend/app/api/routes/datasets.py @@ -97,6 +97,7 @@ from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisi from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService +from app.services.walous_land_cover_service import WalousLandCoverService from app.utils.response import envelope router = APIRouter(prefix="/projects/{project_id}", tags=["datasets"]) @@ -459,6 +460,33 @@ def list_thematic_raster_products(project_id: UUID, db: Session = Depends(get_db return envelope({"items": items, "total": len(items)}) +@router.post("/datasets/walous/acquire", response_model=Envelope[JobRead]) +def acquire_bounded_walous_land_cover( + project_id: UUID, + payload: ThematicRasterAcquireRequest, + db: Session = Depends(get_db), +): + job = JobService.run_sync_job( + db=db, + project_id=project_id, + job_type="raster.walous.acquire", + parameters=payload.model_dump(mode="json"), + operation=lambda: WalousLandCoverService.acquire(db, project_id, payload), + ) + return envelope(job) + + +@router.get( + "/datasets/walous/products", + response_model=Envelope[ItemList[ThematicRasterProductRead]], +) +def list_walous_products(project_id: UUID, db: Session = Depends(get_db)): + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + items = WalousLandCoverService.list_products() + return envelope({"items": items, "total": len(items)}) + + @router.get("/datasets", response_model=Envelope[DatasetList]) def list_datasets( project_id: UUID, @@ -1006,6 +1034,33 @@ def raster_thematic_image( ) +@router.post( + "/datasets/{dataset_id}/raster/walous/select", + response_model=Envelope[ThematicRasterSelectionResponse], +) +def raster_walous_selection( + project_id: UUID, + dataset_id: UUID, + payload: ThematicRasterSelectionRequest, + db: Session = Depends(get_db), +): + return envelope(WalousLandCoverService.analyze(db, project_id, dataset_id, payload)) + + +@router.get("/datasets/{dataset_id}/raster/walous/image") +def raster_walous_image( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +): + content = WalousLandCoverService.render_png(db, project_id, dataset_id) + return Response( + content=content, + media_type="image/png", + headers={"Cache-Control": "private, max-age=86400"}, + ) + + @router.get( "/datasets/{dataset_id}/raster/stats", response_model=Envelope[RasterStatsResponse], diff --git a/backend/app/core/config.py b/backend/app/core/config.py index ad8cec9e..ea10c908 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -160,6 +160,14 @@ class Settings(BaseSettings): ), validation_alias="SPW_PICC_MAPSERVER_URL", ) + spw_flood_hazard_enabled: bool = Field(default=True, validation_alias="SPW_FLOOD_HAZARD_ENABLED") + spw_flood_hazard_mapserver_url: str = Field( + default=( + "https://geoservices.wallonie.be/arcgis/rest/services/" + "EAU/ALEA_INOND/MapServer" + ), + validation_alias="SPW_FLOOD_HAZARD_MAPSERVER_URL", + ) urbis_enabled: bool = Field(default=True, validation_alias="URBIS_ENABLED") urbis_wfs_url: str = Field( default="https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows", @@ -273,6 +281,19 @@ class Settings(BaseSettings): thematic_raster_max_pixels: int = Field(default=30_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS") thematic_raster_timeout_seconds: int = Field(default=300, ge=1, validation_alias="THEMATIC_RASTER_TIMEOUT_SECONDS") thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB") + walous_enabled: bool = Field(default=True, validation_alias="WALOUS_ENABLED") + walous_source_dir: str = Field( + default="/app/storage/source-cache/walous", + validation_alias="WALOUS_SOURCE_DIR", + ) + walous_analysis_resolution_m: float = Field( + default=10.0, + ge=1.0, + le=100.0, + validation_alias="WALOUS_ANALYSIS_RESOLUTION_M", + ) + walous_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="WALOUS_MAX_SIDE_M") + walous_max_pixels: int = Field(default=36_000_000, ge=1, validation_alias="WALOUS_MAX_PIXELS") redis_url: str | None = Field(default=None, validation_alias="REDIS_URL") log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL") sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL") diff --git a/backend/app/schemas/thematic_raster.py b/backend/app/schemas/thematic_raster.py index 6d32cdd8..405dfdb5 100644 --- a/backend/app/schemas/thematic_raster.py +++ b/backend/app/schemas/thematic_raster.py @@ -2,7 +2,7 @@ from __future__ import annotations from uuid import UUID -from pydantic import BaseModel +from pydantic import BaseModel, Field from .operations import VectorSelectionBBox @@ -32,6 +32,30 @@ class ThematicRasterProductRead(BaseModel): legend_max_label: str included_source_values: list[int] limitation_message: str + analysis_resolution_m: float | None = None + coverage_zones: list[str] = Field(default_factory=list) + configured: bool = True + status: str = "configured" + + +class WalousAcquisitionResult(BaseModel): + output_dataset_id: UUID + reused: bool + provider: str + product_key: str + display_name: str + theme: str + metric_kind: str + resolution_m: float + width: int + height: int + valid_pixel_count: int + bbox_epsg4326: list[float] + bbox_epsg3812: list[float] + observation_year: int + source_value_unit: str + attribution: str + limitation_message: str class ThematicRasterAcquisitionResult(BaseModel): diff --git a/backend/app/services/coverage_registry_service.py b/backend/app/services/coverage_registry_service.py index 9ae42407..abb292ef 100644 --- a/backend/app/services/coverage_registry_service.py +++ b/backend/app/services/coverage_registry_service.py @@ -252,11 +252,11 @@ SOURCE_DEFINITIONS = ( attribution="Service public de Wallonie", license_note="Consult the license of each Geoportail Wallonie product.", limitation_message=( - "Bounded PICC buildings, road axes, hydrography and operator-imported SPW bathymetry are operational; " - "other Walloon themes remain unavailable until separately governed." + "Bounded PICC buildings, road axes and hydrography, the legally current flood-hazard polygons, " + "and operator-imported SPW bathymetry are operational; other Walloon themes remain separately governed." ), - materialized_source_names=("spw_picc", "spw_bathymetry"), - operational_themes=("buildings", "roads", "surface_water", "bathymetry"), + materialized_source_names=("spw_picc", "spw_flood_hazard", "spw_walous_land_cover", "spw_bathymetry"), + operational_themes=("buildings", "roads", "surface_water", "land_cover_use", "flood_climate", "bathymetry"), ), _contract( source_name="urbis", @@ -375,6 +375,8 @@ REGIONAL_THEME_DATASETS: dict[str, dict[str, dict[str, tuple[str, ...]]]] = { "buildings": {"spw_picc": ("buildings",)}, "roads": {"spw_picc": ("roads",)}, "surface_water": {"spw_picc": ("water",)}, + "land_cover_use": {"spw_walous_land_cover": ()}, + "flood_climate": {"spw_flood_hazard": ("flood_hazard",)}, "bathymetry": {"spw_bathymetry": ()}, }, "urbis": { diff --git a/backend/app/services/official_vector_acquisition_service.py b/backend/app/services/official_vector_acquisition_service.py index 8c8e8553..8dd7a006 100644 --- a/backend/app/services/official_vector_acquisition_service.py +++ b/backend/app/services/official_vector_acquisition_service.py @@ -460,6 +460,75 @@ class OfficialVectorAcquisitionService: identity_field="GEOREF_ID", requires_coverage_area=True, ), + OfficialVectorProduct( + key="spw_flood_hazard_2021", + display_name="Waalse overstromingsgevaarkaart 2021", + theme="flood_hazard", + provider="Service public de Wallonie", + source_name="spw_flood_hazard", + reference_layer_name="flood_hazard", + service_type="ArcGIS REST", + collection="2", + source_crs="EPSG:31370", + source_version="2021-03-04", + observation_label="Juridisch geldende toestand 2021", + authority_level="authoritative", + catalog_url=( + "https://geoportail.wallonie.be/catalogue/" + "14084108-2c7b-4091-b62d-ff0fc235213a.html" + ), + attribution="Service public de Wallonie (SPW) - Cartographie de l'alea d'inondation", + license_note="CC BY 4.0; cite SPW and identify modifications.", + limitation_message=( + "Juridische gevarenkaart voor overstroming door waterloopoverloop en afstroming. " + "Dit is geen actuele overstroming, gemeten waterdiepte, voorspelling of bathymetrie." + ), + source="SPW flood-hazard ArcGIS REST", + observed_at=datetime(2021, 3, 4, tzinfo=UTC), + valid_from=datetime(2021, 3, 4, tzinfo=UTC), + valid_to=None, + primary_metric={ + "metric_key": "flood_hazard_area", + "method": "intersection_area", + "label": "Oppervlakte met overstromingsgevaar", + "unit": "ha", + "geometry_dimension": 2, + "is_estimate": False, + }, + selection_metrics=( + { + "metric_key": "flood_hazard_high_area", + "method": "intersection_area", + "label": "Hoog overstromingsgevaar", + "unit": "ha", + "geometry_dimension": 2, + "filter_property": "CLASSEMENT", + "filter_values": [130, 230, 330, "130", "230", "330"], + }, + { + "metric_key": "flood_hazard_medium_area", + "method": "intersection_area", + "label": "Middelgroot overstromingsgevaar", + "unit": "ha", + "geometry_dimension": 2, + "filter_property": "CLASSEMENT", + "filter_values": [120, 220, 320, "120", "220", "320"], + }, + { + "metric_key": "flood_hazard_low_area", + "method": "intersection_area", + "label": "Laag overstromingsgevaar", + "unit": "ha", + "geometry_dimension": 2, + "filter_property": "CLASSEMENT", + "filter_values": [110, 210, 310, "110", "210", "310"], + }, + ), + coverage_zones=("wallonia",), + endpoint_kind="spw_flood_arcgis", + identity_field="LOCALID", + requires_coverage_area=True, + ), OfficialVectorProduct( key="urbis_buildings", display_name="UrbIS buildings", @@ -869,6 +938,12 @@ class OfficialVectorAcquisitionService: message="Bounded SPW PICC acquisition is disabled", status_code=503, ) + if product.endpoint_kind == "spw_flood_arcgis" and not settings.spw_flood_hazard_enabled: + raise AppError( + code="SPW_FLOOD_HAZARD_NOT_CONFIGURED", + message="Bounded SPW flood-hazard acquisition is disabled", + status_code=503, + ) if product.endpoint_kind == "urbis_wfs" and not settings.urbis_enabled: raise AppError( code="URBIS_NOT_CONFIGURED", @@ -1132,7 +1207,11 @@ class OfficialVectorAcquisitionService: "f": "geojson", } ) - base = settings.spw_picc_mapserver_url.rstrip("/") + base = ( + settings.spw_flood_hazard_mapserver_url + if product.endpoint_kind == "spw_flood_arcgis" + else settings.spw_picc_mapserver_url + ).rstrip("/") return f"{base}/{product.collection}/query?{query}" @staticmethod @@ -1178,7 +1257,7 @@ class OfficialVectorAcquisitionService: tuple(scope_metric.bounds), start_index, ) - if product.endpoint_kind == "spw_arcgis": + if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis"}: return OfficialVectorAcquisitionService._spw_url( settings, product, @@ -1209,6 +1288,7 @@ class OfficialVectorAcquisitionService: "bwk_wfs": settings.bwk_wfs_url, "dov_wfs": settings.dov_soil_wfs_url, "spw_arcgis": settings.spw_picc_mapserver_url, + "spw_flood_arcgis": settings.spw_flood_hazard_mapserver_url, "urbis_wfs": settings.urbis_wfs_url, }.get(product.endpoint_kind) if configured_url is None: @@ -1220,7 +1300,7 @@ class OfficialVectorAcquisitionService: base = urlparse(configured_url) expected_path = ( f"{base.path.rstrip('/')}/{product.collection}/query" - if product.endpoint_kind == "spw_arcgis" + if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis"} else base.path ) if ( @@ -1361,7 +1441,7 @@ class OfficialVectorAcquisitionService: scope_metric: Any, coverage_scope: str, ) -> dict[str, Any] | None: - if product.endpoint_kind in {"spw_arcgis", "urbis_wfs"}: + if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis", "urbis_wfs"}: return OfficialVectorAcquisitionService._normalize_regional_feature( product, feature, @@ -1601,7 +1681,7 @@ class OfficialVectorAcquisitionService: ) start_index += returned_count arcgis_has_more = payload.get("exceededTransferLimit") is True - if product.endpoint_kind == "spw_arcgis": + if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis"}: if arcgis_has_more and returned_count == 0: raise AppError( code="OFFICIAL_VECTOR_PROVIDER_INCOMPLETE_RESPONSE", diff --git a/backend/app/services/temporal_analysis_service.py b/backend/app/services/temporal_analysis_service.py index 5ea40472..c9cd14cb 100644 --- a/backend/app/services/temporal_analysis_service.py +++ b/backend/app/services/temporal_analysis_service.py @@ -22,7 +22,9 @@ from app.schemas.temporal import ( TemporalSeriesDataset, TemporalSeriesRead, ) +from app.schemas.thematic_raster import ThematicRasterSelectionRequest from app.services.vector_feature_service import VectorFeatureService +from app.services.walous_land_cover_service import WalousLandCoverService class TemporalAnalysisService: @@ -31,6 +33,7 @@ class TemporalAnalysisService: "provision_regional_grb_buildings.py", "provision_regional_grb_context.py", } + SUPPORTED_RASTER_TEMPORAL_SOURCES = {WalousLandCoverService.PROVIDER} @staticmethod def _canonical_observation_snapshots(datasets: list[Dataset]) -> list[Dataset]: @@ -141,6 +144,15 @@ class TemporalAnalysisService: status_code=400, ) + if earlier.dataset_type == "raster" or later.dataset_type == "raster": + return TemporalAnalysisService._compare_walous_rasters( + db, + project_id=project_id, + payload=payload, + earlier=earlier, + later=later, + ) + bbox = payload.bbox.model_dump() selection_area = TemporalAnalysisService._get_selection_area(db, project_id, payload.area_id) selection_geometry = None @@ -259,6 +271,90 @@ class TemporalAnalysisService: raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) return area + @staticmethod + def _compare_walous_rasters( + db: Session, + *, + project_id: UUID, + payload: TemporalComparisonRequest, + earlier: Dataset, + later: Dataset, + ) -> TemporalComparisonResponse: + if { + earlier.dataset_type, + later.dataset_type, + } != {"raster"} or earlier.source_name != WalousLandCoverService.PROVIDER or later.source_name != WalousLandCoverService.PROVIDER: + raise AppError( + code="INCOMPATIBLE_TEMPORAL_DATASET_TYPES", + message="Raster evolution currently supports only two governed WALOUS land-cover snapshots", + status_code=400, + ) + + request = ThematicRasterSelectionRequest(bbox=payload.bbox, area_id=payload.area_id) + summaries: dict[UUID, dict[str, Any]] = {} + + def summarize(dataset: Dataset) -> dict[str, Any]: + cached = summaries.get(dataset.id) + if cached is not None: + return cached + result = WalousLandCoverService.analyze(db, project_id, dataset.id, request) + summary = dict(result["summary"]) + summary["warning"] = result.get("limitation_message") + summaries[dataset.id] = summary + return summary + + earlier_summary = summarize(earlier) + later_summary = summarize(later) + metric_comparisons = TemporalAnalysisService._compare_summary_metrics(earlier_summary, later_summary) + if not metric_comparisons: + raise AppError( + code="INCOMPATIBLE_TEMPORAL_AGGREGATION", + message="WALOUS snapshots use incompatible aggregation semantics", + status_code=400, + ) + primary_key = str(later_summary.get("primary_metric_key") or metric_comparisons[0].metric_key) + primary_metric = next( + (metric for metric in metric_comparisons if metric.metric_key == primary_key), + metric_comparisons[0], + ) + timeline = TemporalAnalysisService._build_timeline( + db, + project_id=project_id, + series_key=str(earlier.temporal_series_key), + fallback_datasets=[earlier, later], + summarize=summarize, + ) + warnings = [ + "WALOUS-evolutie vergelijkt celgebaseerde landbedekkingsoppervlakten; individuele objectwijzigingen zijn niet beschikbaar.", + ] + limitation = str(later_summary.get("warning") or earlier_summary.get("warning") or "").strip() + if limitation: + warnings.append(limitation) + return TemporalComparisonResponse( + temporal_series_key=str(earlier.temporal_series_key), + earlier=TemporalDatasetRef( + id=earlier.id, + name=earlier.name, + observed_at=earlier.observed_at, + source_version=earlier.source_version, + ), + later=TemporalDatasetRef( + id=later.id, + name=later.name, + observed_at=later.observed_at, + source_version=later.source_version, + ), + selection_bbox=payload.bbox, + selection_area_id=payload.area_id, + metric=primary_metric, + metrics=metric_comparisons, + timeline=timeline, + object_changes=TemporalObjectChanges(available=False), + geojson={"type": "FeatureCollection", "features": []}, + warnings=warnings, + generated_at=datetime.now(timezone.utc), + ) + @staticmethod def _summary_metrics(summary: dict[str, Any]) -> list[dict[str, Any]]: configured = summary.get("metrics") @@ -370,10 +466,15 @@ class TemporalAnalysisService: dataset = db.get(Dataset, dataset_id) if not dataset or dataset.project_id != project_id: raise AppError(code="DATASET_NOT_FOUND", message=f"{label} dataset not found", status_code=404) - if dataset.dataset_type not in {"vector", "geojson"}: + supported_vector = dataset.dataset_type in {"vector", "geojson"} + supported_raster = ( + dataset.dataset_type == "raster" + and dataset.source_name in TemporalAnalysisService.SUPPORTED_RASTER_TEMPORAL_SOURCES + ) + if not supported_vector and not supported_raster: raise AppError( - code="DATASET_NOT_VECTOR", - message="Temporal selection comparison currently requires vector datasets", + code="TEMPORAL_DATASET_NOT_SUPPORTED", + message="Temporal comparison requires a vector series or a governed WALOUS raster series", status_code=400, ) if not dataset.temporal_series_key or not dataset.observed_at: diff --git a/backend/app/services/walous_land_cover_service.py b/backend/app/services/walous_land_cover_service.py new file mode 100644 index 00000000..b0ee129b --- /dev/null +++ b/backend/app/services/walous_land_cover_service.py @@ -0,0 +1,551 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +import hashlib +import io +import json +import math +from pathlib import Path +from typing import Any +from uuid import UUID + +from geoalchemy2.shape import to_shape +from pyproj import Transformer +from shapely.geometry import box, mapping +from shapely.ops import transform as shapely_transform + +from app.core.config import Settings, get_settings +from app.core.errors import AppError +from app.models import Area, Dataset, Project +from app.schemas.thematic_raster import ( + ThematicRasterAcquireRequest, + ThematicRasterMetric, + ThematicRasterProductRead, + ThematicRasterSelectionRequest, + ThematicRasterSelectionResponse, + ThematicRasterSelectionSummary, + WalousAcquisitionResult, +) +from app.services.dataset_service import DatasetService + + +@dataclass(frozen=True) +class WalousProduct: + key: str + display_name: str + observation_year: int + source_filename: str + source_version: str + catalog_url: str + download_url: str + source_sha256_filename: str + accuracy_label: str + + +class WalousLandCoverService: + PROVIDER = "spw_walous_land_cover" + SOURCE_CRS = "EPSG:3812" + SOURCE_RESOLUTION_M = 1.0 + SOURCE_VALUE_UNIT = "class_1_11" + THEME = "land_cover_use" + METRIC_KIND = "categorical_area" + NODATA = 255 + ATTRIBUTION = "Service public de Wallonie (SPW), Aerospacelab S.A." + LICENSE_NOTE = "CC BY 4.0; cite the official SPW WALOUS edition and identify modifications." + LIMITATION = ( + "GeoIntel analyseert een nearest-neighbour afgeleide van het officiele 1 m WALOUS-raster op de " + "geconfigureerde analyseresolutie. Oppervlakten zijn celgebaseerde schattingen; de kaart is landbedekking, " + "geen juridisch landgebruik, eigendom, boomtelling of actuele terreinwaarneming." + ) + CLASS_LABELS = { + 1: "Jaarlijks wisselende kruidlaag", + 2: "Jaarronde kruidlaag", + 3: "Naaldbomen hoger dan 3 m", + 4: "Loofbomen hoger dan 3 m", + 5: "Naaldbomen tot 3 m", + 6: "Loofbomen tot 3 m", + 7: "Kale bodem", + 8: "Oppervlaktewater", + 9: "Kunstmatige bodembedekking", + 10: "Spoorweg", + 11: "Kunstmatige constructies boven maaiveld", + } + CLASS_COLORS = { + 1: (236, 202, 73), + 2: (161, 201, 78), + 3: (28, 89, 51), + 4: (52, 132, 72), + 5: (78, 125, 70), + 6: (107, 164, 87), + 7: (194, 165, 119), + 8: (44, 129, 185), + 9: (155, 155, 155), + 10: (68, 68, 68), + 11: (183, 72, 67), + } + + @staticmethod + def _products() -> dict[str, WalousProduct]: + products = ( + WalousProduct( + key="walous_land_cover_2020", + display_name="WALOUS landbedekking 2020", + observation_year=2020, + source_filename="walous_land_cover_2020_3812.tif", + source_version="WAL_OCS_IA__2020", + catalog_url="https://geoportail.wallonie.be/catalogue/47b348f1-6e7a-4baa-963c-0232a43c0cff.html", + download_url=( + "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/" + "47b348f1-6e7a-4baa-963c-0232a43c0cff/WAL_OCS_IA__2020_GEOTIFF_3812.zip" + ), + source_sha256_filename="walous_land_cover_2020_3812.sha256", + accuracy_label="Officiele globale nauwkeurigheid 83,30%", + ), + WalousProduct( + key="walous_land_cover_2023", + display_name="WALOUS landbedekking 2023", + observation_year=2023, + source_filename="walous_land_cover_2023_3812.tif", + source_version="WAL_OCS_IA__2023", + catalog_url="https://geoportail.wallonie.be/catalogue/4e780ba1-463c-478e-95df-d2f1963a150d.html", + download_url=( + "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/" + "4e780ba1-463c-478e-95df-d2f1963a150d/WAL_OCS_IA__2023_GEOTIFF_3812.zip" + ), + source_sha256_filename="walous_land_cover_2023_3812.sha256", + accuracy_label="Officiele globale nauwkeurigheid 87,10%", + ), + ) + return {product.key: product for product in products} + + @staticmethod + def _source_path(settings: Settings, product: WalousProduct) -> Path: + return Path(settings.walous_source_dir) / product.source_filename + + @staticmethod + def list_products(*, settings: Settings | None = None) -> list[dict[str, Any]]: + resolved = settings or get_settings() + result: list[dict[str, Any]] = [] + for product in WalousLandCoverService._products().values(): + configured = resolved.walous_enabled and WalousLandCoverService._source_path(resolved, product).is_file() + result.append( + ThematicRasterProductRead( + key=product.key, + display_name=product.display_name, + theme=WalousLandCoverService.THEME, + metric_kind=WalousLandCoverService.METRIC_KIND, + coverage_id=product.source_version, + native_resolution_m=WalousLandCoverService.SOURCE_RESOLUTION_M, + analysis_resolution_m=resolved.walous_analysis_resolution_m, + source_crs=WalousLandCoverService.SOURCE_CRS, + source_value_unit=WalousLandCoverService.SOURCE_VALUE_UNIT, + observation_year=product.observation_year, + source_version=product.source_version, + catalog_url=product.catalog_url, + attribution=WalousLandCoverService.ATTRIBUTION, + license_note=WalousLandCoverService.LICENSE_NOTE, + legend_min_label="WALOUS klasse 1", + legend_max_label="WALOUS klasse 11", + included_source_values=list(WalousLandCoverService.CLASS_LABELS), + limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.", + coverage_zones=["wallonia"], + configured=configured, + status="configured" if configured else "source_not_provisioned", + ).model_dump() + ) + return result + + @staticmethod + def _product(product_key: str) -> WalousProduct: + product = WalousLandCoverService._products().get(product_key.strip().lower()) + if product is None: + raise AppError( + code="WALOUS_PRODUCT_NOT_SUPPORTED", + message="Select a product from the governed WALOUS registry", + details={"product_key": product_key}, + status_code=422, + ) + return product + + @staticmethod + def _scope_geometry(db, project_id: UUID, payload: ThematicRasterAcquireRequest): + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + if payload.bbox.crs.upper() != "EPSG:4326": + raise AppError(code="INVALID_BBOX_CRS", message="WALOUS acquisition requires EPSG:4326", status_code=400) + values = [payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y] + if not all(math.isfinite(value) for value in values) or values[0] >= values[2] or values[1] >= values[3]: + raise AppError(code="INVALID_BBOX", message="WALOUS selection must be a finite non-empty rectangle", status_code=400) + selection = box(*values) + if payload.area_id is None: + return selection, values + area = db.get(Area, payload.area_id) + if area is None or area.project_id != project_id: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + selection = selection.intersection(to_shape(area.geometry)) + if selection.is_empty or selection.area <= 0: + raise AppError(code="WALOUS_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422) + return selection, values + + @staticmethod + def _read_source_window( + source_path: Path, + scope_4326, + settings: Settings, + ) -> tuple[bytes, dict[str, Any]]: + try: + import numpy as np + import rasterio + from rasterio.enums import Resampling + from rasterio.features import geometry_mask + from rasterio.io import MemoryFile + from rasterio.transform import from_bounds + from rasterio.windows import from_bounds as window_from_bounds + except ImportError as exc: + raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for WALOUS", status_code=503) from exc + + resolution = float(settings.walous_analysis_resolution_m) + transformer = Transformer.from_crs("EPSG:4326", WalousLandCoverService.SOURCE_CRS, always_xy=True) + scope_metric = shapely_transform(transformer.transform, scope_4326) + try: + with rasterio.open(source_path) as source: + if source.crs is None or source.crs.to_epsg() != 3812 or source.count != 1: + raise AppError(code="WALOUS_SOURCE_INVALID", message="WALOUS source must be a one-band EPSG:3812 raster", status_code=409) + if not all(math.isclose(abs(float(value)), 1.0, abs_tol=0.05) for value in source.res): + raise AppError(code="WALOUS_SOURCE_INVALID", message="WALOUS source must retain the official 1 m resolution", status_code=409) + clipped_geometry = scope_metric.intersection(box(*source.bounds)) + if clipped_geometry.is_empty or clipped_geometry.area <= 0: + raise AppError(code="WALOUS_SELECTION_OUTSIDE_COVERAGE", message="Selection does not overlap WALOUS coverage", status_code=422) + min_x, min_y, max_x, max_y = clipped_geometry.bounds + bounds = ( + math.floor(min_x / resolution) * resolution, + math.floor(min_y / resolution) * resolution, + math.ceil(max_x / resolution) * resolution, + math.ceil(max_y / resolution) * resolution, + ) + width_m, height_m = bounds[2] - bounds[0], bounds[3] - bounds[1] + if width_m > settings.walous_max_side_m or height_m > settings.walous_max_side_m: + raise AppError( + code="WALOUS_SELECTION_TOO_LARGE", + message=f"Select no more than {settings.walous_max_side_m:g} by {settings.walous_max_side_m:g} metres", + details={"width_m": width_m, "height_m": height_m}, + status_code=422, + ) + width, height = max(1, round(width_m / resolution)), max(1, round(height_m / resolution)) + if width * height > settings.walous_max_pixels: + raise AppError(code="WALOUS_SELECTION_TOO_LARGE", message="WALOUS selection exceeds the configured cell limit", details={"pixel_count": width * height, "max_pixels": settings.walous_max_pixels}, status_code=422) + window = window_from_bounds(*bounds, transform=source.transform) + band = source.read(1, window=window, out_shape=(height, width), masked=True, resampling=Resampling.nearest) + output_transform = from_bounds(*bounds, width, height) + outside_scope = geometry_mask([mapping(clipped_geometry)], out_shape=(height, width), transform=output_transform, invert=False) + raw = np.asarray(band.filled(WalousLandCoverService.NODATA), dtype="uint8") + invalid = np.ma.getmaskarray(band) | outside_scope + if source.nodata is not None: + invalid |= np.isclose(raw.astype("float64"), float(source.nodata)) + raw[invalid] = WalousLandCoverService.NODATA + valid = raw[raw != WalousLandCoverService.NODATA] + if valid.size == 0: + raise AppError(code="WALOUS_NO_VALID_DATA", message="WALOUS contains no valid cells in this selection", status_code=422) + classes = set(np.unique(valid).astype(int).tolist()) + unexpected = sorted(classes - set(WalousLandCoverService.CLASS_LABELS)) + if unexpected: + raise AppError(code="WALOUS_SOURCE_INVALID_VALUES", message="WALOUS contains classes outside the governed 1-11 legend", details={"unexpected_classes": unexpected}, status_code=409) + profile = { + "driver": "GTiff", + "width": width, + "height": height, + "count": 1, + "dtype": "uint8", + "crs": WalousLandCoverService.SOURCE_CRS, + "transform": output_transform, + "nodata": WalousLandCoverService.NODATA, + "compress": "deflate", + "predictor": 2, + } + with MemoryFile() as memory: + with memory.open(**profile) as output: + output.write(raw, 1) + content = memory.read() + return content, { + "width": width, + "height": height, + "valid_pixel_count": int(valid.size), + "classes_present": sorted(classes), + "bbox_epsg3812": list(bounds), + "source_width": int(source.width), + "source_height": int(source.height), + "source_nodata": None if source.nodata is None else float(source.nodata), + "source_resolution_m": 1.0, + "analysis_resolution_m": resolution, + } + except AppError: + raise + except Exception as exc: + raise AppError(code="WALOUS_SOURCE_READ_FAILED", message="The provisioned WALOUS source could not be read", details={"reason": str(exc)}, status_code=500) from exc + + @staticmethod + def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None: + candidate = ( + db.query(Dataset) + .filter(Dataset.project_id == project_id, Dataset.name == filename, Dataset.source_name == WalousLandCoverService.PROVIDER, Dataset.status == "ready") + .order_by(Dataset.imported_at.desc()) + .first() + ) + return candidate if candidate and candidate.storage_path and Path(candidate.storage_path).is_file() else None + + @staticmethod + def acquire(db, project_id: UUID, payload: ThematicRasterAcquireRequest, *, settings: Settings | None = None) -> dict[str, Any]: + resolved = settings or get_settings() + if not resolved.walous_enabled: + raise AppError(code="WALOUS_NOT_CONFIGURED", message="WALOUS bounded analysis is disabled", status_code=503) + product = WalousLandCoverService._product(payload.product_key) + source_path = WalousLandCoverService._source_path(resolved, product) + if not source_path.is_file(): + raise AppError( + code="WALOUS_SOURCE_NOT_PROVISIONED", + message="The official WALOUS source archive has not been provisioned on this runtime", + details={"expected_path": str(source_path), "operator_command": "python scripts/provision_walous_sources.py --years 2020 2023"}, + status_code=503, + ) + scope, bbox_4326 = WalousLandCoverService._scope_geometry(db, project_id, payload) + identity = { + "product_key": product.key, + "bbox_epsg4326": [round(float(value), 8) for value in bbox_4326], + "area_id": str(payload.area_id) if payload.area_id else None, + "analysis_resolution_m": resolved.walous_analysis_resolution_m, + } + request_hash = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest() + filename = f"walous_{product.observation_year}_{request_hash[:12]}_3812.tif" + if not payload.force_refresh: + cached = WalousLandCoverService._cached_dataset(db, project_id, filename) + if cached is not None: + metadata = cached.source_metadata or {} + return WalousAcquisitionResult( + output_dataset_id=cached.id, + reused=True, + provider=WalousLandCoverService.PROVIDER, + product_key=product.key, + display_name=product.display_name, + theme=WalousLandCoverService.THEME, + metric_kind=WalousLandCoverService.METRIC_KIND, + resolution_m=float(metadata.get("analysis_resolution_m", resolved.walous_analysis_resolution_m)), + width=int((cached.metadata_json or {}).get("width", 0)), + height=int((cached.metadata_json or {}).get("height", 0)), + valid_pixel_count=int(metadata.get("valid_pixel_count", 0)), + bbox_epsg4326=bbox_4326, + bbox_epsg3812=list(metadata.get("bbox_epsg3812") or []), + observation_year=product.observation_year, + source_value_unit=WalousLandCoverService.SOURCE_VALUE_UNIT, + attribution=WalousLandCoverService.ATTRIBUTION, + limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.", + ).model_dump(mode="json") + + content, validation = WalousLandCoverService._read_source_window(source_path, scope, resolved) + source_sha256_path = source_path.with_name(product.source_sha256_filename) + source_sha256 = source_sha256_path.read_text(encoding="ascii").strip().split()[0] if source_sha256_path.is_file() else None + acquired_at = datetime.now(UTC) + observed_at = datetime(product.observation_year, 12, 31, 23, 59, 59, tzinfo=UTC) + spatial_series_hash = hashlib.sha256(json.dumps({"bbox": identity["bbox_epsg4326"], "area_id": identity["area_id"], "resolution": identity["analysis_resolution_m"]}, sort_keys=True).encode()).hexdigest()[:24] + dataset = DatasetService.import_raster_bytes( + db, + project_id=project_id, + area_id=payload.area_id, + filename=filename, + content=content, + source=f"SPW WALOUS {product.source_version} operator-provisioned GeoTIFF", + source_name=WalousLandCoverService.PROVIDER, + temporal_series_key=f"spw:walous:land-cover:{spatial_series_hash}", + observed_at=observed_at, + valid_from=datetime(product.observation_year, 1, 1, tzinfo=UTC), + valid_to=observed_at, + temporal_granularity="year", + source_version=product.source_version, + source_metadata={ + "provider": WalousLandCoverService.PROVIDER, + "service": "official_predefined_dataset_atom", + "product_key": product.key, + "product_display_name": product.display_name, + "theme": WalousLandCoverService.THEME, + "metric_kind": WalousLandCoverService.METRIC_KIND, + "source_crs": WalousLandCoverService.SOURCE_CRS, + "source_resolution_m": WalousLandCoverService.SOURCE_RESOLUTION_M, + "analysis_resolution_m": validation["analysis_resolution_m"], + "source_value_unit": WalousLandCoverService.SOURCE_VALUE_UNIT, + "class_labels": WalousLandCoverService.CLASS_LABELS, + "observation_year": product.observation_year, + "valid_pixel_count": validation["valid_pixel_count"], + "classes_present": validation["classes_present"], + "bbox_epsg4326": bbox_4326, + "bbox_epsg3812": validation["bbox_epsg3812"], + "coverage_zones": ["wallonia"], + "catalog_url": product.catalog_url, + "download_url": product.download_url, + "attribution": WalousLandCoverService.ATTRIBUTION, + "license_note": WalousLandCoverService.LICENSE_NOTE, + "limitation_message": f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.", + }, + provenance_metadata={ + "acquisition": "operator_provisioned_official_archive_bounded_window", + "acquired_at": acquired_at.isoformat(), + "request_hash": request_hash, + "source_filename": product.source_filename, + "source_sha256": source_sha256, + "derived_sha256": hashlib.sha256(content).hexdigest(), + "resampling": "nearest", + "validation": validation, + }, + ) + return WalousAcquisitionResult( + output_dataset_id=dataset.id, + reused=False, + provider=WalousLandCoverService.PROVIDER, + product_key=product.key, + display_name=product.display_name, + theme=WalousLandCoverService.THEME, + metric_kind=WalousLandCoverService.METRIC_KIND, + resolution_m=validation["analysis_resolution_m"], + width=validation["width"], + height=validation["height"], + valid_pixel_count=validation["valid_pixel_count"], + bbox_epsg4326=bbox_4326, + bbox_epsg3812=validation["bbox_epsg3812"], + observation_year=product.observation_year, + source_value_unit=WalousLandCoverService.SOURCE_VALUE_UNIT, + attribution=WalousLandCoverService.ATTRIBUTION, + limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.", + ).model_dump(mode="json") + + @staticmethod + def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> tuple[Dataset, WalousProduct]: + dataset = db.get(Dataset, dataset_id) + if not dataset or dataset.project_id != project_id: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + if dataset.dataset_type != "raster" or dataset.source_name != WalousLandCoverService.PROVIDER: + raise AppError(code="INVALID_WALOUS_DATASET", message="WALOUS analysis requires a governed WALOUS raster", status_code=400) + if dataset.status != "ready" or not dataset.storage_path or not Path(dataset.storage_path).is_file(): + raise AppError(code="DATASET_FILE_MISSING", message="Persisted WALOUS raster is unavailable", status_code=404) + product = WalousLandCoverService._product(str((dataset.source_metadata or {}).get("product_key") or "")) + return dataset, product + + @staticmethod + def _analysis_geometry(db, project_id: UUID, payload: ThematicRasterSelectionRequest): + selection = box(payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y) + if payload.area_id is None: + return selection + area = db.get(Area, payload.area_id) + if area is None or area.project_id != project_id: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + selection = selection.intersection(to_shape(area.geometry)) + if selection.is_empty or selection.area <= 0: + raise AppError(code="WALOUS_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422) + return selection + + @staticmethod + def analyze(db, project_id: UUID, dataset_id: UUID, payload: ThematicRasterSelectionRequest) -> dict[str, Any]: + dataset, product = WalousLandCoverService._load_dataset(db, project_id, dataset_id) + selection_4326 = WalousLandCoverService._analysis_geometry(db, project_id, payload) + try: + import numpy as np + import rasterio + from rasterio.features import geometry_mask + from rasterio.mask import mask + except ImportError as exc: + raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for WALOUS analysis", status_code=503) from exc + try: + with rasterio.open(dataset.storage_path) as source: + transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True) + selection_metric = shapely_transform(transformer.transform, selection_4326) + geometry = selection_metric.intersection(box(*source.bounds)) + if geometry.is_empty or geometry.area <= 0: + raise AppError(code="WALOUS_SELECTION_OUTSIDE_DATASET", message="Selection does not overlap the persisted WALOUS raster", status_code=422) + clipped, transform = mask(source, [mapping(geometry)], crop=True, filled=False, indexes=[1]) + band = np.ma.asarray(clipped[0]) + raw = np.asarray(band.filled(WalousLandCoverService.NODATA), dtype="uint8") + selected = geometry_mask([mapping(geometry)], out_shape=raw.shape, transform=transform, invert=True) + valid = selected & ~np.ma.getmaskarray(band) & (raw != WalousLandCoverService.NODATA) + values = raw[valid] + selected_count = int(selected.sum()) + valid_count = int(values.size) + if not valid_count: + raise AppError(code="WALOUS_NO_VALID_DATA", message="WALOUS contains no valid cells in this selection", status_code=422) + cell_area_m2 = abs(float(source.res[0]) * float(source.res[1])) + except AppError: + raise + except Exception as exc: + raise AppError(code="WALOUS_ANALYSIS_FAILED", message="The persisted WALOUS raster could not be analysed", details={"reason": str(exc)}, status_code=500) from exc + + def area_for(classes: set[int]) -> float: + return float(np.count_nonzero(np.isin(values, list(classes))) * cell_area_m2 / 10_000.0) + + metric_specs = [ + ("land_cover_observed_area_ha", "Gekarteerde landbedekking", set(WalousLandCoverService.CLASS_LABELS)), + ("forest_cover_area_ha", "Boom- en bosbedekking", {3, 4, 5, 6}), + ("surface_water_area_ha", "Oppervlaktewater", {8}), + ("artificial_cover_area_ha", "Kunstmatige bedekking en constructies", {9, 10, 11}), + ("annual_herbaceous_cover_area_ha", "Jaarlijks wisselende kruidlaag", {1}), + ("permanent_herbaceous_cover_area_ha", "Jaarronde kruidlaag", {2}), + ("bare_soil_area_ha", "Kale bodem", {7}), + ] + metrics = [ + ThematicRasterMetric( + metric_key=key, + metric_label=label, + metric_value=round(area_for(classes), 4), + metric_unit="ha", + aggregation_method="nearest_resampled_cells_times_cell_area", + is_estimate=True, + ) + for key, label, classes in metric_specs + ] + primary = metrics[0] + return ThematicRasterSelectionResponse( + dataset_id=dataset.id, + product_key=product.key, + theme=WalousLandCoverService.THEME, + metric_kind=WalousLandCoverService.METRIC_KIND, + selection_bbox=payload.bbox, + selection_area_id=payload.area_id, + selected_cell_count=selected_count, + valid_cell_count=valid_count, + coverage_ratio=round(valid_count / max(1, selected_count), 6), + resolution_m=round(math.sqrt(cell_area_m2), 4), + observation_year=product.observation_year, + summary=ThematicRasterSelectionSummary( + metric_label=primary.metric_label, + metric_value=primary.metric_value, + metric_unit=primary.metric_unit, + aggregation_method=primary.aggregation_method, + primary_metric_key=primary.metric_key, + metrics=metrics, + ), + unsupported_metrics=["legal_land_use", "ownership", "tree_count", "timber_volume", "water_volume"], + limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.", + generated_at=datetime.now(UTC).isoformat(), + ).model_dump(mode="json") + + @staticmethod + def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes: + dataset, _product = WalousLandCoverService._load_dataset(db, project_id, dataset_id) + try: + import numpy as np + import rasterio + from PIL import Image + from rasterio.enums import Resampling + except ImportError as exc: + raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio, numpy and Pillow are required for WALOUS rendering", status_code=503) from exc + with rasterio.open(dataset.storage_path) as source: + scale = min(1.0, max_dimension / max(source.width, source.height)) + width, height = max(1, round(source.width * scale)), max(1, round(source.height * scale)) + values = source.read(1, out_shape=(height, width), masked=True, resampling=Resampling.nearest) + raw = np.asarray(values.filled(WalousLandCoverService.NODATA), dtype="uint8") + rgba = np.zeros((height, width, 4), dtype="uint8") + for value, color in WalousLandCoverService.CLASS_COLORS.items(): + selected = raw == value + rgba[:, :, 0][selected] = color[0] + rgba[:, :, 1][selected] = color[1] + rgba[:, :, 2][selected] = color[2] + rgba[:, :, 3][selected] = 205 + output = io.BytesIO() + Image.fromarray(rgba).save(output, format="PNG", optimize=True) + return output.getvalue() diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py index fd015306..9843c74d 100644 --- a/backend/tests/test_docker_runtime_config.py +++ b/backend/tests/test_docker_runtime_config.py @@ -95,6 +95,7 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None "activate_promoted_yolo_candidate.py", "manage_grb_refresh.py", "orthophoto_release_preflight.py", + "provision_walous_sources.py", } for script_name in required_runtime_scripts: assert f"COPY scripts/{script_name} /app/scripts/{script_name}" in dockerfile @@ -149,6 +150,24 @@ def test_env_example_uses_runtime_env_names_read_by_backend_and_frontend() -> No assert "VITE_API_PROXY_TARGET=http://localhost:8000" in env_example +def test_walloon_runtime_settings_are_editable_in_compose_and_unraid() -> None: + files = [ + (ROOT / "docker-compose.yml").read_text(encoding="utf-8"), + (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8"), + (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8"), + (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8"), + (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(encoding="utf-8"), + ] + for content in files: + assert "SPW_FLOOD_HAZARD_ENABLED" in content + assert "SPW_FLOOD_HAZARD_MAPSERVER_URL" in content + assert "WALOUS_ENABLED" in content + assert "WALOUS_SOURCE_DIR" in content + assert "WALOUS_ANALYSIS_RESOLUTION_M" in content + assert "WALOUS_MAX_SIDE_M" in content + assert "WALOUS_MAX_PIXELS" in content + + def test_frontend_uses_same_origin_api_proxy_by_default() -> None: api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8") nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8") diff --git a/backend/tests/test_post_rc_regional_official_vector.py b/backend/tests/test_post_rc_regional_official_vector.py index 945e07b3..a6aa86f9 100644 --- a/backend/tests/test_post_rc_regional_official_vector.py +++ b/backend/tests/test_post_rc_regional_official_vector.py @@ -116,6 +116,9 @@ def test_regional_product_registry_is_explicit_and_source_specific() -> None: ] assert products["spw_picc_waterways"]["collection"] == "28" assert products["spw_picc_water_surfaces"]["collection"] == "30" + assert products["spw_flood_hazard_2021"]["collection"] == "2" + assert products["spw_flood_hazard_2021"]["theme"] == "flood_hazard" + assert products["spw_flood_hazard_2021"]["coverage_zones"] == ["wallonia"] assert products["urbis_buildings"]["coverage_zones"] == ["brussels"] assert products["urbis_buildings"]["license_note"] == "Buildings are published under CC0." assert "FPS Finance" in products["urbis_cadastral_parcels"]["license_note"] @@ -252,6 +255,82 @@ def test_spw_arcgis_paging_is_bounded_stable_and_clipped() -> None: assert all(item["properties"]["clipped_area_ha"] > 0 for item in features) +def test_spw_flood_hazard_uses_separate_governed_endpoint_and_persists_classification() -> None: + product = OfficialVectorAcquisitionService._product("spw_flood_hazard_2021") + scope = Polygon( + [(4.55, 50.58), (4.56, 50.58), (4.56, 50.59), (4.55, 50.59), (4.55, 50.58)] + ) + scope_metric = Polygon( + [_TO_LAMBERT72.transform(x, y) for x, y in scope.exterior.coords] + ) + + def opener(raw_request, timeout): + assert timeout == 180 + parsed = urlparse(raw_request.full_url) + assert parsed.path.endswith("/EAU/ALEA_INOND/MapServer/2/query") + query = parse_qs(parsed.query) + assert query["outSR"] == ["4326"] + return JsonResponse( + { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": 7, + "geometry": { + "type": "Polygon", + "coordinates": [[ + [4.551, 50.581], + [4.559, 50.581], + [4.559, 50.589], + [4.551, 50.589], + [4.551, 50.581], + ]], + }, + "properties": { + "OBJECTID": 7, + "LOCALID": "ALEA-7", + "TYPEALEA": "Debordement", + "CLASSEMENT": 130, + "MILLESIME": 2021, + }, + } + ], + "exceededTransferLimit": False, + } + ) + + features, transfer = OfficialVectorAcquisitionService._fetch_features( + product, + scope, + scope_metric, + "wallonia", + Settings(_env_file=None), + opener, + ) + + assert transfer["feature_count"] == 1 + assert features[0]["id"] == "2:ALEA-7" + assert features[0]["properties"]["CLASSEMENT"] == 130 + assert features[0]["properties"]["source_name"] == "spw_flood_hazard" + assert features[0]["properties"]["clipped_area_ha"] > 0 + + +def test_spw_flood_hazard_can_be_disabled_independently() -> None: + project_id = uuid4() + db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")}) + + with pytest.raises(AppError) as exc_info: + OfficialVectorAcquisitionService.acquire( + db, + project_id, + request("spw_flood_hazard_2021", (4.55, 50.58, 4.56, 50.59)), + settings=Settings(_env_file=None, SPW_FLOOD_HAZARD_ENABLED=False), + ) + + assert exc_info.value.code == "SPW_FLOOD_HAZARD_NOT_CONFIGURED" + + def test_regional_products_require_the_persisted_authoritative_coverage_area() -> None: project_id = uuid4() db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")}) diff --git a/backend/tests/test_sprint240_official_flemish_themes.py b/backend/tests/test_sprint240_official_flemish_themes.py index 1fedc81d..8e6eb22e 100644 --- a/backend/tests/test_sprint240_official_flemish_themes.py +++ b/backend/tests/test_sprint240_official_flemish_themes.py @@ -130,8 +130,9 @@ def test_product_registries_expose_honest_forest_agriculture_nature_and_soil() - "dov_soil_types", "spw_picc_buildings", "spw_picc_roads", - "spw_picc_waterways", - "spw_picc_water_surfaces", + "spw_picc_waterways", + "spw_picc_water_surfaces", + "spw_flood_hazard_2021", "urbis_buildings", "urbis_cadastral_parcels", "urbis_street_axes", @@ -405,7 +406,7 @@ def test_official_vector_routes_and_frontend_use_canonical_backend_path(monkeypa assert products_response.status_code == 200 assert set(products_response.json()) == {"data"} - assert products_response.json()["data"]["total"] == 12 + assert products_response.json()["data"]["total"] == 13 assert acquire_response.status_code == 200 assert set(acquire_response.json()) == {"data"} assert acquire_response.json()["data"]["job_type"] == "vector.official.acquire" diff --git a/backend/tests/test_walous_land_cover_service.py b/backend/tests/test_walous_land_cover_service.py new file mode 100644 index 00000000..c28a22e4 --- /dev/null +++ b/backend/tests/test_walous_land_cover_service.py @@ -0,0 +1,345 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace +from uuid import uuid4 + +import numpy as np +from fastapi.testclient import TestClient +from pyproj import Transformer +import rasterio +from rasterio.transform import from_origin + +from app.core.config import Settings +from app.db.session import get_db +from app.main import app +from app.models import Dataset, Job, Project +from app.schemas.thematic_raster import ThematicRasterAcquireRequest, ThematicRasterSelectionRequest +from app.schemas.temporal import TemporalComparisonRequest +from app.services.dataset_service import DatasetService +from app.services.temporal_analysis_service import TemporalAnalysisService +from app.services.walous_land_cover_service import WalousLandCoverService + + +class FakeQuery: + def filter(self, *_args): + return self + + def order_by(self, *_args): + return self + + def first(self): + return None + + +class FakeSession: + def __init__(self, project, dataset=None): + self.project = project + self.dataset = dataset + self.added = [] + + def get(self, model, row_id): + if model is Project and row_id == self.project.id: + return self.project + if model is Dataset and self.dataset is not None and row_id == self.dataset.id: + return self.dataset + match = next((item for item in self.added if isinstance(item, model) and item.id == row_id), None) + if match is not None: + return match + return None + + def query(self, _model): + return FakeQuery() + + def add(self, row): + self.added.append(row) + + def commit(self): + return None + + def rollback(self): + return None + + def refresh(self, row): + return row + + +def make_source(path: Path) -> tuple[list[float], np.ndarray]: + to_3812 = Transformer.from_crs("EPSG:4326", "EPSG:3812", always_xy=True) + to_4326 = Transformer.from_crs("EPSG:3812", "EPSG:4326", always_xy=True) + x, y = to_3812.transform(4.85, 50.45) + transform = from_origin(x, y + 100, 1, 1) + values = np.ones((100, 100), dtype="uint8") + values[:, 20:40] = 4 + values[:, 40:50] = 8 + values[:, 50:70] = 9 + values[:, 70:] = 2 + with rasterio.open( + path, + "w", + driver="GTiff", + width=100, + height=100, + count=1, + dtype="uint8", + crs="EPSG:3812", + transform=transform, + nodata=255, + ) as target: + target.write(values, 1) + min_lon, min_lat = to_4326.transform(x, y) + max_lon, max_lat = to_4326.transform(x + 100, y + 100) + return [min_lon, min_lat, max_lon, max_lat], values + + +def settings(source_dir: Path) -> Settings: + return Settings( + _env_file=None, + WALOUS_SOURCE_DIR=str(source_dir), + WALOUS_ANALYSIS_RESOLUTION_M=10, + WALOUS_MAX_SIDE_M=60_000, + WALOUS_MAX_PIXELS=1_000_000, + ) + + +def test_walous_registry_reports_real_provisioning_state(tmp_path: Path) -> None: + before = {item["key"]: item for item in WalousLandCoverService.list_products(settings=settings(tmp_path))} + assert before["walous_land_cover_2023"]["status"] == "source_not_provisioned" + make_source(tmp_path / "walous_land_cover_2023_3812.tif") + after = {item["key"]: item for item in WalousLandCoverService.list_products(settings=settings(tmp_path))} + assert after["walous_land_cover_2023"]["configured"] is True + assert after["walous_land_cover_2023"]["source_crs"] == "EPSG:3812" + assert after["walous_land_cover_2023"]["native_resolution_m"] == 1.0 + assert after["walous_land_cover_2023"]["analysis_resolution_m"] == 10.0 + assert after["walous_land_cover_2023"]["coverage_zones"] == ["wallonia"] + + +def test_walous_acquisition_reads_real_classes_and_persists_provenance(tmp_path: Path, monkeypatch) -> None: + bbox, _values = make_source(tmp_path / "walous_land_cover_2023_3812.tif") + project = Project(id=uuid4(), name="Belgium") + db = FakeSession(project) + captured = {} + output_id = uuid4() + + def persist(_db, **kwargs): + captured.update(kwargs) + return SimpleNamespace(id=output_id) + + monkeypatch.setattr(DatasetService, "import_raster_bytes", persist) + result = WalousLandCoverService.acquire( + db, + project.id, + ThematicRasterAcquireRequest( + bbox={"min_x": bbox[0], "min_y": bbox[1], "max_x": bbox[2], "max_y": bbox[3], "crs": "EPSG:4326"}, + product_key="walous_land_cover_2023", + force_refresh=True, + ), + settings=settings(tmp_path), + ) + + assert result["output_dataset_id"] == str(output_id) + assert result["resolution_m"] == 10 + assert captured["source_name"] == "spw_walous_land_cover" + assert captured["source_metadata"]["classes_present"] == [1, 2, 4, 8, 9] + assert captured["provenance_metadata"]["resampling"] == "nearest" + assert captured["temporal_series_key"].startswith("spw:walous:land-cover:") + assert captured["observed_at"].year == 2023 + + +def test_walous_analysis_returns_semantic_area_metrics(tmp_path: Path, monkeypatch) -> None: + bbox, _values = make_source(tmp_path / "walous_land_cover_2023_3812.tif") + project = Project(id=uuid4(), name="Belgium") + output_id = uuid4() + captured = {} + + def persist(_db, **kwargs): + captured.update(kwargs) + return SimpleNamespace(id=output_id) + + monkeypatch.setattr(DatasetService, "import_raster_bytes", persist) + db = FakeSession(project) + payload = ThematicRasterAcquireRequest( + bbox={"min_x": bbox[0], "min_y": bbox[1], "max_x": bbox[2], "max_y": bbox[3], "crs": "EPSG:4326"}, + product_key="walous_land_cover_2023", + force_refresh=True, + ) + WalousLandCoverService.acquire(db, project.id, payload, settings=settings(tmp_path)) + persisted_path = tmp_path / "derived.tif" + persisted_path.write_bytes(captured["content"]) + dataset = Dataset( + id=output_id, + project_id=project.id, + name="derived.tif", + dataset_type="raster", + source="SPW WALOUS", + source_name="spw_walous_land_cover", + source_metadata=captured["source_metadata"], + provenance_metadata=captured["provenance_metadata"], + storage_path=str(persisted_path), + status="ready", + ) + db.dataset = dataset + result = WalousLandCoverService.analyze( + db, + project.id, + output_id, + ThematicRasterSelectionRequest(bbox=payload.bbox), + ) + metrics = {item["metric_key"]: item["metric_value"] for item in result["summary"]["metrics"]} + + assert result["metric_kind"] == "categorical_area" + assert metrics["land_cover_observed_area_ha"] > 0 + assert metrics["forest_cover_area_ha"] > 0 + assert metrics["surface_water_area_ha"] > 0 + assert metrics["artificial_cover_area_ha"] > 0 + assert "water_volume" in result["unsupported_metrics"] + + +def test_walous_render_png_uses_governed_class_colours(tmp_path: Path) -> None: + bbox, _values = make_source(tmp_path / "walous_land_cover_2023_3812.tif") + project = Project(id=uuid4(), name="Belgium") + dataset = Dataset( + id=uuid4(), + project_id=project.id, + name="walous.tif", + dataset_type="raster", + source="SPW WALOUS", + source_name="spw_walous_land_cover", + source_metadata={"product_key": "walous_land_cover_2023", "bbox_epsg4326": bbox}, + storage_path=str(tmp_path / "walous_land_cover_2023_3812.tif"), + status="ready", + ) + db = FakeSession(project, dataset) + + rendered = WalousLandCoverService.render_png(db, project.id, dataset.id) + + assert rendered.startswith(b"\x89PNG\r\n\x1a\n") + + +def test_walous_temporal_comparison_reuses_persisted_raster_metrics(monkeypatch) -> None: + project_id = uuid4() + earlier = Dataset( + id=uuid4(), + project_id=project_id, + name="walous-2020.tif", + dataset_type="raster", + source="SPW WALOUS", + source_name="spw_walous_land_cover", + temporal_series_key="spw:walous:land-cover:selection", + observed_at=datetime(2020, 12, 31, 23, 59, 59, tzinfo=timezone.utc), + source_version="WAL_OCS_IA__2020", + ) + later = Dataset( + id=uuid4(), + project_id=project_id, + name="walous-2023.tif", + dataset_type="raster", + source="SPW WALOUS", + source_name="spw_walous_land_cover", + temporal_series_key=earlier.temporal_series_key, + observed_at=datetime(2023, 12, 31, 23, 59, 59, tzinfo=timezone.utc), + source_version="WAL_OCS_IA__2023", + ) + rows = {earlier.id: earlier, later.id: later} + + class TemporalSession: + def get(self, model, row_id): + return rows.get(row_id) if model is Dataset else None + + def analyze(_db, _project_id, dataset_id, _payload): + value = 4.0 if dataset_id == earlier.id else 5.5 + return { + "summary": { + "metric_label": "Gekarteerde landbedekking", + "metric_value": value, + "metric_unit": "ha", + "aggregation_method": "nearest_resampled_cells_times_cell_area", + "primary_metric_key": "land_cover_observed_area_ha", + "metrics": [{ + "metric_key": "land_cover_observed_area_ha", + "metric_label": "Gekarteerde landbedekking", + "metric_value": value, + "metric_unit": "ha", + "aggregation_method": "nearest_resampled_cells_times_cell_area", + "is_estimate": True, + }], + }, + "limitation_message": "Cell-based estimate.", + } + + monkeypatch.setattr(WalousLandCoverService, "analyze", analyze) + payload = TemporalComparisonRequest( + earlier_dataset_id=earlier.id, + later_dataset_id=later.id, + bbox={"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"}, + ) + + result = TemporalAnalysisService.compare(TemporalSession(), project_id=project_id, payload=payload) + + assert result.metric.earlier_value == 4.0 + assert result.metric.later_value == 5.5 + assert result.metric.absolute_change == 1.5 + assert result.object_changes.available is False + + +def test_walous_api_routes_use_canonical_envelopes(monkeypatch) -> None: + project = Project(id=uuid4(), name="Belgium") + dataset_id = uuid4() + db = FakeSession(project) + monkeypatch.setattr( + WalousLandCoverService, + "acquire", + lambda *_args, **_kwargs: {"output_dataset_id": str(dataset_id), "provider": WalousLandCoverService.PROVIDER}, + ) + monkeypatch.setattr( + WalousLandCoverService, + "analyze", + lambda *_args, **_kwargs: { + "dataset_id": str(dataset_id), + "product_key": "walous_land_cover_2023", + "theme": "land_cover_use", + "metric_kind": "categorical_area", + "selection_bbox": {"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"}, + "selected_cell_count": 100, + "valid_cell_count": 100, + "coverage_ratio": 1.0, + "resolution_m": 10.0, + "observation_year": 2023, + "summary": { + "metric_label": "Gekarteerde landbedekking", + "metric_value": 1.0, + "metric_unit": "ha", + "aggregation_method": "nearest_resampled_cells_times_cell_area", + "primary_metric_key": "land_cover_observed_area_ha", + "metrics": [], + }, + "unsupported_metrics": ["water_volume"], + "limitation_message": "Cell-based estimate.", + "generated_at": "2026-07-22T00:00:00Z", + }, + ) + app.dependency_overrides[get_db] = lambda: db + try: + client = TestClient(app) + products = client.get(f"/api/v1/projects/{project.id}/datasets/walous/products") + acquisition = client.post( + f"/api/v1/projects/{project.id}/datasets/walous/acquire", + json={ + "bbox": {"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"}, + "product_key": "walous_land_cover_2023", + }, + ) + selection = client.post( + f"/api/v1/projects/{project.id}/datasets/{dataset_id}/raster/walous/select", + json={"bbox": {"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"}}, + ) + finally: + app.dependency_overrides.clear() + + assert products.status_code == 200 and set(products.json()) == {"data"} + assert products.json()["data"]["total"] == 2 + assert acquisition.status_code == 200 and set(acquisition.json()) == {"data"} + assert acquisition.json()["data"]["job_type"] == "raster.walous.acquire" + assert selection.status_code == 200 and selection.json()["data"]["theme"] == "land_cover_use" + assert any(isinstance(item, Job) for item in db.added) diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index dc52f706..e2abff5a 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -101,6 +101,7 @@ COPY scripts/provision_flanders_geographic_scope.py /app/scripts/provision_fland COPY scripts/provision_flanders_bathymetry_profiles.py /app/scripts/provision_flanders_bathymetry_profiles.py COPY scripts/probe_mdk_bathymetry.py /app/scripts/probe_mdk_bathymetry.py COPY scripts/import_spw_bathymetry.py /app/scripts/import_spw_bathymetry.py +COPY scripts/provision_walous_sources.py /app/scripts/provision_walous_sources.py COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py diff --git a/deploy/unraid/geointel-unraid-template.xml b/deploy/unraid/geointel-unraid-template.xml index a82ae6e3..d2afde7f 100644 --- a/deploy/unraid/geointel-unraid-template.xml +++ b/deploy/unraid/geointel-unraid-template.xml @@ -52,6 +52,8 @@ https://www.dov.vlaanderen.be/geoserver/wfs true https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer + true + https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer true https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows 10 @@ -103,6 +105,11 @@ 30000000 300 160 + true + /app/storage/source-cache/walous + 10 + 60000 + 36000000 false /app/models diff --git a/deploy/unraid/geointel.env.example b/deploy/unraid/geointel.env.example index 2b018c8b..19639992 100644 --- a/deploy/unraid/geointel.env.example +++ b/deploy/unraid/geointel.env.example @@ -55,6 +55,8 @@ BWK_WFS_URL=https://geo.api.vlaanderen.be/BWK/wfs DOV_SOIL_WFS_URL=https://www.dov.vlaanderen.be/geoserver/wfs SPW_PICC_ENABLED=true SPW_PICC_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer +SPW_FLOOD_HAZARD_ENABLED=true +SPW_FLOOD_HAZARD_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer URBIS_ENABLED=true URBIS_WFS_URL=https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows OFFICIAL_VECTOR_MIN_SIDE_M=10 @@ -112,6 +114,11 @@ THEMATIC_RASTER_MAX_SIDE_M=60000 THEMATIC_RASTER_MAX_PIXELS=30000000 THEMATIC_RASTER_TIMEOUT_SECONDS=300 THEMATIC_RASTER_MAX_RESPONSE_MB=160 +WALOUS_ENABLED=true +WALOUS_SOURCE_DIR=/app/storage/source-cache/walous +WALOUS_ANALYSIS_RESOLUTION_M=10 +WALOUS_MAX_SIDE_M=60000 +WALOUS_MAX_PIXELS=36000000 # Optional configured-YOLO runtime. Keep disabled unless a local model is mounted. GEOINTEL_INSTALL_AI=false diff --git a/deploy/unraid/run-dockerman-container.sh b/deploy/unraid/run-dockerman-container.sh index 59e0ec04..71abaa59 100644 --- a/deploy/unraid/run-dockerman-container.sh +++ b/deploy/unraid/run-dockerman-container.sh @@ -53,6 +53,8 @@ BWK_WFS_URL="${BWK_WFS_URL:-https://geo.api.vlaanderen.be/BWK/wfs}" DOV_SOIL_WFS_URL="${DOV_SOIL_WFS_URL:-https://www.dov.vlaanderen.be/geoserver/wfs}" SPW_PICC_ENABLED="${SPW_PICC_ENABLED:-true}" SPW_PICC_MAPSERVER_URL="${SPW_PICC_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer}" +SPW_FLOOD_HAZARD_ENABLED="${SPW_FLOOD_HAZARD_ENABLED:-true}" +SPW_FLOOD_HAZARD_MAPSERVER_URL="${SPW_FLOOD_HAZARD_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer}" URBIS_ENABLED="${URBIS_ENABLED:-true}" URBIS_WFS_URL="${URBIS_WFS_URL:-https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows}" OFFICIAL_VECTOR_MIN_SIDE_M="${OFFICIAL_VECTOR_MIN_SIDE_M:-10}" @@ -104,6 +106,11 @@ THEMATIC_RASTER_MAX_SIDE_M="${THEMATIC_RASTER_MAX_SIDE_M:-60000}" THEMATIC_RASTER_MAX_PIXELS="${THEMATIC_RASTER_MAX_PIXELS:-30000000}" THEMATIC_RASTER_TIMEOUT_SECONDS="${THEMATIC_RASTER_TIMEOUT_SECONDS:-300}" THEMATIC_RASTER_MAX_RESPONSE_MB="${THEMATIC_RASTER_MAX_RESPONSE_MB:-160}" +WALOUS_ENABLED="${WALOUS_ENABLED:-true}" +WALOUS_SOURCE_DIR="${WALOUS_SOURCE_DIR:-/app/storage/source-cache/walous}" +WALOUS_ANALYSIS_RESOLUTION_M="${WALOUS_ANALYSIS_RESOLUTION_M:-10}" +WALOUS_MAX_SIDE_M="${WALOUS_MAX_SIDE_M:-60000}" +WALOUS_MAX_PIXELS="${WALOUS_MAX_PIXELS:-36000000}" YOLO_ENABLED="${YOLO_ENABLED:-false}" YOLO_MODELS_DIR="${YOLO_MODELS_DIR:-/app/models}" YOLO_MODEL_PATH="${YOLO_MODEL_PATH:-}" @@ -249,6 +256,8 @@ docker run -d \ -e DOV_SOIL_WFS_URL="$DOV_SOIL_WFS_URL" \ -e SPW_PICC_ENABLED="$SPW_PICC_ENABLED" \ -e SPW_PICC_MAPSERVER_URL="$SPW_PICC_MAPSERVER_URL" \ + -e SPW_FLOOD_HAZARD_ENABLED="$SPW_FLOOD_HAZARD_ENABLED" \ + -e SPW_FLOOD_HAZARD_MAPSERVER_URL="$SPW_FLOOD_HAZARD_MAPSERVER_URL" \ -e URBIS_ENABLED="$URBIS_ENABLED" \ -e URBIS_WFS_URL="$URBIS_WFS_URL" \ -e OFFICIAL_VECTOR_MIN_SIDE_M="$OFFICIAL_VECTOR_MIN_SIDE_M" \ @@ -300,6 +309,11 @@ docker run -d \ -e THEMATIC_RASTER_MAX_PIXELS="$THEMATIC_RASTER_MAX_PIXELS" \ -e THEMATIC_RASTER_TIMEOUT_SECONDS="$THEMATIC_RASTER_TIMEOUT_SECONDS" \ -e THEMATIC_RASTER_MAX_RESPONSE_MB="$THEMATIC_RASTER_MAX_RESPONSE_MB" \ + -e WALOUS_ENABLED="$WALOUS_ENABLED" \ + -e WALOUS_SOURCE_DIR="$WALOUS_SOURCE_DIR" \ + -e WALOUS_ANALYSIS_RESOLUTION_M="$WALOUS_ANALYSIS_RESOLUTION_M" \ + -e WALOUS_MAX_SIDE_M="$WALOUS_MAX_SIDE_M" \ + -e WALOUS_MAX_PIXELS="$WALOUS_MAX_PIXELS" \ -e YOLO_ENABLED="$YOLO_ENABLED" \ -e YOLO_MODELS_DIR="$YOLO_MODELS_DIR" \ -e YOLO_MODEL_PATH="$YOLO_MODEL_PATH" \ diff --git a/docker-compose.unraid.yml b/docker-compose.unraid.yml index 10d0a5e3..8ad3251a 100644 --- a/docker-compose.unraid.yml +++ b/docker-compose.unraid.yml @@ -46,6 +46,8 @@ services: DOV_SOIL_WFS_URL: ${DOV_SOIL_WFS_URL:-https://www.dov.vlaanderen.be/geoserver/wfs} SPW_PICC_ENABLED: ${SPW_PICC_ENABLED:-true} SPW_PICC_MAPSERVER_URL: ${SPW_PICC_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer} + SPW_FLOOD_HAZARD_ENABLED: ${SPW_FLOOD_HAZARD_ENABLED:-true} + SPW_FLOOD_HAZARD_MAPSERVER_URL: ${SPW_FLOOD_HAZARD_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer} URBIS_ENABLED: ${URBIS_ENABLED:-true} URBIS_WFS_URL: ${URBIS_WFS_URL:-https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows} OFFICIAL_VECTOR_MIN_SIDE_M: ${OFFICIAL_VECTOR_MIN_SIDE_M:-10} @@ -80,6 +82,11 @@ services: THEMATIC_RASTER_MAX_PIXELS: ${THEMATIC_RASTER_MAX_PIXELS:-30000000} THEMATIC_RASTER_TIMEOUT_SECONDS: ${THEMATIC_RASTER_TIMEOUT_SECONDS:-300} THEMATIC_RASTER_MAX_RESPONSE_MB: ${THEMATIC_RASTER_MAX_RESPONSE_MB:-160} + WALOUS_ENABLED: ${WALOUS_ENABLED:-true} + WALOUS_SOURCE_DIR: ${WALOUS_SOURCE_DIR:-/app/storage/source-cache/walous} + WALOUS_ANALYSIS_RESOLUTION_M: ${WALOUS_ANALYSIS_RESOLUTION_M:-10} + WALOUS_MAX_SIDE_M: ${WALOUS_MAX_SIDE_M:-60000} + WALOUS_MAX_PIXELS: ${WALOUS_MAX_PIXELS:-36000000} YOLO_ENABLED: ${YOLO_ENABLED:-false} YOLO_MODELS_DIR: ${YOLO_MODELS_DIR:-/app/models} YOLO_MODEL_PATH: ${YOLO_MODEL_PATH:-} diff --git a/docker-compose.yml b/docker-compose.yml index afa019a1..39b4507d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,8 @@ services: DOV_SOIL_WFS_URL: ${DOV_SOIL_WFS_URL:-https://www.dov.vlaanderen.be/geoserver/wfs} SPW_PICC_ENABLED: ${SPW_PICC_ENABLED:-true} SPW_PICC_MAPSERVER_URL: ${SPW_PICC_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer} + SPW_FLOOD_HAZARD_ENABLED: ${SPW_FLOOD_HAZARD_ENABLED:-true} + SPW_FLOOD_HAZARD_MAPSERVER_URL: ${SPW_FLOOD_HAZARD_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer} URBIS_ENABLED: ${URBIS_ENABLED:-true} URBIS_WFS_URL: ${URBIS_WFS_URL:-https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows} OFFICIAL_VECTOR_MIN_SIDE_M: ${OFFICIAL_VECTOR_MIN_SIDE_M:-10} @@ -105,6 +107,11 @@ services: THEMATIC_RASTER_MAX_PIXELS: ${THEMATIC_RASTER_MAX_PIXELS:-30000000} THEMATIC_RASTER_TIMEOUT_SECONDS: ${THEMATIC_RASTER_TIMEOUT_SECONDS:-300} THEMATIC_RASTER_MAX_RESPONSE_MB: ${THEMATIC_RASTER_MAX_RESPONSE_MB:-160} + WALOUS_ENABLED: ${WALOUS_ENABLED:-true} + WALOUS_SOURCE_DIR: ${WALOUS_SOURCE_DIR:-/app/storage/source-cache/walous} + WALOUS_ANALYSIS_RESOLUTION_M: ${WALOUS_ANALYSIS_RESOLUTION_M:-10} + WALOUS_MAX_SIDE_M: ${WALOUS_MAX_SIDE_M:-60000} + WALOUS_MAX_PIXELS: ${WALOUS_MAX_PIXELS:-36000000} YOLO_ENABLED: ${YOLO_ENABLED:-false} YOLO_MODELS_DIR: ${YOLO_MODELS_DIR:-/app/models} YOLO_MODEL_PATH: ${YOLO_MODEL_PATH:-} diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 4ab13474..3302e397 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -482,6 +482,42 @@ Returns a constrained transparent PNG generated from the persisted governed raster. It accepts neither an arbitrary file path nor a provider URL and feeds the existing MapLibre image-overlay path. +### GET `/api/v1/projects/{project_id}/datasets/walous/products` + +Returns the fixed official WALOUS 2020/2023 registry. Each product reports its +observation year, EPSG:3812 source contract, 1 m source semantics, configured +state, attribution, licence and documented edition accuracy. `configured` +becomes true only when the checksum-validated source GeoTIFF exists below +`WALOUS_SOURCE_DIR`; the endpoint never downloads an archive. + +### POST `/api/v1/projects/{project_id}/datasets/walous/acquire` + +Reads a bounded window from one provisioned official 1 m WALOUS source, +applies nearest-neighbour resampling to the configured analysis resolution, +masks `bbox intersect Area`, validates class values 1-11 and persists a normal +raster Dataset through `DatasetService`. URLs, paths, classes and resolutions +are not caller-controlled. Equal spatial requests for 2020 and 2023 share one +temporal series key. + +### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/walous/select` + +Returns mapped hectares for total observed land cover, trees/forest, surface +water, artificial cover, annual and permanent herbaceous cover and bare soil. +Values are cell-area estimates from the persisted derived raster. The response +explicitly rejects legal land use, ownership, tree count, timber volume and +water volume as unsupported interpretations. + +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/walous/image` + +Returns a transparent PNG using the governed 11-class colour table and only +the persisted Dataset geometry. It never proxies the source archive. + +The Map workbench acquires every configured comparable WALOUS observation for +the same Walloon selection when current land cover is first requested. The +latest edition drives the current result; the ordinary temporal comparison API +then compares 2020 and 2023 semantic area metrics. Raster evolution does not +claim individual object additions or removals. + ### GET `/api/v1/projects/{project_id}/datasets` List datasets. diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 672df2a1..edb2660e 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -3652,6 +3652,28 @@ Validation: - `cd backend && python -m alembic heads && python -m alembic upgrade head --sql` # Codex Execution Log +## Post-V1 national coverage completion: Wallonia (2026-07-22) + +- Validated the official WALOUS 2020 and 2023 archives, their EPSG:3812 1 m + raster contract, 11 classes, CC BY 4.0 attribution and published edition + accuracy. Added a fail-closed operator provisioner with archive-size, + extraction, CRS, resolution, class and checksum gates. +- Added bounded WALOUS persistence through `DatasetService`, semantic hectare + metrics, a governed PNG overlay and automatic 2020/2023 temporal-series + materialization. The existing temporal API now compares persisted WALOUS + raster metrics while keeping object-level change unavailable. +- Added the queryable legal SPW flood-hazard polygon product and class-aware + metrics through the existing official-vector engine. +- Wired both integrations through Compose, the all-in-one Unraid runner, + editable DockerMan template and readiness gate. Added canonical API, + persistence, rendering, temporal and runtime-parity regression tests. +- Revalidated the official Walloon DTM distribution. The whole-region files + are too large for implicit startup or per-selection mirroring (about 41 GB + at 1 m and 213 GB at 0.5 m), so elevation remains a documented capacity-plan + prerequisite instead of a fake operational source. +- Reprobed the documented MDK WCS endpoints. Strict TLS still fails hostname + validation; acquisition remains disabled and no insecure fallback was added. + This file must be updated by Codex after each implementation pass. ## Format diff --git a/docs/DATA_COVERAGE_STATUS.md b/docs/DATA_COVERAGE_STATUS.md index 32da1763..bc7e6a49 100644 --- a/docs/DATA_COVERAGE_STATUS.md +++ b/docs/DATA_COVERAGE_STATUS.md @@ -26,7 +26,7 @@ coverage or historical dates. | --- | --- | --- | | Belgium | NGI administrative boundaries; Statbel population/statistical sectors | Statbel population 2021-2025 | | Flanders | GRB buildings, roads, water and parcels; DHMV terrain/surface; VMM flood scenarios; BWK/Natura 2000; DOV soil; policy rasters for space, open space, accessibility and services; agriculture and orthophoto where governed | Population 2021-2025; land-use/land-cover series where retained; agriculture editions; historical maps/orthophotos where the selected product has a real observation date | -| Wallonia | Bounded PICC buildings, roads and hydrography; governed SPW bed-elevation/bathymetry products | No general cross-theme regional history yet | +| Wallonia | Bounded PICC buildings, roads and hydrography; legal SPW flood-hazard polygons; bounded WALOUS 2020/2023 land-cover analysis from provisioned official rasters; governed SPW bed-elevation/bathymetry products | Comparable WALOUS land-cover area metrics for 2020-2023; no general cross-theme regional history yet | | Brussels | Bounded UrbIS buildings, street axes, cadastral parcels and Land Cover blocks; official FO/GB blocks provide forest/park area and WB blocks provide permanent water area | No general cross-theme regional history yet; the live WFS has no per-feature observation date | | Belgian North Sea | RBINS reporting units; Marine Spatial Plan 2026-2034; governed MDK bathymetry only when runtime acquisition is explicitly configured | No multi-epoch bathymetry or marine-plan trend yet | @@ -37,23 +37,21 @@ applicable bounded official source or reports the theme as unsupported. ## Priority coverage gaps -1. Govern the public Walloon WALOUS land-cover editions (including the - published 2018/2020 change product) as a real regional time series. A class - crosswalk is required because the older COSW 2005/2007 methodology differs. - Official catalogue: - `https://geoportail.wallonie.be/catalogue-donnees?search-text=occupation+du+sol`. -2. Govern the current public Walloon flood-hazard vector/raster products and - retain their model scenario semantics separately from observed floods. - Official record: - `https://geoportail.wallonie.be/catalogue/14084108-2c7b-4091-b62d-ff0fc235213a.html`. -3. Add a common Belgium-wide topographic baseline with normalized theme +1. Add the official WALOUS 2018 edition only after SPW restores a stable direct + artifact or another checksum-verifiable acquisition contract. WALOUS 2020 + and 2023 are operational and comparable; COSW 2005/2007 remains a different + methodology and is not silently merged. +2. Add a common Belgium-wide topographic baseline with normalized theme semantics across NGI, Flanders, Wallonia and Brussels. -4. Govern comparable Walloon and Brussels historical editions before exposing +3. Govern comparable Walloon and Brussels historical editions before exposing evolution for buildings, roads, land cover, soil, elevation or flood risk. -5. Add nationally comparable land-cover history with explicit class crosswalks +4. Add nationally comparable land-cover history with explicit class crosswalks and uncertainty; never compare incompatible legends silently. -6. Add multi-epoch marine bathymetry and survey-footprint metadata before +5. Add multi-epoch marine bathymetry and survey-footprint metadata before presenting seabed evolution. +6. Add Walloon DTM only through an operator capacity plan: the official 1 m + national artifact is about 41 GB and the 0.5 m artifact about 213 GB, so it + is not safe as an implicit per-selection dependency. 7. Expand persisted raster partition manifests beyond the regression regions only where repeated use justifies caching; bounded acquisition remains the default for one-off selections. diff --git a/docs/TODO.md b/docs/TODO.md index d4d840ee..c6b0fc88 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -63,9 +63,11 @@ geen open productroadmap meer. - [x] Maak UrbIS Land Cover begrensd operationeel voor Brussel: alle Blocks als landbedekking, FO/GB als bos en park en WB als permanent water, met echte PostGIS-oppervlaktemetrics en broncodes. -- [ ] Implementeer begrensde WALOUS 2018/2020/2023 rasteracquisitie in - EPSG:3812 met officiële klassen, vergelijkbaarheidscontract en pixelbudget. -- [ ] Implementeer de actuele Waalse overstromingsgevaarkaart als afzonderlijk +- [x] Implementeer begrensde WALOUS 2020/2023 rasteracquisitie in EPSG:3812 + met officiële klassen, vergelijkbaarheidscontract, pixelbudget, automatische + tijdreeksmaterialisatie en evolutiemetrics. WALOUS 2018 blijft bewust open + tot SPW opnieuw een stabiel checksum-verifieerbaar direct artifact aanbiedt. +- [x] Implementeer de actuele Waalse overstromingsgevaarkaart als afzonderlijk scenario-/juridisch contract; gebruik WMS alleen als context tenzij analytische pixels of vectorgeometrie officieel beschikbaar zijn. - [ ] Bouw een Belgische AI-validatiematrix met gelabelde golden AOIs in elk diff --git a/frontend/README.md b/frontend/README.md index 4f714937..e3e6ffc7 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -763,6 +763,19 @@ agriculture use thematic raster analysis; nature value and soil use the persisted-vector GeoJSON pattern. The browser calls only the GeoIntel API and never contacts WCS, WFS or OGC providers directly. +## Walloon current and historical land cover + +For a bounded Walloon selection, `Landbedekking` resolves the newest configured +WALOUS product and automatically persists the other configured comparable +edition for the same rectangle. Current analysis shows semantic hectare +metrics and an 11-class MapLibre image overlay. After dataset refresh, +`Evolutie` offers 2020 versus 2023 through the same period selector and trend +chart used by other temporal sources. Individual object-change counts remain +unavailable for categorical rasters and are not simulated. + +The source card remains `Op aanvraag` when the official source files are not +provisioned and never contacts SPW directly from the browser. + ## Belgium and Belgian North Sea coverage When `Belgium and North Sea Workbench` exists with ready reference data, it is diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index 1c022d9e..d10adac5 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -9,7 +9,7 @@ import { getDatasetDisplayName, getDatasetSourceDisplayName } from '../../lib/da import { TemporalTrendChart } from './TemporalTrendChart' import { terrainImageUrl } from '../../lib/terrainImage' import { floodHazardImageUrl } from '../../lib/floodHazardImage' -import { thematicRasterImageUrl } from '../../lib/thematicRaster' +import { thematicRasterImageUrl, walousRasterImageUrl } from '../../lib/thematicRaster' import { bathymetryRasterImageUrl } from '../../lib/bathymetryRaster' import { FLANDERS_WORKSPACE_PROJECT_NAME } from '../../config/primaryFocus' import { @@ -59,6 +59,7 @@ const EMPTY_TEMPORAL_SERIES: DatasetCreateResponse[] = [] type DataThemeId = | 'administrative' | 'buildings' + | 'land_cover' | 'space_occupation' | 'open_space' | 'population' @@ -110,7 +111,7 @@ function productSupportsSelection(product: OnDemandMapProduct, bbox: VectorSelec if (product.kind === 'dhmv' || product.kind === 'flood_hazard') { return dimensions.areaSquareMetres <= 280_000_000 } - if (product.kind === 'thematic_raster') { + if (product.kind === 'thematic_raster' || product.kind === 'walous') { return dimensions.widthMetres <= 50_000 && dimensions.heightMetres <= 50_000 && dimensions.areaSquareMetres <= 2_800_000_000 @@ -133,6 +134,13 @@ const DATA_THEMES: DataTheme[] = [ description: 'Gebouwen en gebouwcontouren uit GRB of een andere persistente bron.', tokens: ['buildings', 'building', 'gebouwen', 'gebouw', 'bebouwing', 'gbg'], }, + { + id: 'land_cover', + label: 'Landbedekking', + shortLabel: 'Landbedekking', + description: 'Fysieke en biologische bodembedekking uit een officieel regionaal classificatieraster.', + tokens: ['land_cover', 'land_cover_use', 'landbedekking', 'walous', 'occupation du sol'], + }, { id: 'space_occupation', label: 'Ruimtebeslag', @@ -257,6 +265,7 @@ const DATA_THEMES: DataTheme[] = [ const COVERAGE_THEME_BY_MAP_THEME: Record = { administrative: 'admin', buildings: 'buildings', + land_cover: 'land_cover_use', space_occupation: 'land_cover_use', open_space: 'land_cover_use', population: 'population', @@ -303,6 +312,7 @@ function coverageZoneLabel(zone: string): string { const DATA_THEME_MAP_STYLES: Record = { administrative: { fill: '#5f6f7f', line: '#344554' }, buildings: { fill: '#d45f3d', line: '#9f3e24' }, + land_cover: { fill: '#4f7b4f', line: '#315c39' }, space_occupation: { fill: '#be3e33', line: '#8f2c24' }, open_space: { fill: '#267a46', line: '#175c32' }, population: { fill: '#7559a6', line: '#5b3f88' }, @@ -358,6 +368,12 @@ function datasetAvailabilityLabel( const resolutionLabel = Number.isFinite(resolution) ? `${resolution.toLocaleString('nl-BE')} m` : 'Raster' return `${resolutionLabel} officiële bron${Number.isFinite(year) ? ` · ${year}` : ''}` } + if (dataset.dataset_type === 'raster' && dataset.source_name === 'spw_walous_land_cover') { + const resolution = Number(dataset.source_metadata?.['analysis_resolution_m']) + const year = Number(dataset.source_metadata?.['observation_year']) + const resolutionLabel = Number.isFinite(resolution) ? `${resolution.toLocaleString('nl-BE')} m` : 'Raster' + return `${resolutionLabel} WALOUS-landbedekking${Number.isFinite(year) ? ` · ${year}` : ''}` + } if (dataset.source_name === 'ngi_adminvector') { return `${(dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0).toLocaleString('nl-BE')} officiële bestuursgebieden` } @@ -406,6 +422,9 @@ function datasetMatchesTheme(dataset: DatasetCreateResponse, theme: DataTheme): if (dataset.source_name === 'department_omgeving_thematic_raster') { return dataset.source_metadata?.['theme'] === theme.id } + if (dataset.source_name === 'spw_walous_land_cover') { + return theme.id === 'land_cover' + } if (dataset.source_name === 'dov_soil_map') { return theme.id === 'soil' } @@ -522,6 +541,7 @@ function pickThemeDataset( (dataset.source_name === 'inbo_bwk_natura2000' ? 95_000 : 0) + (dataset.source_name === 'agentschap_landbouw_zeevisserij_agricultural_parcels' ? 98_000 : 0) + (dataset.source_name === 'department_omgeving_thematic_raster' ? 5_000_000 : 0) + + (dataset.source_name === 'spw_walous_land_cover' ? 5_000_000 : 0) + (dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register' ? 5_000_000 : 0) + (dataset.source_name === 'digitaal_vlaanderen_dhmv' ? 5_000_000 : 0) + (dataset.source_name === 'vmm_flood_hazard' ? 5_000_000 : 0) + @@ -636,6 +656,10 @@ function formatDatasetObservation(dataset: DatasetCreateResponse): string { return `referentiejaar ${observationYear}` } } + if (dataset.source_name === 'spw_walous_land_cover') { + const observationYear = Number(dataset.source_metadata?.['observation_year']) + return Number.isFinite(observationYear) ? `WALOUS referentiejaar ${observationYear}` : 'WALOUS landbedekking' + } const period = dataset.source_metadata?.['acquisition_period'] if (typeof period === 'string' && period.trim()) { return `opnameperiode ${period}` @@ -1085,6 +1109,29 @@ export function MapWorkspace({ }) } } + const latestWalous = officialMapProducts.walous + .filter((product) => product.configured && productCoversZones(product.coverage_zones, effectiveZones)) + .sort((left, right) => right.observation_year - left.observation_year)[0] + if (latestWalous) { + result.push({ + kind: 'walous', + productKey: latestWalous.key, + historyProductKeys: officialMapProducts.walous + .filter( + (product) => + product.configured + && product.key !== latestWalous.key + && productCoversZones(product.coverage_zones, effectiveZones), + ) + .map((product) => product.key), + displayName: latestWalous.display_name, + theme: 'land_cover', + availabilityLabel: `${latestWalous.analysis_resolution_m ?? latestWalous.native_resolution_m} m analyse · ${latestWalous.observation_year} · automatisch bij selectie`, + attribution: latestWalous.attribution, + limitationMessage: latestWalous.limitation_message, + coverageZones: latestWalous.coverage_zones, + }) + } for (const product of officialMapProducts.officialVector.filter((item) => productCoversZones(item.coverage_zones, effectiveZones), )) { @@ -1319,6 +1366,20 @@ export function MapWorkspace({ : [], [activeThemeDataset, selectedProjectId, thematicRasterBounds], ) + const walousRasterBounds = activeThemeDataset?.source_name === 'spw_walous_land_cover' + ? activeThemeDataset.source_metadata?.['bbox_epsg4326'] + : null + const walousRasterImageOverlays = useMemo( + () => activeThemeDataset?.source_name === 'spw_walous_land_cover' && selectedProjectId && Array.isArray(walousRasterBounds) && walousRasterBounds.length === 4 + ? [{ + url: walousRasterImageUrl(selectedProjectId, activeThemeDataset.id), + bbox: walousRasterBounds.map(Number) as [number, number, number, number], + label: getDatasetDisplayName(activeThemeDataset), + opacity: 0.82, + }] + : [], + [activeThemeDataset, selectedProjectId, walousRasterBounds], + ) const bathymetryRasterBounds = activeThemeDataset?.source_name === 'spw_bathymetry' ? activeThemeDataset.source_metadata?.['bbox_epsg4326'] : null @@ -1338,6 +1399,8 @@ export function MapWorkspace({ const activeImageOverlays = useMemo( () => bathymetryRasterImageOverlays.length > 0 ? bathymetryRasterImageOverlays + : walousRasterImageOverlays.length > 0 + ? walousRasterImageOverlays : thematicRasterImageOverlays.length > 0 ? thematicRasterImageOverlays : floodHazardImageOverlays.length > 0 @@ -1345,7 +1408,7 @@ export function MapWorkspace({ : terrainImageOverlays.length > 0 ? terrainImageOverlays : orthophotoImageOverlay ? [orthophotoImageOverlay] : [], - [bathymetryRasterImageOverlays, floodHazardImageOverlays, orthophotoImageOverlay, terrainImageOverlays, thematicRasterImageOverlays], + [bathymetryRasterImageOverlays, floodHazardImageOverlays, orthophotoImageOverlay, terrainImageOverlays, thematicRasterImageOverlays, walousRasterImageOverlays], ) const municipalityAreaCount = areas.filter((area) => /^Gemeente\s/i.test(area.name)).length const themeTemporalSeriesMap = useMemo( @@ -1908,6 +1971,7 @@ export function MapWorkspace({ kind: onDemandProduct.kind, productKey: onDemandProduct.productKey, displayName: onDemandProduct.displayName, + historyProductKeys: onDemandProduct.historyProductKeys, }, acquisitionBboxes: onDemandProduct.acquisitionBboxes, featureLimit: resultFeatureLimit, @@ -2471,7 +2535,7 @@ export function MapWorkspace({ />
Werkgebied - {thematicRasterImageOverlays.length > 0 ? ( + {thematicRasterImageOverlays.length > 0 || walousRasterImageOverlays.length > 0 ? ( {thematicLegendMin} → {thematicLegendMax} diff --git a/frontend/src/hooks/useMapThemeSelectionInsights.ts b/frontend/src/hooks/useMapThemeSelectionInsights.ts index 40ea3066..fa86960f 100644 --- a/frontend/src/hooks/useMapThemeSelectionInsights.ts +++ b/frontend/src/hooks/useMapThemeSelectionInsights.ts @@ -9,6 +9,7 @@ import { bathymetryRasterSelectionToMapSelection } from '../lib/bathymetryRaster export type MapThemeAcquisitionKind = | 'thematic_raster' + | 'walous' | 'dhmv' | 'flood_hazard' | 'grb' @@ -19,6 +20,7 @@ export interface MapThemeAcquisition { kind: MapThemeAcquisitionKind productKey: string displayName: string + historyProductKeys?: string[] } export interface MapThemeQuery { @@ -113,7 +115,7 @@ export function useMapThemeSelectionInsights( const resultLimit = featureLimit ?? 1000 if (acquisition) { const requestedBboxes = acquisitionBboxes?.length ? acquisitionBboxes : [bbox] - const acquisitionResults = await settleWithConcurrency(requestedBboxes, 1, async (acquisitionBbox) => { + const acquireProduct = async (acquisitionBbox: VectorSelectionBBox, productKey: string) => { const commonPayload = { bbox: acquisitionBbox, area_id: areaId, @@ -124,6 +126,11 @@ export function useMapThemeSelectionInsights( ...commonPayload, product_key: acquisition.productKey, }) + : acquisition.kind === 'walous' + ? await datasetsApi.acquireWalous(selectedProjectId, { + ...commonPayload, + product_key: productKey, + }) : acquisition.kind === 'dhmv' ? await datasetsApi.acquireDhmv(selectedProjectId, { ...commonPayload, @@ -152,13 +159,38 @@ export function useMapThemeSelectionInsights( ) } return datasetsApi.get(selectedProjectId, acquisitionJob.output_dataset_id) - }) + } + const acquisitionResults = await settleWithConcurrency( + requestedBboxes, + 1, + (acquisitionBbox) => acquireProduct(acquisitionBbox, acquisition.productKey), + ) const failedAcquisition = acquisitionResults.find((item) => item.status === 'rejected') if (failedAcquisition?.status === 'rejected') { throw failedAcquisition.reason } acquiredDatasets = acquisitionResults.flatMap((item) => item.status === 'fulfilled' ? [item.value] : []) dataset = acquiredDatasets[0] + const historyProductKeys = acquisition.kind === 'walous' + ? [...new Set(acquisition.historyProductKeys ?? [])].filter((key) => key !== acquisition.productKey) + : [] + if (historyProductKeys.length > 0) { + const historyRequests = historyProductKeys.flatMap((productKey) => + requestedBboxes.map((acquisitionBbox) => ({ acquisitionBbox, productKey })), + ) + const historyResults = await settleWithConcurrency( + historyRequests, + 1, + ({ acquisitionBbox, productKey }) => acquireProduct(acquisitionBbox, productKey), + ) + const failedHistory = historyResults.find((item) => item.status === 'rejected') + if (failedHistory?.status === 'rejected') { + throw failedHistory.reason + } + acquiredDatasets.push( + ...historyResults.flatMap((item) => item.status === 'fulfilled' ? [item.value] : []), + ) + } } if (!dataset) { throw new Error(`Geen persistente databron beschikbaar voor thema ${themeId}.`) @@ -197,8 +229,13 @@ export function useMapThemeSelectionInsights( : dataset.dataset_type === 'raster' && dataset.source_name === 'department_omgeving_thematic_raster' ? thematicRasterSelectionToMapSelection(await datasetsApi.selectThematicRaster(selectedProjectId, dataset.id, { bbox, - area_id: areaId, - })) + area_id: areaId, + })) + : dataset.dataset_type === 'raster' && dataset.source_name === 'spw_walous_land_cover' + ? thematicRasterSelectionToMapSelection(await datasetsApi.selectWalous(selectedProjectId, dataset.id, { + bbox, + area_id: areaId, + })) : dataset.dataset_type === 'raster' && dataset.source_name === 'spw_bathymetry' ? bathymetryRasterSelectionToMapSelection(await datasetsApi.selectBathymetryRaster(selectedProjectId, dataset.id, { bbox, diff --git a/frontend/src/hooks/useOfficialMapProducts.ts b/frontend/src/hooks/useOfficialMapProducts.ts index b22e5015..2f987f47 100644 --- a/frontend/src/hooks/useOfficialMapProducts.ts +++ b/frontend/src/hooks/useOfficialMapProducts.ts @@ -14,6 +14,7 @@ import type { export interface OfficialMapProducts { thematic: ThematicRasterProductRead[] + walous: ThematicRasterProductRead[] dhmv: DhmvProductRead[] floodHazard: FloodHazardProductRead[] grb: GrbProductRead[] @@ -23,6 +24,7 @@ export interface OfficialMapProducts { const EMPTY_PRODUCTS: OfficialMapProducts = { thematic: [], + walous: [], dhmv: [], floodHazard: [], grb: [], @@ -50,16 +52,18 @@ export function useOfficialMapProducts(selectedProjectId: string | null) { setError(null) void Promise.all([ datasetsApi.listThematicRasterProducts(selectedProjectId), + datasetsApi.listWalousProducts(selectedProjectId), datasetsApi.listDhmvProducts(selectedProjectId), datasetsApi.listFloodHazardProducts(selectedProjectId), datasetsApi.listGrbProducts(selectedProjectId), datasetsApi.listOfficialVectorProducts(selectedProjectId), datasetsApi.listBathymetrySources(selectedProjectId), ]) - .then(([thematic, dhmv, floodHazard, grb, officialVector, bathymetry]) => { + .then(([thematic, walous, dhmv, floodHazard, grb, officialVector, bathymetry]) => { if (!cancelled) { setProducts({ thematic: thematic.items, + walous: walous.items, dhmv: dhmv.items, floodHazard: floodHazard.items, grb: grb.items, diff --git a/frontend/src/lib/datasetCapabilities.ts b/frontend/src/lib/datasetCapabilities.ts index 97097248..8b4aed88 100644 --- a/frontend/src/lib/datasetCapabilities.ts +++ b/frontend/src/lib/datasetCapabilities.ts @@ -5,6 +5,7 @@ const NON_IMAGERY_RASTER_SOURCES = new Set([ 'digitaal_vlaanderen_dhmv', 'vmm_flood_hazard', 'spw_bathymetry', + 'spw_walous_land_cover', ]) export function isDetectionImageryDataset(dataset: DatasetCreateResponse): boolean { diff --git a/frontend/src/lib/datasetDisplay.ts b/frontend/src/lib/datasetDisplay.ts index c80c3e23..82a02332 100644 --- a/frontend/src/lib/datasetDisplay.ts +++ b/frontend/src/lib/datasetDisplay.ts @@ -17,6 +17,7 @@ const DATASET_LABEL_BY_LAYER: Record = { open_space: 'Open ruimte', accessibility: 'Knooppuntwaarde', services: 'Voorzieningenniveau', + land_cover: 'Landbedekking', building_registry: 'Gebouwenregister', regional_boundary: 'Grens vervoerregio Kempen', municipality_boundaries: 'Gemeentegrenzen Kempen', @@ -37,6 +38,8 @@ const DATASET_SOURCE_LABELS: Record = { vmm_flood_hazard: 'Vlaamse Milieumaatschappij', vmm_vha_bathymetry_profiles: 'VMM / Vlaamse Hydrografische Atlas', spw_bathymetry: 'Service public de Wallonie', + spw_walous_land_cover: 'Service public de Wallonie', + spw_flood_hazard: 'Service public de Wallonie', department_omgeving_thematic_raster: 'Departement Omgeving', dov_soil_map: 'Databank Ondergrond Vlaanderen', } @@ -65,6 +68,10 @@ export function getDatasetDisplayName(dataset: DatasetCreateResponse): string { const productName = dataset.source_metadata?.['product_display_name'] return typeof productName === 'string' && productName.trim() ? productName : 'Officieel Vlaams themaraster' } + if (dataset.source_name === 'spw_walous_land_cover') { + const productName = dataset.source_metadata?.['product_display_name'] + return typeof productName === 'string' && productName.trim() ? productName : 'WALOUS landbedekking' + } const layer = (dataset.reference_layer_name ?? dataset.source_metadata?.layer_name ?? dataset.source_metadata?.layer_type ?? '') .toString() .toLowerCase() diff --git a/frontend/src/lib/thematicRaster.ts b/frontend/src/lib/thematicRaster.ts index fe1af233..2e83b325 100644 --- a/frontend/src/lib/thematicRaster.ts +++ b/frontend/src/lib/thematicRaster.ts @@ -4,6 +4,10 @@ export function thematicRasterImageUrl(projectId: string, datasetId: string): st return `/api/v1/projects/${projectId}/datasets/${datasetId}/raster/thematic/image` } +export function walousRasterImageUrl(projectId: string, datasetId: string): string { + return `/api/v1/projects/${projectId}/datasets/${datasetId}/raster/walous/image` +} + export function thematicRasterSelectionToMapSelection(result: ThematicRasterSelectionResponse): VectorSelectionResponse { return { selection_bbox: result.selection_bbox, diff --git a/frontend/src/services/api/datasets.ts b/frontend/src/services/api/datasets.ts index c6d42023..e3bc3c3d 100644 --- a/frontend/src/services/api/datasets.ts +++ b/frontend/src/services/api/datasets.ts @@ -220,12 +220,22 @@ export const datasetsApi = { apiPost(`/api/v1/projects/${projectId}/datasets/thematic-raster/acquire`, payload), listThematicRasterProducts: (projectId: string): Promise<{ items: ThematicRasterProductRead[]; total: number }> => apiGet<{ items: ThematicRasterProductRead[]; total: number }>(`/api/v1/projects/${projectId}/datasets/thematic-raster/products`), + acquireWalous: (projectId: string, payload: ThematicRasterAcquireRequest): Promise => + apiPost(`/api/v1/projects/${projectId}/datasets/walous/acquire`, payload), + listWalousProducts: (projectId: string): Promise<{ items: ThematicRasterProductRead[]; total: number }> => + apiGet<{ items: ThematicRasterProductRead[]; total: number }>(`/api/v1/projects/${projectId}/datasets/walous/products`), selectThematicRaster: ( projectId: string, datasetId: string, payload: { bbox: VectorSelectionRequest['bbox']; area_id?: string }, ): Promise => apiPost(`/api/v1/projects/${projectId}/datasets/${datasetId}/raster/thematic/select`, payload), + selectWalous: ( + projectId: string, + datasetId: string, + payload: { bbox: VectorSelectionRequest['bbox']; area_id?: string }, + ): Promise => + apiPost(`/api/v1/projects/${projectId}/datasets/${datasetId}/raster/walous/select`, payload), refreshMetadata: (projectId: string, datasetId: string): Promise => apiPost(`/api/v1/projects/${projectId}/datasets/${datasetId}/metadata/refresh`, {}), inspectRaster: (projectId: string, datasetId: string): Promise => diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 08e530a0..98d4b229 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -576,11 +576,12 @@ export interface ThematicRasterAcquireRequest { export interface ThematicRasterProductRead { key: string display_name: string - theme: 'space_occupation' | 'open_space' | 'forest' | 'agriculture' | 'population' | 'accessibility' | 'services' - metric_kind: 'binary_area' | 'population_density' | 'index_score' | 'normalized_score' + theme: 'space_occupation' | 'open_space' | 'forest' | 'agriculture' | 'population' | 'accessibility' | 'services' | 'land_cover' + metric_kind: 'binary_area' | 'population_density' | 'index_score' | 'normalized_score' | 'categorical_area' coverage_id: string native_resolution_m: number - source_crs: 'EPSG:31370' + analysis_resolution_m?: number | null + source_crs: 'EPSG:31370' | 'EPSG:3812' source_value_unit: string observation_year: number source_version: string @@ -591,6 +592,9 @@ export interface ThematicRasterProductRead { legend_max_label: string included_source_values: number[] limitation_message: string + coverage_zones: string[] + configured: boolean + status: 'configured' | 'source_not_provisioned' } export interface ThematicRasterSelectionResponse { diff --git a/scripts/README.md b/scripts/README.md index 4c667ddf..ea7a73e6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -2,6 +2,24 @@ Setup-, import-, demo- en maintenance-scripts voor GeoIntel. +## WALOUS source provisioning + +Run the networked operator only after checking at least 2 GB of archive space +plus room for the extracted official GeoTIFFs: + +```bash +python scripts/provision_walous_sources.py \ + --years 2020 2023 \ + --destination storage/source-cache/walous +``` + +The command accepts only the hard-coded official SPW 2020/2023 archives, +streams with a 1 GB per-archive cap, rejects changed content lengths, extracts +only the single GeoTIFF by basename, validates the raster contract and writes +checksums plus `provisioning-report.json`. Existing valid sources are reused; +`--force` performs a new download. This is an operator acquisition, not an +application startup task. + ## Runtime verification Inspect interrupted runtime state without changing it: diff --git a/scripts/audit_api_contracts.py b/scripts/audit_api_contracts.py index 2c55c5b4..bfb31236 100644 --- a/scripts/audit_api_contracts.py +++ b/scripts/audit_api_contracts.py @@ -19,6 +19,7 @@ ALLOWED_NON_ENVELOPE_ENDPOINTS = { ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/bathymetry/image"), ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/flood-hazard/image"), ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/thematic/image"), + ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/walous/image"), } IGNORED_OPENAPI_PATHS = { diff --git a/scripts/provision_walous_sources.py b/scripts/provision_walous_sources.py new file mode 100644 index 00000000..2dab8343 --- /dev/null +++ b/scripts/provision_walous_sources.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Provision official WALOUS GeoTIFF source rasters for bounded runtime analysis.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import shutil +import sys +from urllib.request import Request, urlopen +from zipfile import BadZipFile, ZipFile + + +SOURCES = { + 2020: { + "url": ( + "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/" + "47b348f1-6e7a-4baa-963c-0232a43c0cff/WAL_OCS_IA__2020_GEOTIFF_3812.zip" + ), + "expected_archive_bytes": 728_244_755, + "target": "walous_land_cover_2020_3812.tif", + }, + 2023: { + "url": ( + "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/" + "4e780ba1-463c-478e-95df-d2f1963a150d/WAL_OCS_IA__2023_GEOTIFF_3812.zip" + ), + "expected_archive_bytes": 876_014_572, + "target": "walous_land_cover_2023_3812.tif", + }, +} +MAX_ARCHIVE_BYTES = 1_000_000_000 +MAX_EXTRACTED_BYTES = 50_000_000_000 + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(8 * 1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def download(url: str, destination: Path, expected_bytes: int) -> str: + temporary = destination.with_suffix(destination.suffix + ".part") + temporary.unlink(missing_ok=True) + digest = hashlib.sha256() + received = 0 + request = Request(url, headers={"User-Agent": "GeoIntel/1.0 WALOUS-source-provisioner"}) + try: + with urlopen(request, timeout=300) as response, temporary.open("wb") as output: + content_length = int(response.headers.get("Content-Length") or 0) + if content_length and content_length != expected_bytes: + raise RuntimeError(f"official archive size changed: expected {expected_bytes}, advertised {content_length}") + while chunk := response.read(8 * 1024 * 1024): + received += len(chunk) + if received > MAX_ARCHIVE_BYTES: + raise RuntimeError("official archive exceeds the governed 1 GB transfer limit") + digest.update(chunk) + output.write(chunk) + if received % (128 * 1024 * 1024) < len(chunk): + print(f" downloaded {received / 1024 / 1024:.0f} MiB", flush=True) + if received != expected_bytes: + raise RuntimeError(f"archive is incomplete: expected {expected_bytes} bytes, received {received}") + temporary.replace(destination) + return digest.hexdigest() + except Exception: + temporary.unlink(missing_ok=True) + raise + + +def extract_single_geotiff(archive: Path, target: Path) -> None: + try: + with ZipFile(archive) as bundle: + candidates = [item for item in bundle.infolist() if not item.is_dir() and item.filename.lower().endswith((".tif", ".tiff"))] + if len(candidates) != 1: + raise RuntimeError(f"archive must contain exactly one GeoTIFF, found {len(candidates)}") + member = candidates[0] + if member.file_size <= 0 or member.file_size > MAX_EXTRACTED_BYTES: + raise RuntimeError(f"GeoTIFF uncompressed size is outside the governed limit: {member.file_size}") + if Path(member.filename).name != member.filename.replace("\\", "/").split("/")[-1]: + # Nested paths are accepted only by basename; extraction never trusts archive paths. + pass + temporary = target.with_suffix(target.suffix + ".part") + temporary.unlink(missing_ok=True) + with bundle.open(member) as source, temporary.open("wb") as output: + shutil.copyfileobj(source, output, length=8 * 1024 * 1024) + temporary.replace(target) + except BadZipFile as exc: + raise RuntimeError("official WALOUS archive is not a valid ZIP file") from exc + + +def validate_raster(path: Path) -> dict: + try: + import numpy as np + import rasterio + from rasterio.enums import Resampling + except ImportError as exc: + raise RuntimeError("rasterio and numpy are required to validate WALOUS sources") from exc + with rasterio.open(path) as source: + if source.crs is None or source.crs.to_epsg() != 3812: + raise RuntimeError(f"WALOUS raster must use EPSG:3812, found {source.crs}") + if source.count != 1: + raise RuntimeError(f"WALOUS raster must have one band, found {source.count}") + if not all(abs(abs(float(value)) - 1.0) <= 0.05 for value in source.res): + raise RuntimeError(f"WALOUS raster must retain 1 m cells, found {source.res}") + sample_height = min(2048, source.height) + sample_width = min(2048, source.width) + sample = source.read(1, out_shape=(sample_height, sample_width), masked=True, resampling=Resampling.nearest) + values = np.unique(sample.compressed()).astype(int).tolist() + unexpected = sorted(set(values) - set(range(1, 12))) + if unexpected: + raise RuntimeError(f"WALOUS sample contains classes outside 1-11: {unexpected}") + return { + "path": str(path), + "crs": str(source.crs), + "width": int(source.width), + "height": int(source.height), + "resolution": [float(value) for value in source.res], + "bounds": [float(value) for value in source.bounds], + "nodata": None if source.nodata is None else float(source.nodata), + "sample_classes": values, + } + + +def provision(year: int, destination: Path, force: bool) -> dict: + source = SOURCES[year] + target = destination / source["target"] + archive = destination / f"{Path(source['target']).stem}.zip" + if target.is_file() and not force: + print(f"WALOUS {year}: validating existing source {target}") + validation = validate_raster(target) + digest = sha256_file(target) + else: + print(f"WALOUS {year}: downloading official archive") + archive_digest = download(source["url"], archive, source["expected_archive_bytes"]) + print(f"WALOUS {year}: archive sha256 {archive_digest}") + extract_single_geotiff(archive, target) + validation = validate_raster(target) + digest = sha256_file(target) + archive.unlink(missing_ok=True) + checksum_path = target.with_suffix(".sha256") + checksum_path.write_text(f"{digest} {target.name}\n", encoding="ascii") + validation.update({"year": year, "sha256": digest, "download_url": source["url"]}) + print(f"WALOUS {year}: ready ({target.stat().st_size / 1024 / 1024:.0f} MiB)") + return validation + + +def main() -> int: + parser = argparse.ArgumentParser(description="Provision official WALOUS 2020/2023 GeoTIFF sources.") + parser.add_argument("--years", nargs="+", type=int, choices=sorted(SOURCES), default=sorted(SOURCES)) + parser.add_argument("--destination", type=Path, default=Path("storage/source-cache/walous")) + parser.add_argument("--force", action="store_true") + args = parser.parse_args() + args.destination.mkdir(parents=True, exist_ok=True) + report = [provision(year, args.destination.resolve(), args.force) for year in args.years] + report_path = args.destination / "provisioning-report.json" + report_path.write_text(json.dumps({"sources": report}, indent=2) + "\n", encoding="utf-8") + print(f"Provisioning report: {report_path}") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Exception as exc: + print(f"WALOUS_PROVISIONING_FAILED: {exc}", file=sys.stderr) + raise SystemExit(1) from exc diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 82dab99c..4dcf5388 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -81,6 +81,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_flanders_geographic_scope.py ${PYTHON_BIN} -m py_compile scripts/provision_flanders_bathymetry_profiles.py ${PYTHON_BIN} -m py_compile scripts/probe_mdk_bathymetry.py ${PYTHON_BIN} -m py_compile scripts/import_spw_bathymetry.py +${PYTHON_BIN} -m py_compile scripts/provision_walous_sources.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py