feat: complete governed Walloon coverage sources
This commit is contained in:
@@ -30,6 +30,8 @@ BWK_WFS_URL=https://geo.api.vlaanderen.be/BWK/wfs
|
||||
DOV_SOIL_WFS_URL=https://www.dov.vlaanderen.be/geoserver/wfs
|
||||
SPW_PICC_ENABLED=true
|
||||
SPW_PICC_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer
|
||||
SPW_FLOOD_HAZARD_ENABLED=true
|
||||
SPW_FLOOD_HAZARD_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer
|
||||
URBIS_ENABLED=true
|
||||
URBIS_WFS_URL=https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows
|
||||
OFFICIAL_VECTOR_MIN_SIDE_M=10
|
||||
@@ -89,6 +91,11 @@ THEMATIC_RASTER_MAX_SIDE_M=60000
|
||||
THEMATIC_RASTER_MAX_PIXELS=30000000
|
||||
THEMATIC_RASTER_TIMEOUT_SECONDS=300
|
||||
THEMATIC_RASTER_MAX_RESPONSE_MB=160
|
||||
WALOUS_ENABLED=true
|
||||
WALOUS_SOURCE_DIR=/app/storage/source-cache/walous
|
||||
WALOUS_ANALYSIS_RESOLUTION_M=10
|
||||
WALOUS_MAX_SIDE_M=60000
|
||||
WALOUS_MAX_PIXELS=36000000
|
||||
YOLO_ENABLED=false
|
||||
YOLO_MODELS_DIR=/app/models
|
||||
YOLO_MODEL_PATH=
|
||||
|
||||
@@ -9,6 +9,20 @@
|
||||
|
||||
## Unreleased - Post-V1 capability completion (2026-07-19)
|
||||
|
||||
- Added operational Walloon WALOUS 2020/2023 land-cover analysis: an
|
||||
allowlisted checksum-validating provisioner, bounded EPSG:3812 window
|
||||
persistence, semantic hectare metrics, governed map rendering and raster
|
||||
temporal comparison. The map acquires comparable configured editions for
|
||||
the same selection so 2020-2023 evolution becomes available without manual
|
||||
dataset administration.
|
||||
- Added the current legal SPW Walloon flood-hazard polygons as a bounded
|
||||
authoritative vector product with class-aware hectare metrics and canonical
|
||||
persistence. No WMS pixels or modeled depths are fabricated.
|
||||
- Kept the Walloon DTM and MDK North Sea depth model honest: the published DTM
|
||||
artifacts require an explicit large-storage partition plan, while the MDK
|
||||
endpoint still fails strict hostname validation. Neither is presented as a
|
||||
measured analytical layer.
|
||||
|
||||
- Made `Belgium and North Sea Workbench` the unconditional frontend startup
|
||||
context, moved the initial MapLibre viewport to national extent and removed
|
||||
Mol/Kempen defaults from project/area forms and end-user source copy. Mol and
|
||||
|
||||
@@ -1599,6 +1599,40 @@ Datasets. A full-Flanders raster request remains blocked by the same 60 km and
|
||||
`Gemeente ...` as `coverage_scope=municipality`; regional Area clipping is
|
||||
stored as `bounded_selection`.
|
||||
|
||||
## Walloon WALOUS land cover and flood hazard
|
||||
|
||||
The Wallonia map flow uses bounded PICC vector products, the queryable legal
|
||||
SPW flood-hazard polygon layer and provisioned official WALOUS land-cover
|
||||
rasters. Provision the 2020 and 2023 source editions once in the persistent
|
||||
storage mount:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_walous_sources.py \
|
||||
--years 2020 2023 \
|
||||
--destination /app/storage/source-cache/walous
|
||||
```
|
||||
|
||||
The provisioner verifies advertised archive sizes, safe ZIP structure,
|
||||
EPSG:3812, one band, 1 m cells, class values 1-11 and SHA-256 checksums. It
|
||||
does not run at application startup. `GET .../datasets/walous/products`
|
||||
therefore reports `source_not_provisioned` until both source files exist.
|
||||
|
||||
For a bounded Walloon selection the browser persists the latest edition and
|
||||
all other configured comparable editions. `POST .../raster/walous/select`
|
||||
returns cell-area hectares; the temporal API compares the same semantic metric
|
||||
keys for 2020 and 2023. WALOUS is land cover, not legal land use, ownership,
|
||||
tree count, timber volume or water volume.
|
||||
|
||||
Settings: `WALOUS_ENABLED`, `WALOUS_SOURCE_DIR`,
|
||||
`WALOUS_ANALYSIS_RESOLUTION_M`, `WALOUS_MAX_SIDE_M` and
|
||||
`WALOUS_MAX_PIXELS`. The SPW flood polygon adapter uses
|
||||
`SPW_FLOOD_HAZARD_ENABLED` and `SPW_FLOOD_HAZARD_MAPSERVER_URL`.
|
||||
|
||||
The official Walloon 2021-2022 DTM is currently not an implicit runtime asset:
|
||||
the published 1 m whole-region artifact is about 41 GB and the 0.5 m INSPIRE
|
||||
artifact about 213 GB. A later operator capacity plan must define storage,
|
||||
partitioning and refresh before it can be called operational.
|
||||
|
||||
Provision the official DOV soil polygons for Mol through the existing vector
|
||||
upload path:
|
||||
|
||||
|
||||
@@ -97,6 +97,7 @@ from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisi
|
||||
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
|
||||
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
||||
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
||||
from app.services.walous_land_cover_service import WalousLandCoverService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}", tags=["datasets"])
|
||||
@@ -459,6 +460,33 @@ def list_thematic_raster_products(project_id: UUID, db: Session = Depends(get_db
|
||||
return envelope({"items": items, "total": len(items)})
|
||||
|
||||
|
||||
@router.post("/datasets/walous/acquire", response_model=Envelope[JobRead])
|
||||
def acquire_bounded_walous_land_cover(
|
||||
project_id: UUID,
|
||||
payload: ThematicRasterAcquireRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
job = JobService.run_sync_job(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
job_type="raster.walous.acquire",
|
||||
parameters=payload.model_dump(mode="json"),
|
||||
operation=lambda: WalousLandCoverService.acquire(db, project_id, payload),
|
||||
)
|
||||
return envelope(job)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/datasets/walous/products",
|
||||
response_model=Envelope[ItemList[ThematicRasterProductRead]],
|
||||
)
|
||||
def list_walous_products(project_id: UUID, db: Session = Depends(get_db)):
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
items = WalousLandCoverService.list_products()
|
||||
return envelope({"items": items, "total": len(items)})
|
||||
|
||||
|
||||
@router.get("/datasets", response_model=Envelope[DatasetList])
|
||||
def list_datasets(
|
||||
project_id: UUID,
|
||||
@@ -1006,6 +1034,33 @@ def raster_thematic_image(
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/datasets/{dataset_id}/raster/walous/select",
|
||||
response_model=Envelope[ThematicRasterSelectionResponse],
|
||||
)
|
||||
def raster_walous_selection(
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
payload: ThematicRasterSelectionRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
return envelope(WalousLandCoverService.analyze(db, project_id, dataset_id, payload))
|
||||
|
||||
|
||||
@router.get("/datasets/{dataset_id}/raster/walous/image")
|
||||
def raster_walous_image(
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
content = WalousLandCoverService.render_png(db, project_id, dataset_id)
|
||||
return Response(
|
||||
content=content,
|
||||
media_type="image/png",
|
||||
headers={"Cache-Control": "private, max-age=86400"},
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/datasets/{dataset_id}/raster/stats",
|
||||
response_model=Envelope[RasterStatsResponse],
|
||||
|
||||
@@ -160,6 +160,14 @@ class Settings(BaseSettings):
|
||||
),
|
||||
validation_alias="SPW_PICC_MAPSERVER_URL",
|
||||
)
|
||||
spw_flood_hazard_enabled: bool = Field(default=True, validation_alias="SPW_FLOOD_HAZARD_ENABLED")
|
||||
spw_flood_hazard_mapserver_url: str = Field(
|
||||
default=(
|
||||
"https://geoservices.wallonie.be/arcgis/rest/services/"
|
||||
"EAU/ALEA_INOND/MapServer"
|
||||
),
|
||||
validation_alias="SPW_FLOOD_HAZARD_MAPSERVER_URL",
|
||||
)
|
||||
urbis_enabled: bool = Field(default=True, validation_alias="URBIS_ENABLED")
|
||||
urbis_wfs_url: str = Field(
|
||||
default="https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows",
|
||||
@@ -273,6 +281,19 @@ class Settings(BaseSettings):
|
||||
thematic_raster_max_pixels: int = Field(default=30_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS")
|
||||
thematic_raster_timeout_seconds: int = Field(default=300, ge=1, validation_alias="THEMATIC_RASTER_TIMEOUT_SECONDS")
|
||||
thematic_raster_max_response_mb: int = Field(default=160, ge=1, validation_alias="THEMATIC_RASTER_MAX_RESPONSE_MB")
|
||||
walous_enabled: bool = Field(default=True, validation_alias="WALOUS_ENABLED")
|
||||
walous_source_dir: str = Field(
|
||||
default="/app/storage/source-cache/walous",
|
||||
validation_alias="WALOUS_SOURCE_DIR",
|
||||
)
|
||||
walous_analysis_resolution_m: float = Field(
|
||||
default=10.0,
|
||||
ge=1.0,
|
||||
le=100.0,
|
||||
validation_alias="WALOUS_ANALYSIS_RESOLUTION_M",
|
||||
)
|
||||
walous_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="WALOUS_MAX_SIDE_M")
|
||||
walous_max_pixels: int = Field(default=36_000_000, ge=1, validation_alias="WALOUS_MAX_PIXELS")
|
||||
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
|
||||
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
|
||||
sql_log_level: str = Field(default="WARNING", validation_alias="GEOINTEL_SQL_LOG_LEVEL")
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
@@ -32,6 +32,30 @@ class ThematicRasterProductRead(BaseModel):
|
||||
legend_max_label: str
|
||||
included_source_values: list[int]
|
||||
limitation_message: str
|
||||
analysis_resolution_m: float | None = None
|
||||
coverage_zones: list[str] = Field(default_factory=list)
|
||||
configured: bool = True
|
||||
status: str = "configured"
|
||||
|
||||
|
||||
class WalousAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
product_key: str
|
||||
display_name: str
|
||||
theme: str
|
||||
metric_kind: str
|
||||
resolution_m: float
|
||||
width: int
|
||||
height: int
|
||||
valid_pixel_count: int
|
||||
bbox_epsg4326: list[float]
|
||||
bbox_epsg3812: list[float]
|
||||
observation_year: int
|
||||
source_value_unit: str
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class ThematicRasterAcquisitionResult(BaseModel):
|
||||
|
||||
@@ -252,11 +252,11 @@ SOURCE_DEFINITIONS = (
|
||||
attribution="Service public de Wallonie",
|
||||
license_note="Consult the license of each Geoportail Wallonie product.",
|
||||
limitation_message=(
|
||||
"Bounded PICC buildings, road axes, hydrography and operator-imported SPW bathymetry are operational; "
|
||||
"other Walloon themes remain unavailable until separately governed."
|
||||
"Bounded PICC buildings, road axes and hydrography, the legally current flood-hazard polygons, "
|
||||
"and operator-imported SPW bathymetry are operational; other Walloon themes remain separately governed."
|
||||
),
|
||||
materialized_source_names=("spw_picc", "spw_bathymetry"),
|
||||
operational_themes=("buildings", "roads", "surface_water", "bathymetry"),
|
||||
materialized_source_names=("spw_picc", "spw_flood_hazard", "spw_walous_land_cover", "spw_bathymetry"),
|
||||
operational_themes=("buildings", "roads", "surface_water", "land_cover_use", "flood_climate", "bathymetry"),
|
||||
),
|
||||
_contract(
|
||||
source_name="urbis",
|
||||
@@ -375,6 +375,8 @@ REGIONAL_THEME_DATASETS: dict[str, dict[str, dict[str, tuple[str, ...]]]] = {
|
||||
"buildings": {"spw_picc": ("buildings",)},
|
||||
"roads": {"spw_picc": ("roads",)},
|
||||
"surface_water": {"spw_picc": ("water",)},
|
||||
"land_cover_use": {"spw_walous_land_cover": ()},
|
||||
"flood_climate": {"spw_flood_hazard": ("flood_hazard",)},
|
||||
"bathymetry": {"spw_bathymetry": ()},
|
||||
},
|
||||
"urbis": {
|
||||
|
||||
@@ -460,6 +460,75 @@ class OfficialVectorAcquisitionService:
|
||||
identity_field="GEOREF_ID",
|
||||
requires_coverage_area=True,
|
||||
),
|
||||
OfficialVectorProduct(
|
||||
key="spw_flood_hazard_2021",
|
||||
display_name="Waalse overstromingsgevaarkaart 2021",
|
||||
theme="flood_hazard",
|
||||
provider="Service public de Wallonie",
|
||||
source_name="spw_flood_hazard",
|
||||
reference_layer_name="flood_hazard",
|
||||
service_type="ArcGIS REST",
|
||||
collection="2",
|
||||
source_crs="EPSG:31370",
|
||||
source_version="2021-03-04",
|
||||
observation_label="Juridisch geldende toestand 2021",
|
||||
authority_level="authoritative",
|
||||
catalog_url=(
|
||||
"https://geoportail.wallonie.be/catalogue/"
|
||||
"14084108-2c7b-4091-b62d-ff0fc235213a.html"
|
||||
),
|
||||
attribution="Service public de Wallonie (SPW) - Cartographie de l'alea d'inondation",
|
||||
license_note="CC BY 4.0; cite SPW and identify modifications.",
|
||||
limitation_message=(
|
||||
"Juridische gevarenkaart voor overstroming door waterloopoverloop en afstroming. "
|
||||
"Dit is geen actuele overstroming, gemeten waterdiepte, voorspelling of bathymetrie."
|
||||
),
|
||||
source="SPW flood-hazard ArcGIS REST",
|
||||
observed_at=datetime(2021, 3, 4, tzinfo=UTC),
|
||||
valid_from=datetime(2021, 3, 4, tzinfo=UTC),
|
||||
valid_to=None,
|
||||
primary_metric={
|
||||
"metric_key": "flood_hazard_area",
|
||||
"method": "intersection_area",
|
||||
"label": "Oppervlakte met overstromingsgevaar",
|
||||
"unit": "ha",
|
||||
"geometry_dimension": 2,
|
||||
"is_estimate": False,
|
||||
},
|
||||
selection_metrics=(
|
||||
{
|
||||
"metric_key": "flood_hazard_high_area",
|
||||
"method": "intersection_area",
|
||||
"label": "Hoog overstromingsgevaar",
|
||||
"unit": "ha",
|
||||
"geometry_dimension": 2,
|
||||
"filter_property": "CLASSEMENT",
|
||||
"filter_values": [130, 230, 330, "130", "230", "330"],
|
||||
},
|
||||
{
|
||||
"metric_key": "flood_hazard_medium_area",
|
||||
"method": "intersection_area",
|
||||
"label": "Middelgroot overstromingsgevaar",
|
||||
"unit": "ha",
|
||||
"geometry_dimension": 2,
|
||||
"filter_property": "CLASSEMENT",
|
||||
"filter_values": [120, 220, 320, "120", "220", "320"],
|
||||
},
|
||||
{
|
||||
"metric_key": "flood_hazard_low_area",
|
||||
"method": "intersection_area",
|
||||
"label": "Laag overstromingsgevaar",
|
||||
"unit": "ha",
|
||||
"geometry_dimension": 2,
|
||||
"filter_property": "CLASSEMENT",
|
||||
"filter_values": [110, 210, 310, "110", "210", "310"],
|
||||
},
|
||||
),
|
||||
coverage_zones=("wallonia",),
|
||||
endpoint_kind="spw_flood_arcgis",
|
||||
identity_field="LOCALID",
|
||||
requires_coverage_area=True,
|
||||
),
|
||||
OfficialVectorProduct(
|
||||
key="urbis_buildings",
|
||||
display_name="UrbIS buildings",
|
||||
@@ -869,6 +938,12 @@ class OfficialVectorAcquisitionService:
|
||||
message="Bounded SPW PICC acquisition is disabled",
|
||||
status_code=503,
|
||||
)
|
||||
if product.endpoint_kind == "spw_flood_arcgis" and not settings.spw_flood_hazard_enabled:
|
||||
raise AppError(
|
||||
code="SPW_FLOOD_HAZARD_NOT_CONFIGURED",
|
||||
message="Bounded SPW flood-hazard acquisition is disabled",
|
||||
status_code=503,
|
||||
)
|
||||
if product.endpoint_kind == "urbis_wfs" and not settings.urbis_enabled:
|
||||
raise AppError(
|
||||
code="URBIS_NOT_CONFIGURED",
|
||||
@@ -1132,7 +1207,11 @@ class OfficialVectorAcquisitionService:
|
||||
"f": "geojson",
|
||||
}
|
||||
)
|
||||
base = settings.spw_picc_mapserver_url.rstrip("/")
|
||||
base = (
|
||||
settings.spw_flood_hazard_mapserver_url
|
||||
if product.endpoint_kind == "spw_flood_arcgis"
|
||||
else settings.spw_picc_mapserver_url
|
||||
).rstrip("/")
|
||||
return f"{base}/{product.collection}/query?{query}"
|
||||
|
||||
@staticmethod
|
||||
@@ -1178,7 +1257,7 @@ class OfficialVectorAcquisitionService:
|
||||
tuple(scope_metric.bounds),
|
||||
start_index,
|
||||
)
|
||||
if product.endpoint_kind == "spw_arcgis":
|
||||
if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis"}:
|
||||
return OfficialVectorAcquisitionService._spw_url(
|
||||
settings,
|
||||
product,
|
||||
@@ -1209,6 +1288,7 @@ class OfficialVectorAcquisitionService:
|
||||
"bwk_wfs": settings.bwk_wfs_url,
|
||||
"dov_wfs": settings.dov_soil_wfs_url,
|
||||
"spw_arcgis": settings.spw_picc_mapserver_url,
|
||||
"spw_flood_arcgis": settings.spw_flood_hazard_mapserver_url,
|
||||
"urbis_wfs": settings.urbis_wfs_url,
|
||||
}.get(product.endpoint_kind)
|
||||
if configured_url is None:
|
||||
@@ -1220,7 +1300,7 @@ class OfficialVectorAcquisitionService:
|
||||
base = urlparse(configured_url)
|
||||
expected_path = (
|
||||
f"{base.path.rstrip('/')}/{product.collection}/query"
|
||||
if product.endpoint_kind == "spw_arcgis"
|
||||
if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis"}
|
||||
else base.path
|
||||
)
|
||||
if (
|
||||
@@ -1361,7 +1441,7 @@ class OfficialVectorAcquisitionService:
|
||||
scope_metric: Any,
|
||||
coverage_scope: str,
|
||||
) -> dict[str, Any] | None:
|
||||
if product.endpoint_kind in {"spw_arcgis", "urbis_wfs"}:
|
||||
if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis", "urbis_wfs"}:
|
||||
return OfficialVectorAcquisitionService._normalize_regional_feature(
|
||||
product,
|
||||
feature,
|
||||
@@ -1601,7 +1681,7 @@ class OfficialVectorAcquisitionService:
|
||||
)
|
||||
start_index += returned_count
|
||||
arcgis_has_more = payload.get("exceededTransferLimit") is True
|
||||
if product.endpoint_kind == "spw_arcgis":
|
||||
if product.endpoint_kind in {"spw_arcgis", "spw_flood_arcgis"}:
|
||||
if arcgis_has_more and returned_count == 0:
|
||||
raise AppError(
|
||||
code="OFFICIAL_VECTOR_PROVIDER_INCOMPLETE_RESPONSE",
|
||||
|
||||
@@ -22,7 +22,9 @@ from app.schemas.temporal import (
|
||||
TemporalSeriesDataset,
|
||||
TemporalSeriesRead,
|
||||
)
|
||||
from app.schemas.thematic_raster import ThematicRasterSelectionRequest
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
from app.services.walous_land_cover_service import WalousLandCoverService
|
||||
|
||||
|
||||
class TemporalAnalysisService:
|
||||
@@ -31,6 +33,7 @@ class TemporalAnalysisService:
|
||||
"provision_regional_grb_buildings.py",
|
||||
"provision_regional_grb_context.py",
|
||||
}
|
||||
SUPPORTED_RASTER_TEMPORAL_SOURCES = {WalousLandCoverService.PROVIDER}
|
||||
|
||||
@staticmethod
|
||||
def _canonical_observation_snapshots(datasets: list[Dataset]) -> list[Dataset]:
|
||||
@@ -141,6 +144,15 @@ class TemporalAnalysisService:
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
if earlier.dataset_type == "raster" or later.dataset_type == "raster":
|
||||
return TemporalAnalysisService._compare_walous_rasters(
|
||||
db,
|
||||
project_id=project_id,
|
||||
payload=payload,
|
||||
earlier=earlier,
|
||||
later=later,
|
||||
)
|
||||
|
||||
bbox = payload.bbox.model_dump()
|
||||
selection_area = TemporalAnalysisService._get_selection_area(db, project_id, payload.area_id)
|
||||
selection_geometry = None
|
||||
@@ -259,6 +271,90 @@ class TemporalAnalysisService:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
return area
|
||||
|
||||
@staticmethod
|
||||
def _compare_walous_rasters(
|
||||
db: Session,
|
||||
*,
|
||||
project_id: UUID,
|
||||
payload: TemporalComparisonRequest,
|
||||
earlier: Dataset,
|
||||
later: Dataset,
|
||||
) -> TemporalComparisonResponse:
|
||||
if {
|
||||
earlier.dataset_type,
|
||||
later.dataset_type,
|
||||
} != {"raster"} or earlier.source_name != WalousLandCoverService.PROVIDER or later.source_name != WalousLandCoverService.PROVIDER:
|
||||
raise AppError(
|
||||
code="INCOMPATIBLE_TEMPORAL_DATASET_TYPES",
|
||||
message="Raster evolution currently supports only two governed WALOUS land-cover snapshots",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
request = ThematicRasterSelectionRequest(bbox=payload.bbox, area_id=payload.area_id)
|
||||
summaries: dict[UUID, dict[str, Any]] = {}
|
||||
|
||||
def summarize(dataset: Dataset) -> dict[str, Any]:
|
||||
cached = summaries.get(dataset.id)
|
||||
if cached is not None:
|
||||
return cached
|
||||
result = WalousLandCoverService.analyze(db, project_id, dataset.id, request)
|
||||
summary = dict(result["summary"])
|
||||
summary["warning"] = result.get("limitation_message")
|
||||
summaries[dataset.id] = summary
|
||||
return summary
|
||||
|
||||
earlier_summary = summarize(earlier)
|
||||
later_summary = summarize(later)
|
||||
metric_comparisons = TemporalAnalysisService._compare_summary_metrics(earlier_summary, later_summary)
|
||||
if not metric_comparisons:
|
||||
raise AppError(
|
||||
code="INCOMPATIBLE_TEMPORAL_AGGREGATION",
|
||||
message="WALOUS snapshots use incompatible aggregation semantics",
|
||||
status_code=400,
|
||||
)
|
||||
primary_key = str(later_summary.get("primary_metric_key") or metric_comparisons[0].metric_key)
|
||||
primary_metric = next(
|
||||
(metric for metric in metric_comparisons if metric.metric_key == primary_key),
|
||||
metric_comparisons[0],
|
||||
)
|
||||
timeline = TemporalAnalysisService._build_timeline(
|
||||
db,
|
||||
project_id=project_id,
|
||||
series_key=str(earlier.temporal_series_key),
|
||||
fallback_datasets=[earlier, later],
|
||||
summarize=summarize,
|
||||
)
|
||||
warnings = [
|
||||
"WALOUS-evolutie vergelijkt celgebaseerde landbedekkingsoppervlakten; individuele objectwijzigingen zijn niet beschikbaar.",
|
||||
]
|
||||
limitation = str(later_summary.get("warning") or earlier_summary.get("warning") or "").strip()
|
||||
if limitation:
|
||||
warnings.append(limitation)
|
||||
return TemporalComparisonResponse(
|
||||
temporal_series_key=str(earlier.temporal_series_key),
|
||||
earlier=TemporalDatasetRef(
|
||||
id=earlier.id,
|
||||
name=earlier.name,
|
||||
observed_at=earlier.observed_at,
|
||||
source_version=earlier.source_version,
|
||||
),
|
||||
later=TemporalDatasetRef(
|
||||
id=later.id,
|
||||
name=later.name,
|
||||
observed_at=later.observed_at,
|
||||
source_version=later.source_version,
|
||||
),
|
||||
selection_bbox=payload.bbox,
|
||||
selection_area_id=payload.area_id,
|
||||
metric=primary_metric,
|
||||
metrics=metric_comparisons,
|
||||
timeline=timeline,
|
||||
object_changes=TemporalObjectChanges(available=False),
|
||||
geojson={"type": "FeatureCollection", "features": []},
|
||||
warnings=warnings,
|
||||
generated_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _summary_metrics(summary: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
configured = summary.get("metrics")
|
||||
@@ -370,10 +466,15 @@ class TemporalAnalysisService:
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if not dataset or dataset.project_id != project_id:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message=f"{label} dataset not found", status_code=404)
|
||||
if dataset.dataset_type not in {"vector", "geojson"}:
|
||||
supported_vector = dataset.dataset_type in {"vector", "geojson"}
|
||||
supported_raster = (
|
||||
dataset.dataset_type == "raster"
|
||||
and dataset.source_name in TemporalAnalysisService.SUPPORTED_RASTER_TEMPORAL_SOURCES
|
||||
)
|
||||
if not supported_vector and not supported_raster:
|
||||
raise AppError(
|
||||
code="DATASET_NOT_VECTOR",
|
||||
message="Temporal selection comparison currently requires vector datasets",
|
||||
code="TEMPORAL_DATASET_NOT_SUPPORTED",
|
||||
message="Temporal comparison requires a vector series or a governed WALOUS raster series",
|
||||
status_code=400,
|
||||
)
|
||||
if not dataset.temporal_series_key or not dataset.observed_at:
|
||||
|
||||
@@ -0,0 +1,551 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import box, mapping
|
||||
from shapely.ops import transform as shapely_transform
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.schemas.thematic_raster import (
|
||||
ThematicRasterAcquireRequest,
|
||||
ThematicRasterMetric,
|
||||
ThematicRasterProductRead,
|
||||
ThematicRasterSelectionRequest,
|
||||
ThematicRasterSelectionResponse,
|
||||
ThematicRasterSelectionSummary,
|
||||
WalousAcquisitionResult,
|
||||
)
|
||||
from app.services.dataset_service import DatasetService
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WalousProduct:
|
||||
key: str
|
||||
display_name: str
|
||||
observation_year: int
|
||||
source_filename: str
|
||||
source_version: str
|
||||
catalog_url: str
|
||||
download_url: str
|
||||
source_sha256_filename: str
|
||||
accuracy_label: str
|
||||
|
||||
|
||||
class WalousLandCoverService:
|
||||
PROVIDER = "spw_walous_land_cover"
|
||||
SOURCE_CRS = "EPSG:3812"
|
||||
SOURCE_RESOLUTION_M = 1.0
|
||||
SOURCE_VALUE_UNIT = "class_1_11"
|
||||
THEME = "land_cover_use"
|
||||
METRIC_KIND = "categorical_area"
|
||||
NODATA = 255
|
||||
ATTRIBUTION = "Service public de Wallonie (SPW), Aerospacelab S.A."
|
||||
LICENSE_NOTE = "CC BY 4.0; cite the official SPW WALOUS edition and identify modifications."
|
||||
LIMITATION = (
|
||||
"GeoIntel analyseert een nearest-neighbour afgeleide van het officiele 1 m WALOUS-raster op de "
|
||||
"geconfigureerde analyseresolutie. Oppervlakten zijn celgebaseerde schattingen; de kaart is landbedekking, "
|
||||
"geen juridisch landgebruik, eigendom, boomtelling of actuele terreinwaarneming."
|
||||
)
|
||||
CLASS_LABELS = {
|
||||
1: "Jaarlijks wisselende kruidlaag",
|
||||
2: "Jaarronde kruidlaag",
|
||||
3: "Naaldbomen hoger dan 3 m",
|
||||
4: "Loofbomen hoger dan 3 m",
|
||||
5: "Naaldbomen tot 3 m",
|
||||
6: "Loofbomen tot 3 m",
|
||||
7: "Kale bodem",
|
||||
8: "Oppervlaktewater",
|
||||
9: "Kunstmatige bodembedekking",
|
||||
10: "Spoorweg",
|
||||
11: "Kunstmatige constructies boven maaiveld",
|
||||
}
|
||||
CLASS_COLORS = {
|
||||
1: (236, 202, 73),
|
||||
2: (161, 201, 78),
|
||||
3: (28, 89, 51),
|
||||
4: (52, 132, 72),
|
||||
5: (78, 125, 70),
|
||||
6: (107, 164, 87),
|
||||
7: (194, 165, 119),
|
||||
8: (44, 129, 185),
|
||||
9: (155, 155, 155),
|
||||
10: (68, 68, 68),
|
||||
11: (183, 72, 67),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _products() -> dict[str, WalousProduct]:
|
||||
products = (
|
||||
WalousProduct(
|
||||
key="walous_land_cover_2020",
|
||||
display_name="WALOUS landbedekking 2020",
|
||||
observation_year=2020,
|
||||
source_filename="walous_land_cover_2020_3812.tif",
|
||||
source_version="WAL_OCS_IA__2020",
|
||||
catalog_url="https://geoportail.wallonie.be/catalogue/47b348f1-6e7a-4baa-963c-0232a43c0cff.html",
|
||||
download_url=(
|
||||
"https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
|
||||
"47b348f1-6e7a-4baa-963c-0232a43c0cff/WAL_OCS_IA__2020_GEOTIFF_3812.zip"
|
||||
),
|
||||
source_sha256_filename="walous_land_cover_2020_3812.sha256",
|
||||
accuracy_label="Officiele globale nauwkeurigheid 83,30%",
|
||||
),
|
||||
WalousProduct(
|
||||
key="walous_land_cover_2023",
|
||||
display_name="WALOUS landbedekking 2023",
|
||||
observation_year=2023,
|
||||
source_filename="walous_land_cover_2023_3812.tif",
|
||||
source_version="WAL_OCS_IA__2023",
|
||||
catalog_url="https://geoportail.wallonie.be/catalogue/4e780ba1-463c-478e-95df-d2f1963a150d.html",
|
||||
download_url=(
|
||||
"https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
|
||||
"4e780ba1-463c-478e-95df-d2f1963a150d/WAL_OCS_IA__2023_GEOTIFF_3812.zip"
|
||||
),
|
||||
source_sha256_filename="walous_land_cover_2023_3812.sha256",
|
||||
accuracy_label="Officiele globale nauwkeurigheid 87,10%",
|
||||
),
|
||||
)
|
||||
return {product.key: product for product in products}
|
||||
|
||||
@staticmethod
|
||||
def _source_path(settings: Settings, product: WalousProduct) -> Path:
|
||||
return Path(settings.walous_source_dir) / product.source_filename
|
||||
|
||||
@staticmethod
|
||||
def list_products(*, settings: Settings | None = None) -> list[dict[str, Any]]:
|
||||
resolved = settings or get_settings()
|
||||
result: list[dict[str, Any]] = []
|
||||
for product in WalousLandCoverService._products().values():
|
||||
configured = resolved.walous_enabled and WalousLandCoverService._source_path(resolved, product).is_file()
|
||||
result.append(
|
||||
ThematicRasterProductRead(
|
||||
key=product.key,
|
||||
display_name=product.display_name,
|
||||
theme=WalousLandCoverService.THEME,
|
||||
metric_kind=WalousLandCoverService.METRIC_KIND,
|
||||
coverage_id=product.source_version,
|
||||
native_resolution_m=WalousLandCoverService.SOURCE_RESOLUTION_M,
|
||||
analysis_resolution_m=resolved.walous_analysis_resolution_m,
|
||||
source_crs=WalousLandCoverService.SOURCE_CRS,
|
||||
source_value_unit=WalousLandCoverService.SOURCE_VALUE_UNIT,
|
||||
observation_year=product.observation_year,
|
||||
source_version=product.source_version,
|
||||
catalog_url=product.catalog_url,
|
||||
attribution=WalousLandCoverService.ATTRIBUTION,
|
||||
license_note=WalousLandCoverService.LICENSE_NOTE,
|
||||
legend_min_label="WALOUS klasse 1",
|
||||
legend_max_label="WALOUS klasse 11",
|
||||
included_source_values=list(WalousLandCoverService.CLASS_LABELS),
|
||||
limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.",
|
||||
coverage_zones=["wallonia"],
|
||||
configured=configured,
|
||||
status="configured" if configured else "source_not_provisioned",
|
||||
).model_dump()
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _product(product_key: str) -> WalousProduct:
|
||||
product = WalousLandCoverService._products().get(product_key.strip().lower())
|
||||
if product is None:
|
||||
raise AppError(
|
||||
code="WALOUS_PRODUCT_NOT_SUPPORTED",
|
||||
message="Select a product from the governed WALOUS registry",
|
||||
details={"product_key": product_key},
|
||||
status_code=422,
|
||||
)
|
||||
return product
|
||||
|
||||
@staticmethod
|
||||
def _scope_geometry(db, project_id: UUID, payload: ThematicRasterAcquireRequest):
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
if payload.bbox.crs.upper() != "EPSG:4326":
|
||||
raise AppError(code="INVALID_BBOX_CRS", message="WALOUS acquisition requires EPSG:4326", status_code=400)
|
||||
values = [payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y]
|
||||
if not all(math.isfinite(value) for value in values) or values[0] >= values[2] or values[1] >= values[3]:
|
||||
raise AppError(code="INVALID_BBOX", message="WALOUS selection must be a finite non-empty rectangle", status_code=400)
|
||||
selection = box(*values)
|
||||
if payload.area_id is None:
|
||||
return selection, values
|
||||
area = db.get(Area, payload.area_id)
|
||||
if area is None or area.project_id != project_id:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
selection = selection.intersection(to_shape(area.geometry))
|
||||
if selection.is_empty or selection.area <= 0:
|
||||
raise AppError(code="WALOUS_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422)
|
||||
return selection, values
|
||||
|
||||
@staticmethod
|
||||
def _read_source_window(
|
||||
source_path: Path,
|
||||
scope_4326,
|
||||
settings: Settings,
|
||||
) -> tuple[bytes, dict[str, Any]]:
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from rasterio.enums import Resampling
|
||||
from rasterio.features import geometry_mask
|
||||
from rasterio.io import MemoryFile
|
||||
from rasterio.transform import from_bounds
|
||||
from rasterio.windows import from_bounds as window_from_bounds
|
||||
except ImportError as exc:
|
||||
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for WALOUS", status_code=503) from exc
|
||||
|
||||
resolution = float(settings.walous_analysis_resolution_m)
|
||||
transformer = Transformer.from_crs("EPSG:4326", WalousLandCoverService.SOURCE_CRS, always_xy=True)
|
||||
scope_metric = shapely_transform(transformer.transform, scope_4326)
|
||||
try:
|
||||
with rasterio.open(source_path) as source:
|
||||
if source.crs is None or source.crs.to_epsg() != 3812 or source.count != 1:
|
||||
raise AppError(code="WALOUS_SOURCE_INVALID", message="WALOUS source must be a one-band EPSG:3812 raster", status_code=409)
|
||||
if not all(math.isclose(abs(float(value)), 1.0, abs_tol=0.05) for value in source.res):
|
||||
raise AppError(code="WALOUS_SOURCE_INVALID", message="WALOUS source must retain the official 1 m resolution", status_code=409)
|
||||
clipped_geometry = scope_metric.intersection(box(*source.bounds))
|
||||
if clipped_geometry.is_empty or clipped_geometry.area <= 0:
|
||||
raise AppError(code="WALOUS_SELECTION_OUTSIDE_COVERAGE", message="Selection does not overlap WALOUS coverage", status_code=422)
|
||||
min_x, min_y, max_x, max_y = clipped_geometry.bounds
|
||||
bounds = (
|
||||
math.floor(min_x / resolution) * resolution,
|
||||
math.floor(min_y / resolution) * resolution,
|
||||
math.ceil(max_x / resolution) * resolution,
|
||||
math.ceil(max_y / resolution) * resolution,
|
||||
)
|
||||
width_m, height_m = bounds[2] - bounds[0], bounds[3] - bounds[1]
|
||||
if width_m > settings.walous_max_side_m or height_m > settings.walous_max_side_m:
|
||||
raise AppError(
|
||||
code="WALOUS_SELECTION_TOO_LARGE",
|
||||
message=f"Select no more than {settings.walous_max_side_m:g} by {settings.walous_max_side_m:g} metres",
|
||||
details={"width_m": width_m, "height_m": height_m},
|
||||
status_code=422,
|
||||
)
|
||||
width, height = max(1, round(width_m / resolution)), max(1, round(height_m / resolution))
|
||||
if width * height > settings.walous_max_pixels:
|
||||
raise AppError(code="WALOUS_SELECTION_TOO_LARGE", message="WALOUS selection exceeds the configured cell limit", details={"pixel_count": width * height, "max_pixels": settings.walous_max_pixels}, status_code=422)
|
||||
window = window_from_bounds(*bounds, transform=source.transform)
|
||||
band = source.read(1, window=window, out_shape=(height, width), masked=True, resampling=Resampling.nearest)
|
||||
output_transform = from_bounds(*bounds, width, height)
|
||||
outside_scope = geometry_mask([mapping(clipped_geometry)], out_shape=(height, width), transform=output_transform, invert=False)
|
||||
raw = np.asarray(band.filled(WalousLandCoverService.NODATA), dtype="uint8")
|
||||
invalid = np.ma.getmaskarray(band) | outside_scope
|
||||
if source.nodata is not None:
|
||||
invalid |= np.isclose(raw.astype("float64"), float(source.nodata))
|
||||
raw[invalid] = WalousLandCoverService.NODATA
|
||||
valid = raw[raw != WalousLandCoverService.NODATA]
|
||||
if valid.size == 0:
|
||||
raise AppError(code="WALOUS_NO_VALID_DATA", message="WALOUS contains no valid cells in this selection", status_code=422)
|
||||
classes = set(np.unique(valid).astype(int).tolist())
|
||||
unexpected = sorted(classes - set(WalousLandCoverService.CLASS_LABELS))
|
||||
if unexpected:
|
||||
raise AppError(code="WALOUS_SOURCE_INVALID_VALUES", message="WALOUS contains classes outside the governed 1-11 legend", details={"unexpected_classes": unexpected}, status_code=409)
|
||||
profile = {
|
||||
"driver": "GTiff",
|
||||
"width": width,
|
||||
"height": height,
|
||||
"count": 1,
|
||||
"dtype": "uint8",
|
||||
"crs": WalousLandCoverService.SOURCE_CRS,
|
||||
"transform": output_transform,
|
||||
"nodata": WalousLandCoverService.NODATA,
|
||||
"compress": "deflate",
|
||||
"predictor": 2,
|
||||
}
|
||||
with MemoryFile() as memory:
|
||||
with memory.open(**profile) as output:
|
||||
output.write(raw, 1)
|
||||
content = memory.read()
|
||||
return content, {
|
||||
"width": width,
|
||||
"height": height,
|
||||
"valid_pixel_count": int(valid.size),
|
||||
"classes_present": sorted(classes),
|
||||
"bbox_epsg3812": list(bounds),
|
||||
"source_width": int(source.width),
|
||||
"source_height": int(source.height),
|
||||
"source_nodata": None if source.nodata is None else float(source.nodata),
|
||||
"source_resolution_m": 1.0,
|
||||
"analysis_resolution_m": resolution,
|
||||
}
|
||||
except AppError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AppError(code="WALOUS_SOURCE_READ_FAILED", message="The provisioned WALOUS source could not be read", details={"reason": str(exc)}, status_code=500) from exc
|
||||
|
||||
@staticmethod
|
||||
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
|
||||
candidate = (
|
||||
db.query(Dataset)
|
||||
.filter(Dataset.project_id == project_id, Dataset.name == filename, Dataset.source_name == WalousLandCoverService.PROVIDER, Dataset.status == "ready")
|
||||
.order_by(Dataset.imported_at.desc())
|
||||
.first()
|
||||
)
|
||||
return candidate if candidate and candidate.storage_path and Path(candidate.storage_path).is_file() else None
|
||||
|
||||
@staticmethod
|
||||
def acquire(db, project_id: UUID, payload: ThematicRasterAcquireRequest, *, settings: Settings | None = None) -> dict[str, Any]:
|
||||
resolved = settings or get_settings()
|
||||
if not resolved.walous_enabled:
|
||||
raise AppError(code="WALOUS_NOT_CONFIGURED", message="WALOUS bounded analysis is disabled", status_code=503)
|
||||
product = WalousLandCoverService._product(payload.product_key)
|
||||
source_path = WalousLandCoverService._source_path(resolved, product)
|
||||
if not source_path.is_file():
|
||||
raise AppError(
|
||||
code="WALOUS_SOURCE_NOT_PROVISIONED",
|
||||
message="The official WALOUS source archive has not been provisioned on this runtime",
|
||||
details={"expected_path": str(source_path), "operator_command": "python scripts/provision_walous_sources.py --years 2020 2023"},
|
||||
status_code=503,
|
||||
)
|
||||
scope, bbox_4326 = WalousLandCoverService._scope_geometry(db, project_id, payload)
|
||||
identity = {
|
||||
"product_key": product.key,
|
||||
"bbox_epsg4326": [round(float(value), 8) for value in bbox_4326],
|
||||
"area_id": str(payload.area_id) if payload.area_id else None,
|
||||
"analysis_resolution_m": resolved.walous_analysis_resolution_m,
|
||||
}
|
||||
request_hash = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()
|
||||
filename = f"walous_{product.observation_year}_{request_hash[:12]}_3812.tif"
|
||||
if not payload.force_refresh:
|
||||
cached = WalousLandCoverService._cached_dataset(db, project_id, filename)
|
||||
if cached is not None:
|
||||
metadata = cached.source_metadata or {}
|
||||
return WalousAcquisitionResult(
|
||||
output_dataset_id=cached.id,
|
||||
reused=True,
|
||||
provider=WalousLandCoverService.PROVIDER,
|
||||
product_key=product.key,
|
||||
display_name=product.display_name,
|
||||
theme=WalousLandCoverService.THEME,
|
||||
metric_kind=WalousLandCoverService.METRIC_KIND,
|
||||
resolution_m=float(metadata.get("analysis_resolution_m", resolved.walous_analysis_resolution_m)),
|
||||
width=int((cached.metadata_json or {}).get("width", 0)),
|
||||
height=int((cached.metadata_json or {}).get("height", 0)),
|
||||
valid_pixel_count=int(metadata.get("valid_pixel_count", 0)),
|
||||
bbox_epsg4326=bbox_4326,
|
||||
bbox_epsg3812=list(metadata.get("bbox_epsg3812") or []),
|
||||
observation_year=product.observation_year,
|
||||
source_value_unit=WalousLandCoverService.SOURCE_VALUE_UNIT,
|
||||
attribution=WalousLandCoverService.ATTRIBUTION,
|
||||
limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.",
|
||||
).model_dump(mode="json")
|
||||
|
||||
content, validation = WalousLandCoverService._read_source_window(source_path, scope, resolved)
|
||||
source_sha256_path = source_path.with_name(product.source_sha256_filename)
|
||||
source_sha256 = source_sha256_path.read_text(encoding="ascii").strip().split()[0] if source_sha256_path.is_file() else None
|
||||
acquired_at = datetime.now(UTC)
|
||||
observed_at = datetime(product.observation_year, 12, 31, 23, 59, 59, tzinfo=UTC)
|
||||
spatial_series_hash = hashlib.sha256(json.dumps({"bbox": identity["bbox_epsg4326"], "area_id": identity["area_id"], "resolution": identity["analysis_resolution_m"]}, sort_keys=True).encode()).hexdigest()[:24]
|
||||
dataset = DatasetService.import_raster_bytes(
|
||||
db,
|
||||
project_id=project_id,
|
||||
area_id=payload.area_id,
|
||||
filename=filename,
|
||||
content=content,
|
||||
source=f"SPW WALOUS {product.source_version} operator-provisioned GeoTIFF",
|
||||
source_name=WalousLandCoverService.PROVIDER,
|
||||
temporal_series_key=f"spw:walous:land-cover:{spatial_series_hash}",
|
||||
observed_at=observed_at,
|
||||
valid_from=datetime(product.observation_year, 1, 1, tzinfo=UTC),
|
||||
valid_to=observed_at,
|
||||
temporal_granularity="year",
|
||||
source_version=product.source_version,
|
||||
source_metadata={
|
||||
"provider": WalousLandCoverService.PROVIDER,
|
||||
"service": "official_predefined_dataset_atom",
|
||||
"product_key": product.key,
|
||||
"product_display_name": product.display_name,
|
||||
"theme": WalousLandCoverService.THEME,
|
||||
"metric_kind": WalousLandCoverService.METRIC_KIND,
|
||||
"source_crs": WalousLandCoverService.SOURCE_CRS,
|
||||
"source_resolution_m": WalousLandCoverService.SOURCE_RESOLUTION_M,
|
||||
"analysis_resolution_m": validation["analysis_resolution_m"],
|
||||
"source_value_unit": WalousLandCoverService.SOURCE_VALUE_UNIT,
|
||||
"class_labels": WalousLandCoverService.CLASS_LABELS,
|
||||
"observation_year": product.observation_year,
|
||||
"valid_pixel_count": validation["valid_pixel_count"],
|
||||
"classes_present": validation["classes_present"],
|
||||
"bbox_epsg4326": bbox_4326,
|
||||
"bbox_epsg3812": validation["bbox_epsg3812"],
|
||||
"coverage_zones": ["wallonia"],
|
||||
"catalog_url": product.catalog_url,
|
||||
"download_url": product.download_url,
|
||||
"attribution": WalousLandCoverService.ATTRIBUTION,
|
||||
"license_note": WalousLandCoverService.LICENSE_NOTE,
|
||||
"limitation_message": f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.",
|
||||
},
|
||||
provenance_metadata={
|
||||
"acquisition": "operator_provisioned_official_archive_bounded_window",
|
||||
"acquired_at": acquired_at.isoformat(),
|
||||
"request_hash": request_hash,
|
||||
"source_filename": product.source_filename,
|
||||
"source_sha256": source_sha256,
|
||||
"derived_sha256": hashlib.sha256(content).hexdigest(),
|
||||
"resampling": "nearest",
|
||||
"validation": validation,
|
||||
},
|
||||
)
|
||||
return WalousAcquisitionResult(
|
||||
output_dataset_id=dataset.id,
|
||||
reused=False,
|
||||
provider=WalousLandCoverService.PROVIDER,
|
||||
product_key=product.key,
|
||||
display_name=product.display_name,
|
||||
theme=WalousLandCoverService.THEME,
|
||||
metric_kind=WalousLandCoverService.METRIC_KIND,
|
||||
resolution_m=validation["analysis_resolution_m"],
|
||||
width=validation["width"],
|
||||
height=validation["height"],
|
||||
valid_pixel_count=validation["valid_pixel_count"],
|
||||
bbox_epsg4326=bbox_4326,
|
||||
bbox_epsg3812=validation["bbox_epsg3812"],
|
||||
observation_year=product.observation_year,
|
||||
source_value_unit=WalousLandCoverService.SOURCE_VALUE_UNIT,
|
||||
attribution=WalousLandCoverService.ATTRIBUTION,
|
||||
limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.",
|
||||
).model_dump(mode="json")
|
||||
|
||||
@staticmethod
|
||||
def _load_dataset(db, project_id: UUID, dataset_id: UUID) -> tuple[Dataset, WalousProduct]:
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if not dataset or dataset.project_id != project_id:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||
if dataset.dataset_type != "raster" or dataset.source_name != WalousLandCoverService.PROVIDER:
|
||||
raise AppError(code="INVALID_WALOUS_DATASET", message="WALOUS analysis requires a governed WALOUS raster", status_code=400)
|
||||
if dataset.status != "ready" or not dataset.storage_path or not Path(dataset.storage_path).is_file():
|
||||
raise AppError(code="DATASET_FILE_MISSING", message="Persisted WALOUS raster is unavailable", status_code=404)
|
||||
product = WalousLandCoverService._product(str((dataset.source_metadata or {}).get("product_key") or ""))
|
||||
return dataset, product
|
||||
|
||||
@staticmethod
|
||||
def _analysis_geometry(db, project_id: UUID, payload: ThematicRasterSelectionRequest):
|
||||
selection = box(payload.bbox.min_x, payload.bbox.min_y, payload.bbox.max_x, payload.bbox.max_y)
|
||||
if payload.area_id is None:
|
||||
return selection
|
||||
area = db.get(Area, payload.area_id)
|
||||
if area is None or area.project_id != project_id:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
selection = selection.intersection(to_shape(area.geometry))
|
||||
if selection.is_empty or selection.area <= 0:
|
||||
raise AppError(code="WALOUS_SELECTION_OUTSIDE_AREA", message="Selection does not overlap the selected work area", status_code=422)
|
||||
return selection
|
||||
|
||||
@staticmethod
|
||||
def analyze(db, project_id: UUID, dataset_id: UUID, payload: ThematicRasterSelectionRequest) -> dict[str, Any]:
|
||||
dataset, product = WalousLandCoverService._load_dataset(db, project_id, dataset_id)
|
||||
selection_4326 = WalousLandCoverService._analysis_geometry(db, project_id, payload)
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from rasterio.features import geometry_mask
|
||||
from rasterio.mask import mask
|
||||
except ImportError as exc:
|
||||
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for WALOUS analysis", status_code=503) from exc
|
||||
try:
|
||||
with rasterio.open(dataset.storage_path) as source:
|
||||
transformer = Transformer.from_crs("EPSG:4326", source.crs, always_xy=True)
|
||||
selection_metric = shapely_transform(transformer.transform, selection_4326)
|
||||
geometry = selection_metric.intersection(box(*source.bounds))
|
||||
if geometry.is_empty or geometry.area <= 0:
|
||||
raise AppError(code="WALOUS_SELECTION_OUTSIDE_DATASET", message="Selection does not overlap the persisted WALOUS raster", status_code=422)
|
||||
clipped, transform = mask(source, [mapping(geometry)], crop=True, filled=False, indexes=[1])
|
||||
band = np.ma.asarray(clipped[0])
|
||||
raw = np.asarray(band.filled(WalousLandCoverService.NODATA), dtype="uint8")
|
||||
selected = geometry_mask([mapping(geometry)], out_shape=raw.shape, transform=transform, invert=True)
|
||||
valid = selected & ~np.ma.getmaskarray(band) & (raw != WalousLandCoverService.NODATA)
|
||||
values = raw[valid]
|
||||
selected_count = int(selected.sum())
|
||||
valid_count = int(values.size)
|
||||
if not valid_count:
|
||||
raise AppError(code="WALOUS_NO_VALID_DATA", message="WALOUS contains no valid cells in this selection", status_code=422)
|
||||
cell_area_m2 = abs(float(source.res[0]) * float(source.res[1]))
|
||||
except AppError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AppError(code="WALOUS_ANALYSIS_FAILED", message="The persisted WALOUS raster could not be analysed", details={"reason": str(exc)}, status_code=500) from exc
|
||||
|
||||
def area_for(classes: set[int]) -> float:
|
||||
return float(np.count_nonzero(np.isin(values, list(classes))) * cell_area_m2 / 10_000.0)
|
||||
|
||||
metric_specs = [
|
||||
("land_cover_observed_area_ha", "Gekarteerde landbedekking", set(WalousLandCoverService.CLASS_LABELS)),
|
||||
("forest_cover_area_ha", "Boom- en bosbedekking", {3, 4, 5, 6}),
|
||||
("surface_water_area_ha", "Oppervlaktewater", {8}),
|
||||
("artificial_cover_area_ha", "Kunstmatige bedekking en constructies", {9, 10, 11}),
|
||||
("annual_herbaceous_cover_area_ha", "Jaarlijks wisselende kruidlaag", {1}),
|
||||
("permanent_herbaceous_cover_area_ha", "Jaarronde kruidlaag", {2}),
|
||||
("bare_soil_area_ha", "Kale bodem", {7}),
|
||||
]
|
||||
metrics = [
|
||||
ThematicRasterMetric(
|
||||
metric_key=key,
|
||||
metric_label=label,
|
||||
metric_value=round(area_for(classes), 4),
|
||||
metric_unit="ha",
|
||||
aggregation_method="nearest_resampled_cells_times_cell_area",
|
||||
is_estimate=True,
|
||||
)
|
||||
for key, label, classes in metric_specs
|
||||
]
|
||||
primary = metrics[0]
|
||||
return ThematicRasterSelectionResponse(
|
||||
dataset_id=dataset.id,
|
||||
product_key=product.key,
|
||||
theme=WalousLandCoverService.THEME,
|
||||
metric_kind=WalousLandCoverService.METRIC_KIND,
|
||||
selection_bbox=payload.bbox,
|
||||
selection_area_id=payload.area_id,
|
||||
selected_cell_count=selected_count,
|
||||
valid_cell_count=valid_count,
|
||||
coverage_ratio=round(valid_count / max(1, selected_count), 6),
|
||||
resolution_m=round(math.sqrt(cell_area_m2), 4),
|
||||
observation_year=product.observation_year,
|
||||
summary=ThematicRasterSelectionSummary(
|
||||
metric_label=primary.metric_label,
|
||||
metric_value=primary.metric_value,
|
||||
metric_unit=primary.metric_unit,
|
||||
aggregation_method=primary.aggregation_method,
|
||||
primary_metric_key=primary.metric_key,
|
||||
metrics=metrics,
|
||||
),
|
||||
unsupported_metrics=["legal_land_use", "ownership", "tree_count", "timber_volume", "water_volume"],
|
||||
limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.",
|
||||
generated_at=datetime.now(UTC).isoformat(),
|
||||
).model_dump(mode="json")
|
||||
|
||||
@staticmethod
|
||||
def render_png(db, project_id: UUID, dataset_id: UUID, *, max_dimension: int = 1800) -> bytes:
|
||||
dataset, _product = WalousLandCoverService._load_dataset(db, project_id, dataset_id)
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from PIL import Image
|
||||
from rasterio.enums import Resampling
|
||||
except ImportError as exc:
|
||||
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio, numpy and Pillow are required for WALOUS rendering", status_code=503) from exc
|
||||
with rasterio.open(dataset.storage_path) as source:
|
||||
scale = min(1.0, max_dimension / max(source.width, source.height))
|
||||
width, height = max(1, round(source.width * scale)), max(1, round(source.height * scale))
|
||||
values = source.read(1, out_shape=(height, width), masked=True, resampling=Resampling.nearest)
|
||||
raw = np.asarray(values.filled(WalousLandCoverService.NODATA), dtype="uint8")
|
||||
rgba = np.zeros((height, width, 4), dtype="uint8")
|
||||
for value, color in WalousLandCoverService.CLASS_COLORS.items():
|
||||
selected = raw == value
|
||||
rgba[:, :, 0][selected] = color[0]
|
||||
rgba[:, :, 1][selected] = color[1]
|
||||
rgba[:, :, 2][selected] = color[2]
|
||||
rgba[:, :, 3][selected] = 205
|
||||
output = io.BytesIO()
|
||||
Image.fromarray(rgba).save(output, format="PNG", optimize=True)
|
||||
return output.getvalue()
|
||||
@@ -95,6 +95,7 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None
|
||||
"activate_promoted_yolo_candidate.py",
|
||||
"manage_grb_refresh.py",
|
||||
"orthophoto_release_preflight.py",
|
||||
"provision_walous_sources.py",
|
||||
}
|
||||
for script_name in required_runtime_scripts:
|
||||
assert f"COPY scripts/{script_name} /app/scripts/{script_name}" in dockerfile
|
||||
@@ -149,6 +150,24 @@ def test_env_example_uses_runtime_env_names_read_by_backend_and_frontend() -> No
|
||||
assert "VITE_API_PROXY_TARGET=http://localhost:8000" in env_example
|
||||
|
||||
|
||||
def test_walloon_runtime_settings_are_editable_in_compose_and_unraid() -> None:
|
||||
files = [
|
||||
(ROOT / "docker-compose.yml").read_text(encoding="utf-8"),
|
||||
(ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8"),
|
||||
(ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(encoding="utf-8"),
|
||||
(ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8"),
|
||||
(ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(encoding="utf-8"),
|
||||
]
|
||||
for content in files:
|
||||
assert "SPW_FLOOD_HAZARD_ENABLED" in content
|
||||
assert "SPW_FLOOD_HAZARD_MAPSERVER_URL" in content
|
||||
assert "WALOUS_ENABLED" in content
|
||||
assert "WALOUS_SOURCE_DIR" in content
|
||||
assert "WALOUS_ANALYSIS_RESOLUTION_M" in content
|
||||
assert "WALOUS_MAX_SIDE_M" in content
|
||||
assert "WALOUS_MAX_PIXELS" in content
|
||||
|
||||
|
||||
def test_frontend_uses_same_origin_api_proxy_by_default() -> None:
|
||||
api_client = (ROOT / "frontend" / "src" / "services" / "api" / "client.ts").read_text(encoding="utf-8")
|
||||
nginx_config = (ROOT / "frontend" / "nginx.conf").read_text(encoding="utf-8")
|
||||
|
||||
@@ -116,6 +116,9 @@ def test_regional_product_registry_is_explicit_and_source_specific() -> None:
|
||||
]
|
||||
assert products["spw_picc_waterways"]["collection"] == "28"
|
||||
assert products["spw_picc_water_surfaces"]["collection"] == "30"
|
||||
assert products["spw_flood_hazard_2021"]["collection"] == "2"
|
||||
assert products["spw_flood_hazard_2021"]["theme"] == "flood_hazard"
|
||||
assert products["spw_flood_hazard_2021"]["coverage_zones"] == ["wallonia"]
|
||||
assert products["urbis_buildings"]["coverage_zones"] == ["brussels"]
|
||||
assert products["urbis_buildings"]["license_note"] == "Buildings are published under CC0."
|
||||
assert "FPS Finance" in products["urbis_cadastral_parcels"]["license_note"]
|
||||
@@ -252,6 +255,82 @@ def test_spw_arcgis_paging_is_bounded_stable_and_clipped() -> None:
|
||||
assert all(item["properties"]["clipped_area_ha"] > 0 for item in features)
|
||||
|
||||
|
||||
def test_spw_flood_hazard_uses_separate_governed_endpoint_and_persists_classification() -> None:
|
||||
product = OfficialVectorAcquisitionService._product("spw_flood_hazard_2021")
|
||||
scope = Polygon(
|
||||
[(4.55, 50.58), (4.56, 50.58), (4.56, 50.59), (4.55, 50.59), (4.55, 50.58)]
|
||||
)
|
||||
scope_metric = Polygon(
|
||||
[_TO_LAMBERT72.transform(x, y) for x, y in scope.exterior.coords]
|
||||
)
|
||||
|
||||
def opener(raw_request, timeout):
|
||||
assert timeout == 180
|
||||
parsed = urlparse(raw_request.full_url)
|
||||
assert parsed.path.endswith("/EAU/ALEA_INOND/MapServer/2/query")
|
||||
query = parse_qs(parsed.query)
|
||||
assert query["outSR"] == ["4326"]
|
||||
return JsonResponse(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": 7,
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[
|
||||
[4.551, 50.581],
|
||||
[4.559, 50.581],
|
||||
[4.559, 50.589],
|
||||
[4.551, 50.589],
|
||||
[4.551, 50.581],
|
||||
]],
|
||||
},
|
||||
"properties": {
|
||||
"OBJECTID": 7,
|
||||
"LOCALID": "ALEA-7",
|
||||
"TYPEALEA": "Debordement",
|
||||
"CLASSEMENT": 130,
|
||||
"MILLESIME": 2021,
|
||||
},
|
||||
}
|
||||
],
|
||||
"exceededTransferLimit": False,
|
||||
}
|
||||
)
|
||||
|
||||
features, transfer = OfficialVectorAcquisitionService._fetch_features(
|
||||
product,
|
||||
scope,
|
||||
scope_metric,
|
||||
"wallonia",
|
||||
Settings(_env_file=None),
|
||||
opener,
|
||||
)
|
||||
|
||||
assert transfer["feature_count"] == 1
|
||||
assert features[0]["id"] == "2:ALEA-7"
|
||||
assert features[0]["properties"]["CLASSEMENT"] == 130
|
||||
assert features[0]["properties"]["source_name"] == "spw_flood_hazard"
|
||||
assert features[0]["properties"]["clipped_area_ha"] > 0
|
||||
|
||||
|
||||
def test_spw_flood_hazard_can_be_disabled_independently() -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
OfficialVectorAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
request("spw_flood_hazard_2021", (4.55, 50.58, 4.56, 50.59)),
|
||||
settings=Settings(_env_file=None, SPW_FLOOD_HAZARD_ENABLED=False),
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "SPW_FLOOD_HAZARD_NOT_CONFIGURED"
|
||||
|
||||
|
||||
def test_regional_products_require_the_persisted_authoritative_coverage_area() -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
|
||||
|
||||
@@ -130,8 +130,9 @@ def test_product_registries_expose_honest_forest_agriculture_nature_and_soil() -
|
||||
"dov_soil_types",
|
||||
"spw_picc_buildings",
|
||||
"spw_picc_roads",
|
||||
"spw_picc_waterways",
|
||||
"spw_picc_water_surfaces",
|
||||
"spw_picc_waterways",
|
||||
"spw_picc_water_surfaces",
|
||||
"spw_flood_hazard_2021",
|
||||
"urbis_buildings",
|
||||
"urbis_cadastral_parcels",
|
||||
"urbis_street_axes",
|
||||
@@ -405,7 +406,7 @@ def test_official_vector_routes_and_frontend_use_canonical_backend_path(monkeypa
|
||||
|
||||
assert products_response.status_code == 200
|
||||
assert set(products_response.json()) == {"data"}
|
||||
assert products_response.json()["data"]["total"] == 12
|
||||
assert products_response.json()["data"]["total"] == 13
|
||||
assert acquire_response.status_code == 200
|
||||
assert set(acquire_response.json()) == {"data"}
|
||||
assert acquire_response.json()["data"]["job_type"] == "vector.official.acquire"
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
from fastapi.testclient import TestClient
|
||||
from pyproj import Transformer
|
||||
import rasterio
|
||||
from rasterio.transform import from_origin
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models import Dataset, Job, Project
|
||||
from app.schemas.thematic_raster import ThematicRasterAcquireRequest, ThematicRasterSelectionRequest
|
||||
from app.schemas.temporal import TemporalComparisonRequest
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.temporal_analysis_service import TemporalAnalysisService
|
||||
from app.services.walous_land_cover_service import WalousLandCoverService
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
|
||||
def order_by(self, *_args):
|
||||
return self
|
||||
|
||||
def first(self):
|
||||
return None
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, project, dataset=None):
|
||||
self.project = project
|
||||
self.dataset = dataset
|
||||
self.added = []
|
||||
|
||||
def get(self, model, row_id):
|
||||
if model is Project and row_id == self.project.id:
|
||||
return self.project
|
||||
if model is Dataset and self.dataset is not None and row_id == self.dataset.id:
|
||||
return self.dataset
|
||||
match = next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
|
||||
if match is not None:
|
||||
return match
|
||||
return None
|
||||
|
||||
def query(self, _model):
|
||||
return FakeQuery()
|
||||
|
||||
def add(self, row):
|
||||
self.added.append(row)
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
def rollback(self):
|
||||
return None
|
||||
|
||||
def refresh(self, row):
|
||||
return row
|
||||
|
||||
|
||||
def make_source(path: Path) -> tuple[list[float], np.ndarray]:
|
||||
to_3812 = Transformer.from_crs("EPSG:4326", "EPSG:3812", always_xy=True)
|
||||
to_4326 = Transformer.from_crs("EPSG:3812", "EPSG:4326", always_xy=True)
|
||||
x, y = to_3812.transform(4.85, 50.45)
|
||||
transform = from_origin(x, y + 100, 1, 1)
|
||||
values = np.ones((100, 100), dtype="uint8")
|
||||
values[:, 20:40] = 4
|
||||
values[:, 40:50] = 8
|
||||
values[:, 50:70] = 9
|
||||
values[:, 70:] = 2
|
||||
with rasterio.open(
|
||||
path,
|
||||
"w",
|
||||
driver="GTiff",
|
||||
width=100,
|
||||
height=100,
|
||||
count=1,
|
||||
dtype="uint8",
|
||||
crs="EPSG:3812",
|
||||
transform=transform,
|
||||
nodata=255,
|
||||
) as target:
|
||||
target.write(values, 1)
|
||||
min_lon, min_lat = to_4326.transform(x, y)
|
||||
max_lon, max_lat = to_4326.transform(x + 100, y + 100)
|
||||
return [min_lon, min_lat, max_lon, max_lat], values
|
||||
|
||||
|
||||
def settings(source_dir: Path) -> Settings:
|
||||
return Settings(
|
||||
_env_file=None,
|
||||
WALOUS_SOURCE_DIR=str(source_dir),
|
||||
WALOUS_ANALYSIS_RESOLUTION_M=10,
|
||||
WALOUS_MAX_SIDE_M=60_000,
|
||||
WALOUS_MAX_PIXELS=1_000_000,
|
||||
)
|
||||
|
||||
|
||||
def test_walous_registry_reports_real_provisioning_state(tmp_path: Path) -> None:
|
||||
before = {item["key"]: item for item in WalousLandCoverService.list_products(settings=settings(tmp_path))}
|
||||
assert before["walous_land_cover_2023"]["status"] == "source_not_provisioned"
|
||||
make_source(tmp_path / "walous_land_cover_2023_3812.tif")
|
||||
after = {item["key"]: item for item in WalousLandCoverService.list_products(settings=settings(tmp_path))}
|
||||
assert after["walous_land_cover_2023"]["configured"] is True
|
||||
assert after["walous_land_cover_2023"]["source_crs"] == "EPSG:3812"
|
||||
assert after["walous_land_cover_2023"]["native_resolution_m"] == 1.0
|
||||
assert after["walous_land_cover_2023"]["analysis_resolution_m"] == 10.0
|
||||
assert after["walous_land_cover_2023"]["coverage_zones"] == ["wallonia"]
|
||||
|
||||
|
||||
def test_walous_acquisition_reads_real_classes_and_persists_provenance(tmp_path: Path, monkeypatch) -> None:
|
||||
bbox, _values = make_source(tmp_path / "walous_land_cover_2023_3812.tif")
|
||||
project = Project(id=uuid4(), name="Belgium")
|
||||
db = FakeSession(project)
|
||||
captured = {}
|
||||
output_id = uuid4()
|
||||
|
||||
def persist(_db, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return SimpleNamespace(id=output_id)
|
||||
|
||||
monkeypatch.setattr(DatasetService, "import_raster_bytes", persist)
|
||||
result = WalousLandCoverService.acquire(
|
||||
db,
|
||||
project.id,
|
||||
ThematicRasterAcquireRequest(
|
||||
bbox={"min_x": bbox[0], "min_y": bbox[1], "max_x": bbox[2], "max_y": bbox[3], "crs": "EPSG:4326"},
|
||||
product_key="walous_land_cover_2023",
|
||||
force_refresh=True,
|
||||
),
|
||||
settings=settings(tmp_path),
|
||||
)
|
||||
|
||||
assert result["output_dataset_id"] == str(output_id)
|
||||
assert result["resolution_m"] == 10
|
||||
assert captured["source_name"] == "spw_walous_land_cover"
|
||||
assert captured["source_metadata"]["classes_present"] == [1, 2, 4, 8, 9]
|
||||
assert captured["provenance_metadata"]["resampling"] == "nearest"
|
||||
assert captured["temporal_series_key"].startswith("spw:walous:land-cover:")
|
||||
assert captured["observed_at"].year == 2023
|
||||
|
||||
|
||||
def test_walous_analysis_returns_semantic_area_metrics(tmp_path: Path, monkeypatch) -> None:
|
||||
bbox, _values = make_source(tmp_path / "walous_land_cover_2023_3812.tif")
|
||||
project = Project(id=uuid4(), name="Belgium")
|
||||
output_id = uuid4()
|
||||
captured = {}
|
||||
|
||||
def persist(_db, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return SimpleNamespace(id=output_id)
|
||||
|
||||
monkeypatch.setattr(DatasetService, "import_raster_bytes", persist)
|
||||
db = FakeSession(project)
|
||||
payload = ThematicRasterAcquireRequest(
|
||||
bbox={"min_x": bbox[0], "min_y": bbox[1], "max_x": bbox[2], "max_y": bbox[3], "crs": "EPSG:4326"},
|
||||
product_key="walous_land_cover_2023",
|
||||
force_refresh=True,
|
||||
)
|
||||
WalousLandCoverService.acquire(db, project.id, payload, settings=settings(tmp_path))
|
||||
persisted_path = tmp_path / "derived.tif"
|
||||
persisted_path.write_bytes(captured["content"])
|
||||
dataset = Dataset(
|
||||
id=output_id,
|
||||
project_id=project.id,
|
||||
name="derived.tif",
|
||||
dataset_type="raster",
|
||||
source="SPW WALOUS",
|
||||
source_name="spw_walous_land_cover",
|
||||
source_metadata=captured["source_metadata"],
|
||||
provenance_metadata=captured["provenance_metadata"],
|
||||
storage_path=str(persisted_path),
|
||||
status="ready",
|
||||
)
|
||||
db.dataset = dataset
|
||||
result = WalousLandCoverService.analyze(
|
||||
db,
|
||||
project.id,
|
||||
output_id,
|
||||
ThematicRasterSelectionRequest(bbox=payload.bbox),
|
||||
)
|
||||
metrics = {item["metric_key"]: item["metric_value"] for item in result["summary"]["metrics"]}
|
||||
|
||||
assert result["metric_kind"] == "categorical_area"
|
||||
assert metrics["land_cover_observed_area_ha"] > 0
|
||||
assert metrics["forest_cover_area_ha"] > 0
|
||||
assert metrics["surface_water_area_ha"] > 0
|
||||
assert metrics["artificial_cover_area_ha"] > 0
|
||||
assert "water_volume" in result["unsupported_metrics"]
|
||||
|
||||
|
||||
def test_walous_render_png_uses_governed_class_colours(tmp_path: Path) -> None:
|
||||
bbox, _values = make_source(tmp_path / "walous_land_cover_2023_3812.tif")
|
||||
project = Project(id=uuid4(), name="Belgium")
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=project.id,
|
||||
name="walous.tif",
|
||||
dataset_type="raster",
|
||||
source="SPW WALOUS",
|
||||
source_name="spw_walous_land_cover",
|
||||
source_metadata={"product_key": "walous_land_cover_2023", "bbox_epsg4326": bbox},
|
||||
storage_path=str(tmp_path / "walous_land_cover_2023_3812.tif"),
|
||||
status="ready",
|
||||
)
|
||||
db = FakeSession(project, dataset)
|
||||
|
||||
rendered = WalousLandCoverService.render_png(db, project.id, dataset.id)
|
||||
|
||||
assert rendered.startswith(b"\x89PNG\r\n\x1a\n")
|
||||
|
||||
|
||||
def test_walous_temporal_comparison_reuses_persisted_raster_metrics(monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
earlier = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=project_id,
|
||||
name="walous-2020.tif",
|
||||
dataset_type="raster",
|
||||
source="SPW WALOUS",
|
||||
source_name="spw_walous_land_cover",
|
||||
temporal_series_key="spw:walous:land-cover:selection",
|
||||
observed_at=datetime(2020, 12, 31, 23, 59, 59, tzinfo=timezone.utc),
|
||||
source_version="WAL_OCS_IA__2020",
|
||||
)
|
||||
later = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=project_id,
|
||||
name="walous-2023.tif",
|
||||
dataset_type="raster",
|
||||
source="SPW WALOUS",
|
||||
source_name="spw_walous_land_cover",
|
||||
temporal_series_key=earlier.temporal_series_key,
|
||||
observed_at=datetime(2023, 12, 31, 23, 59, 59, tzinfo=timezone.utc),
|
||||
source_version="WAL_OCS_IA__2023",
|
||||
)
|
||||
rows = {earlier.id: earlier, later.id: later}
|
||||
|
||||
class TemporalSession:
|
||||
def get(self, model, row_id):
|
||||
return rows.get(row_id) if model is Dataset else None
|
||||
|
||||
def analyze(_db, _project_id, dataset_id, _payload):
|
||||
value = 4.0 if dataset_id == earlier.id else 5.5
|
||||
return {
|
||||
"summary": {
|
||||
"metric_label": "Gekarteerde landbedekking",
|
||||
"metric_value": value,
|
||||
"metric_unit": "ha",
|
||||
"aggregation_method": "nearest_resampled_cells_times_cell_area",
|
||||
"primary_metric_key": "land_cover_observed_area_ha",
|
||||
"metrics": [{
|
||||
"metric_key": "land_cover_observed_area_ha",
|
||||
"metric_label": "Gekarteerde landbedekking",
|
||||
"metric_value": value,
|
||||
"metric_unit": "ha",
|
||||
"aggregation_method": "nearest_resampled_cells_times_cell_area",
|
||||
"is_estimate": True,
|
||||
}],
|
||||
},
|
||||
"limitation_message": "Cell-based estimate.",
|
||||
}
|
||||
|
||||
monkeypatch.setattr(WalousLandCoverService, "analyze", analyze)
|
||||
payload = TemporalComparisonRequest(
|
||||
earlier_dataset_id=earlier.id,
|
||||
later_dataset_id=later.id,
|
||||
bbox={"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"},
|
||||
)
|
||||
|
||||
result = TemporalAnalysisService.compare(TemporalSession(), project_id=project_id, payload=payload)
|
||||
|
||||
assert result.metric.earlier_value == 4.0
|
||||
assert result.metric.later_value == 5.5
|
||||
assert result.metric.absolute_change == 1.5
|
||||
assert result.object_changes.available is False
|
||||
|
||||
|
||||
def test_walous_api_routes_use_canonical_envelopes(monkeypatch) -> None:
|
||||
project = Project(id=uuid4(), name="Belgium")
|
||||
dataset_id = uuid4()
|
||||
db = FakeSession(project)
|
||||
monkeypatch.setattr(
|
||||
WalousLandCoverService,
|
||||
"acquire",
|
||||
lambda *_args, **_kwargs: {"output_dataset_id": str(dataset_id), "provider": WalousLandCoverService.PROVIDER},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
WalousLandCoverService,
|
||||
"analyze",
|
||||
lambda *_args, **_kwargs: {
|
||||
"dataset_id": str(dataset_id),
|
||||
"product_key": "walous_land_cover_2023",
|
||||
"theme": "land_cover_use",
|
||||
"metric_kind": "categorical_area",
|
||||
"selection_bbox": {"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"},
|
||||
"selected_cell_count": 100,
|
||||
"valid_cell_count": 100,
|
||||
"coverage_ratio": 1.0,
|
||||
"resolution_m": 10.0,
|
||||
"observation_year": 2023,
|
||||
"summary": {
|
||||
"metric_label": "Gekarteerde landbedekking",
|
||||
"metric_value": 1.0,
|
||||
"metric_unit": "ha",
|
||||
"aggregation_method": "nearest_resampled_cells_times_cell_area",
|
||||
"primary_metric_key": "land_cover_observed_area_ha",
|
||||
"metrics": [],
|
||||
},
|
||||
"unsupported_metrics": ["water_volume"],
|
||||
"limitation_message": "Cell-based estimate.",
|
||||
"generated_at": "2026-07-22T00:00:00Z",
|
||||
},
|
||||
)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
try:
|
||||
client = TestClient(app)
|
||||
products = client.get(f"/api/v1/projects/{project.id}/datasets/walous/products")
|
||||
acquisition = client.post(
|
||||
f"/api/v1/projects/{project.id}/datasets/walous/acquire",
|
||||
json={
|
||||
"bbox": {"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"},
|
||||
"product_key": "walous_land_cover_2023",
|
||||
},
|
||||
)
|
||||
selection = client.post(
|
||||
f"/api/v1/projects/{project.id}/datasets/{dataset_id}/raster/walous/select",
|
||||
json={"bbox": {"min_x": 4.8, "min_y": 50.4, "max_x": 4.9, "max_y": 50.5, "crs": "EPSG:4326"}},
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert products.status_code == 200 and set(products.json()) == {"data"}
|
||||
assert products.json()["data"]["total"] == 2
|
||||
assert acquisition.status_code == 200 and set(acquisition.json()) == {"data"}
|
||||
assert acquisition.json()["data"]["job_type"] == "raster.walous.acquire"
|
||||
assert selection.status_code == 200 and selection.json()["data"]["theme"] == "land_cover_use"
|
||||
assert any(isinstance(item, Job) for item in db.added)
|
||||
@@ -101,6 +101,7 @@ COPY scripts/provision_flanders_geographic_scope.py /app/scripts/provision_fland
|
||||
COPY scripts/provision_flanders_bathymetry_profiles.py /app/scripts/provision_flanders_bathymetry_profiles.py
|
||||
COPY scripts/probe_mdk_bathymetry.py /app/scripts/probe_mdk_bathymetry.py
|
||||
COPY scripts/import_spw_bathymetry.py /app/scripts/import_spw_bathymetry.py
|
||||
COPY scripts/provision_walous_sources.py /app/scripts/provision_walous_sources.py
|
||||
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
|
||||
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
|
||||
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
|
||||
|
||||
@@ -52,6 +52,8 @@
|
||||
<Config Name="DOV Soil WFS URL" Target="DOV_SOIL_WFS_URL" Default="https://www.dov.vlaanderen.be/geoserver/wfs" Mode="" Description="Official allowlisted DOV WFS endpoint for historical soil type polygons." Type="Variable" Display="advanced" Required="true" Mask="false">https://www.dov.vlaanderen.be/geoserver/wfs</Config>
|
||||
<Config Name="SPW PICC Acquisition" Target="SPW_PICC_ENABLED" Default="true" Mode="" Description="Enable bounded Walloon PICC building, road and hydrography queries. Requests remain clipped, paged and read-only at source." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||
<Config Name="SPW PICC MapServer URL" Target="SPW_PICC_MAPSERVER_URL" Default="https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer" Mode="" Description="Official allowlisted SPW PICC ArcGIS REST MapServer root." Type="Variable" Display="advanced" Required="true" Mask="false">https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer</Config>
|
||||
<Config Name="SPW Flood Hazard Acquisition" Target="SPW_FLOOD_HAZARD_ENABLED" Default="true" Mode="" Description="Enable bounded authoritative Walloon flood-hazard polygon acquisition." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||
<Config Name="SPW Flood Hazard MapServer URL" Target="SPW_FLOOD_HAZARD_MAPSERVER_URL" Default="https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer" Mode="" Description="Official allowlisted SPW legal flood-hazard ArcGIS REST MapServer root." Type="Variable" Display="advanced" Required="true" Mask="false">https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer</Config>
|
||||
<Config Name="UrbIS Acquisition" Target="URBIS_ENABLED" Default="true" Mode="" Description="Enable bounded Brussels UrbIS building and cadastral parcel queries. Requests remain clipped, paged and read-only at source." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||
<Config Name="UrbIS WFS URL" Target="URBIS_WFS_URL" Default="https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows" Mode="" Description="Official allowlisted Paradigm Brussels UrbIS WFS 2.0 endpoint." Type="Variable" Display="advanced" Required="true" Mask="false">https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows</Config>
|
||||
<Config Name="Official Vector Minimum Side (m)" Target="OFFICIAL_VECTOR_MIN_SIDE_M" Default="10" Mode="" Description="Minimum bounded official vector request side length." Type="Variable" Display="advanced" Required="true" Mask="false">10</Config>
|
||||
@@ -103,6 +105,11 @@
|
||||
<Config Name="Thematic Raster Maximum Cells" Target="THEMATIC_RASTER_MAX_PIXELS" Default="30000000" Mode="" Description="Maximum raster cells per allowlisted thematic acquisition or selection analysis." Type="Variable" Display="advanced" Required="true" Mask="false">30000000</Config>
|
||||
<Config Name="Thematic Raster Timeout (seconds)" Target="THEMATIC_RASTER_TIMEOUT_SECONDS" Default="300" Mode="" Description="Maximum wait for one bounded thematic raster request." Type="Variable" Display="advanced" Required="true" Mask="false">300</Config>
|
||||
<Config Name="Thematic Raster Maximum Response (MiB)" Target="THEMATIC_RASTER_MAX_RESPONSE_MB" Default="160" Mode="" Description="Maximum accepted thematic raster response size." Type="Variable" Display="advanced" Required="true" Mask="false">160</Config>
|
||||
<Config Name="WALOUS Land Cover" Target="WALOUS_ENABLED" Default="true" Mode="" Description="Enable bounded analysis from operator-provisioned official WALOUS 2020/2023 rasters." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||
<Config Name="WALOUS Source Directory" Target="WALOUS_SOURCE_DIR" Default="/app/storage/source-cache/walous" Mode="" Description="Persistent directory containing the checksum-validated official WALOUS GeoTIFF sources." Type="Variable" Display="advanced" Required="true" Mask="false">/app/storage/source-cache/walous</Config>
|
||||
<Config Name="WALOUS Analysis Resolution (m)" Target="WALOUS_ANALYSIS_RESOLUTION_M" Default="10" Mode="" Description="Nearest-neighbour analysis resolution used for bounded WALOUS derivatives; the 1 m source remains unchanged." Type="Variable" Display="advanced" Required="true" Mask="false">10</Config>
|
||||
<Config Name="WALOUS Maximum Side (m)" Target="WALOUS_MAX_SIDE_M" Default="60000" Mode="" Description="Maximum side length for one bounded WALOUS selection." Type="Variable" Display="advanced" Required="true" Mask="false">60000</Config>
|
||||
<Config Name="WALOUS Maximum Cells" Target="WALOUS_MAX_PIXELS" Default="36000000" Mode="" Description="Maximum persisted analysis cells per bounded WALOUS acquisition." Type="Variable" Display="advanced" Required="true" Mask="false">36000000</Config>
|
||||
<Config Name="Configured YOLO" Target="YOLO_ENABLED" Default="false" Mode="" Description="Enable only a locally mounted and explicitly configured detection model." Type="Variable" Display="advanced" Required="true" Mask="false">false</Config>
|
||||
<Config Name="YOLO Models Directory" Target="YOLO_MODELS_DIR" Default="/app/models" Mode="" Description="In-container directory containing local model assets." Type="Variable" Display="advanced" Required="true" Mask="false">/app/models</Config>
|
||||
<Config Name="YOLO Model Path" Target="YOLO_MODEL_PATH" Default="" Mode="" Description="Absolute in-container path to a local model asset; no download occurs." Type="Variable" Display="advanced" Required="false" Mask="false"></Config>
|
||||
|
||||
@@ -55,6 +55,8 @@ BWK_WFS_URL=https://geo.api.vlaanderen.be/BWK/wfs
|
||||
DOV_SOIL_WFS_URL=https://www.dov.vlaanderen.be/geoserver/wfs
|
||||
SPW_PICC_ENABLED=true
|
||||
SPW_PICC_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer
|
||||
SPW_FLOOD_HAZARD_ENABLED=true
|
||||
SPW_FLOOD_HAZARD_MAPSERVER_URL=https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer
|
||||
URBIS_ENABLED=true
|
||||
URBIS_WFS_URL=https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows
|
||||
OFFICIAL_VECTOR_MIN_SIDE_M=10
|
||||
@@ -112,6 +114,11 @@ THEMATIC_RASTER_MAX_SIDE_M=60000
|
||||
THEMATIC_RASTER_MAX_PIXELS=30000000
|
||||
THEMATIC_RASTER_TIMEOUT_SECONDS=300
|
||||
THEMATIC_RASTER_MAX_RESPONSE_MB=160
|
||||
WALOUS_ENABLED=true
|
||||
WALOUS_SOURCE_DIR=/app/storage/source-cache/walous
|
||||
WALOUS_ANALYSIS_RESOLUTION_M=10
|
||||
WALOUS_MAX_SIDE_M=60000
|
||||
WALOUS_MAX_PIXELS=36000000
|
||||
|
||||
# Optional configured-YOLO runtime. Keep disabled unless a local model is mounted.
|
||||
GEOINTEL_INSTALL_AI=false
|
||||
|
||||
@@ -53,6 +53,8 @@ BWK_WFS_URL="${BWK_WFS_URL:-https://geo.api.vlaanderen.be/BWK/wfs}"
|
||||
DOV_SOIL_WFS_URL="${DOV_SOIL_WFS_URL:-https://www.dov.vlaanderen.be/geoserver/wfs}"
|
||||
SPW_PICC_ENABLED="${SPW_PICC_ENABLED:-true}"
|
||||
SPW_PICC_MAPSERVER_URL="${SPW_PICC_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer}"
|
||||
SPW_FLOOD_HAZARD_ENABLED="${SPW_FLOOD_HAZARD_ENABLED:-true}"
|
||||
SPW_FLOOD_HAZARD_MAPSERVER_URL="${SPW_FLOOD_HAZARD_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer}"
|
||||
URBIS_ENABLED="${URBIS_ENABLED:-true}"
|
||||
URBIS_WFS_URL="${URBIS_WFS_URL:-https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows}"
|
||||
OFFICIAL_VECTOR_MIN_SIDE_M="${OFFICIAL_VECTOR_MIN_SIDE_M:-10}"
|
||||
@@ -104,6 +106,11 @@ THEMATIC_RASTER_MAX_SIDE_M="${THEMATIC_RASTER_MAX_SIDE_M:-60000}"
|
||||
THEMATIC_RASTER_MAX_PIXELS="${THEMATIC_RASTER_MAX_PIXELS:-30000000}"
|
||||
THEMATIC_RASTER_TIMEOUT_SECONDS="${THEMATIC_RASTER_TIMEOUT_SECONDS:-300}"
|
||||
THEMATIC_RASTER_MAX_RESPONSE_MB="${THEMATIC_RASTER_MAX_RESPONSE_MB:-160}"
|
||||
WALOUS_ENABLED="${WALOUS_ENABLED:-true}"
|
||||
WALOUS_SOURCE_DIR="${WALOUS_SOURCE_DIR:-/app/storage/source-cache/walous}"
|
||||
WALOUS_ANALYSIS_RESOLUTION_M="${WALOUS_ANALYSIS_RESOLUTION_M:-10}"
|
||||
WALOUS_MAX_SIDE_M="${WALOUS_MAX_SIDE_M:-60000}"
|
||||
WALOUS_MAX_PIXELS="${WALOUS_MAX_PIXELS:-36000000}"
|
||||
YOLO_ENABLED="${YOLO_ENABLED:-false}"
|
||||
YOLO_MODELS_DIR="${YOLO_MODELS_DIR:-/app/models}"
|
||||
YOLO_MODEL_PATH="${YOLO_MODEL_PATH:-}"
|
||||
@@ -249,6 +256,8 @@ docker run -d \
|
||||
-e DOV_SOIL_WFS_URL="$DOV_SOIL_WFS_URL" \
|
||||
-e SPW_PICC_ENABLED="$SPW_PICC_ENABLED" \
|
||||
-e SPW_PICC_MAPSERVER_URL="$SPW_PICC_MAPSERVER_URL" \
|
||||
-e SPW_FLOOD_HAZARD_ENABLED="$SPW_FLOOD_HAZARD_ENABLED" \
|
||||
-e SPW_FLOOD_HAZARD_MAPSERVER_URL="$SPW_FLOOD_HAZARD_MAPSERVER_URL" \
|
||||
-e URBIS_ENABLED="$URBIS_ENABLED" \
|
||||
-e URBIS_WFS_URL="$URBIS_WFS_URL" \
|
||||
-e OFFICIAL_VECTOR_MIN_SIDE_M="$OFFICIAL_VECTOR_MIN_SIDE_M" \
|
||||
@@ -300,6 +309,11 @@ docker run -d \
|
||||
-e THEMATIC_RASTER_MAX_PIXELS="$THEMATIC_RASTER_MAX_PIXELS" \
|
||||
-e THEMATIC_RASTER_TIMEOUT_SECONDS="$THEMATIC_RASTER_TIMEOUT_SECONDS" \
|
||||
-e THEMATIC_RASTER_MAX_RESPONSE_MB="$THEMATIC_RASTER_MAX_RESPONSE_MB" \
|
||||
-e WALOUS_ENABLED="$WALOUS_ENABLED" \
|
||||
-e WALOUS_SOURCE_DIR="$WALOUS_SOURCE_DIR" \
|
||||
-e WALOUS_ANALYSIS_RESOLUTION_M="$WALOUS_ANALYSIS_RESOLUTION_M" \
|
||||
-e WALOUS_MAX_SIDE_M="$WALOUS_MAX_SIDE_M" \
|
||||
-e WALOUS_MAX_PIXELS="$WALOUS_MAX_PIXELS" \
|
||||
-e YOLO_ENABLED="$YOLO_ENABLED" \
|
||||
-e YOLO_MODELS_DIR="$YOLO_MODELS_DIR" \
|
||||
-e YOLO_MODEL_PATH="$YOLO_MODEL_PATH" \
|
||||
|
||||
@@ -46,6 +46,8 @@ services:
|
||||
DOV_SOIL_WFS_URL: ${DOV_SOIL_WFS_URL:-https://www.dov.vlaanderen.be/geoserver/wfs}
|
||||
SPW_PICC_ENABLED: ${SPW_PICC_ENABLED:-true}
|
||||
SPW_PICC_MAPSERVER_URL: ${SPW_PICC_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer}
|
||||
SPW_FLOOD_HAZARD_ENABLED: ${SPW_FLOOD_HAZARD_ENABLED:-true}
|
||||
SPW_FLOOD_HAZARD_MAPSERVER_URL: ${SPW_FLOOD_HAZARD_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer}
|
||||
URBIS_ENABLED: ${URBIS_ENABLED:-true}
|
||||
URBIS_WFS_URL: ${URBIS_WFS_URL:-https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows}
|
||||
OFFICIAL_VECTOR_MIN_SIDE_M: ${OFFICIAL_VECTOR_MIN_SIDE_M:-10}
|
||||
@@ -80,6 +82,11 @@ services:
|
||||
THEMATIC_RASTER_MAX_PIXELS: ${THEMATIC_RASTER_MAX_PIXELS:-30000000}
|
||||
THEMATIC_RASTER_TIMEOUT_SECONDS: ${THEMATIC_RASTER_TIMEOUT_SECONDS:-300}
|
||||
THEMATIC_RASTER_MAX_RESPONSE_MB: ${THEMATIC_RASTER_MAX_RESPONSE_MB:-160}
|
||||
WALOUS_ENABLED: ${WALOUS_ENABLED:-true}
|
||||
WALOUS_SOURCE_DIR: ${WALOUS_SOURCE_DIR:-/app/storage/source-cache/walous}
|
||||
WALOUS_ANALYSIS_RESOLUTION_M: ${WALOUS_ANALYSIS_RESOLUTION_M:-10}
|
||||
WALOUS_MAX_SIDE_M: ${WALOUS_MAX_SIDE_M:-60000}
|
||||
WALOUS_MAX_PIXELS: ${WALOUS_MAX_PIXELS:-36000000}
|
||||
YOLO_ENABLED: ${YOLO_ENABLED:-false}
|
||||
YOLO_MODELS_DIR: ${YOLO_MODELS_DIR:-/app/models}
|
||||
YOLO_MODEL_PATH: ${YOLO_MODEL_PATH:-}
|
||||
|
||||
@@ -48,6 +48,8 @@ services:
|
||||
DOV_SOIL_WFS_URL: ${DOV_SOIL_WFS_URL:-https://www.dov.vlaanderen.be/geoserver/wfs}
|
||||
SPW_PICC_ENABLED: ${SPW_PICC_ENABLED:-true}
|
||||
SPW_PICC_MAPSERVER_URL: ${SPW_PICC_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/TOPOGRAPHIE/PICC_VDIFF/MapServer}
|
||||
SPW_FLOOD_HAZARD_ENABLED: ${SPW_FLOOD_HAZARD_ENABLED:-true}
|
||||
SPW_FLOOD_HAZARD_MAPSERVER_URL: ${SPW_FLOOD_HAZARD_MAPSERVER_URL:-https://geoservices.wallonie.be/arcgis/rest/services/EAU/ALEA_INOND/MapServer}
|
||||
URBIS_ENABLED: ${URBIS_ENABLED:-true}
|
||||
URBIS_WFS_URL: ${URBIS_WFS_URL:-https://geoservices-vector.irisnet.be/geoserver/urbisvector/ows}
|
||||
OFFICIAL_VECTOR_MIN_SIDE_M: ${OFFICIAL_VECTOR_MIN_SIDE_M:-10}
|
||||
@@ -105,6 +107,11 @@ services:
|
||||
THEMATIC_RASTER_MAX_PIXELS: ${THEMATIC_RASTER_MAX_PIXELS:-30000000}
|
||||
THEMATIC_RASTER_TIMEOUT_SECONDS: ${THEMATIC_RASTER_TIMEOUT_SECONDS:-300}
|
||||
THEMATIC_RASTER_MAX_RESPONSE_MB: ${THEMATIC_RASTER_MAX_RESPONSE_MB:-160}
|
||||
WALOUS_ENABLED: ${WALOUS_ENABLED:-true}
|
||||
WALOUS_SOURCE_DIR: ${WALOUS_SOURCE_DIR:-/app/storage/source-cache/walous}
|
||||
WALOUS_ANALYSIS_RESOLUTION_M: ${WALOUS_ANALYSIS_RESOLUTION_M:-10}
|
||||
WALOUS_MAX_SIDE_M: ${WALOUS_MAX_SIDE_M:-60000}
|
||||
WALOUS_MAX_PIXELS: ${WALOUS_MAX_PIXELS:-36000000}
|
||||
YOLO_ENABLED: ${YOLO_ENABLED:-false}
|
||||
YOLO_MODELS_DIR: ${YOLO_MODELS_DIR:-/app/models}
|
||||
YOLO_MODEL_PATH: ${YOLO_MODEL_PATH:-}
|
||||
|
||||
@@ -482,6 +482,42 @@ Returns a constrained transparent PNG generated from the persisted governed
|
||||
raster. It accepts neither an arbitrary file path nor a provider URL and feeds
|
||||
the existing MapLibre image-overlay path.
|
||||
|
||||
### GET `/api/v1/projects/{project_id}/datasets/walous/products`
|
||||
|
||||
Returns the fixed official WALOUS 2020/2023 registry. Each product reports its
|
||||
observation year, EPSG:3812 source contract, 1 m source semantics, configured
|
||||
state, attribution, licence and documented edition accuracy. `configured`
|
||||
becomes true only when the checksum-validated source GeoTIFF exists below
|
||||
`WALOUS_SOURCE_DIR`; the endpoint never downloads an archive.
|
||||
|
||||
### POST `/api/v1/projects/{project_id}/datasets/walous/acquire`
|
||||
|
||||
Reads a bounded window from one provisioned official 1 m WALOUS source,
|
||||
applies nearest-neighbour resampling to the configured analysis resolution,
|
||||
masks `bbox intersect Area`, validates class values 1-11 and persists a normal
|
||||
raster Dataset through `DatasetService`. URLs, paths, classes and resolutions
|
||||
are not caller-controlled. Equal spatial requests for 2020 and 2023 share one
|
||||
temporal series key.
|
||||
|
||||
### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/walous/select`
|
||||
|
||||
Returns mapped hectares for total observed land cover, trees/forest, surface
|
||||
water, artificial cover, annual and permanent herbaceous cover and bare soil.
|
||||
Values are cell-area estimates from the persisted derived raster. The response
|
||||
explicitly rejects legal land use, ownership, tree count, timber volume and
|
||||
water volume as unsupported interpretations.
|
||||
|
||||
### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/walous/image`
|
||||
|
||||
Returns a transparent PNG using the governed 11-class colour table and only
|
||||
the persisted Dataset geometry. It never proxies the source archive.
|
||||
|
||||
The Map workbench acquires every configured comparable WALOUS observation for
|
||||
the same Walloon selection when current land cover is first requested. The
|
||||
latest edition drives the current result; the ordinary temporal comparison API
|
||||
then compares 2020 and 2023 semantic area metrics. Raster evolution does not
|
||||
claim individual object additions or removals.
|
||||
|
||||
### GET `/api/v1/projects/{project_id}/datasets`
|
||||
|
||||
List datasets.
|
||||
|
||||
@@ -3652,6 +3652,28 @@ Validation:
|
||||
- `cd backend && python -m alembic heads && python -m alembic upgrade head --sql`
|
||||
# Codex Execution Log
|
||||
|
||||
## Post-V1 national coverage completion: Wallonia (2026-07-22)
|
||||
|
||||
- Validated the official WALOUS 2020 and 2023 archives, their EPSG:3812 1 m
|
||||
raster contract, 11 classes, CC BY 4.0 attribution and published edition
|
||||
accuracy. Added a fail-closed operator provisioner with archive-size,
|
||||
extraction, CRS, resolution, class and checksum gates.
|
||||
- Added bounded WALOUS persistence through `DatasetService`, semantic hectare
|
||||
metrics, a governed PNG overlay and automatic 2020/2023 temporal-series
|
||||
materialization. The existing temporal API now compares persisted WALOUS
|
||||
raster metrics while keeping object-level change unavailable.
|
||||
- Added the queryable legal SPW flood-hazard polygon product and class-aware
|
||||
metrics through the existing official-vector engine.
|
||||
- Wired both integrations through Compose, the all-in-one Unraid runner,
|
||||
editable DockerMan template and readiness gate. Added canonical API,
|
||||
persistence, rendering, temporal and runtime-parity regression tests.
|
||||
- Revalidated the official Walloon DTM distribution. The whole-region files
|
||||
are too large for implicit startup or per-selection mirroring (about 41 GB
|
||||
at 1 m and 213 GB at 0.5 m), so elevation remains a documented capacity-plan
|
||||
prerequisite instead of a fake operational source.
|
||||
- Reprobed the documented MDK WCS endpoints. Strict TLS still fails hostname
|
||||
validation; acquisition remains disabled and no insecure fallback was added.
|
||||
|
||||
This file must be updated by Codex after each implementation pass.
|
||||
|
||||
## Format
|
||||
|
||||
@@ -26,7 +26,7 @@ coverage or historical dates.
|
||||
| --- | --- | --- |
|
||||
| Belgium | NGI administrative boundaries; Statbel population/statistical sectors | Statbel population 2021-2025 |
|
||||
| Flanders | GRB buildings, roads, water and parcels; DHMV terrain/surface; VMM flood scenarios; BWK/Natura 2000; DOV soil; policy rasters for space, open space, accessibility and services; agriculture and orthophoto where governed | Population 2021-2025; land-use/land-cover series where retained; agriculture editions; historical maps/orthophotos where the selected product has a real observation date |
|
||||
| Wallonia | Bounded PICC buildings, roads and hydrography; governed SPW bed-elevation/bathymetry products | No general cross-theme regional history yet |
|
||||
| Wallonia | Bounded PICC buildings, roads and hydrography; legal SPW flood-hazard polygons; bounded WALOUS 2020/2023 land-cover analysis from provisioned official rasters; governed SPW bed-elevation/bathymetry products | Comparable WALOUS land-cover area metrics for 2020-2023; no general cross-theme regional history yet |
|
||||
| Brussels | Bounded UrbIS buildings, street axes, cadastral parcels and Land Cover blocks; official FO/GB blocks provide forest/park area and WB blocks provide permanent water area | No general cross-theme regional history yet; the live WFS has no per-feature observation date |
|
||||
| Belgian North Sea | RBINS reporting units; Marine Spatial Plan 2026-2034; governed MDK bathymetry only when runtime acquisition is explicitly configured | No multi-epoch bathymetry or marine-plan trend yet |
|
||||
|
||||
@@ -37,23 +37,21 @@ applicable bounded official source or reports the theme as unsupported.
|
||||
|
||||
## Priority coverage gaps
|
||||
|
||||
1. Govern the public Walloon WALOUS land-cover editions (including the
|
||||
published 2018/2020 change product) as a real regional time series. A class
|
||||
crosswalk is required because the older COSW 2005/2007 methodology differs.
|
||||
Official catalogue:
|
||||
`https://geoportail.wallonie.be/catalogue-donnees?search-text=occupation+du+sol`.
|
||||
2. Govern the current public Walloon flood-hazard vector/raster products and
|
||||
retain their model scenario semantics separately from observed floods.
|
||||
Official record:
|
||||
`https://geoportail.wallonie.be/catalogue/14084108-2c7b-4091-b62d-ff0fc235213a.html`.
|
||||
3. Add a common Belgium-wide topographic baseline with normalized theme
|
||||
1. Add the official WALOUS 2018 edition only after SPW restores a stable direct
|
||||
artifact or another checksum-verifiable acquisition contract. WALOUS 2020
|
||||
and 2023 are operational and comparable; COSW 2005/2007 remains a different
|
||||
methodology and is not silently merged.
|
||||
2. Add a common Belgium-wide topographic baseline with normalized theme
|
||||
semantics across NGI, Flanders, Wallonia and Brussels.
|
||||
4. Govern comparable Walloon and Brussels historical editions before exposing
|
||||
3. Govern comparable Walloon and Brussels historical editions before exposing
|
||||
evolution for buildings, roads, land cover, soil, elevation or flood risk.
|
||||
5. Add nationally comparable land-cover history with explicit class crosswalks
|
||||
4. Add nationally comparable land-cover history with explicit class crosswalks
|
||||
and uncertainty; never compare incompatible legends silently.
|
||||
6. Add multi-epoch marine bathymetry and survey-footprint metadata before
|
||||
5. Add multi-epoch marine bathymetry and survey-footprint metadata before
|
||||
presenting seabed evolution.
|
||||
6. Add Walloon DTM only through an operator capacity plan: the official 1 m
|
||||
national artifact is about 41 GB and the 0.5 m artifact about 213 GB, so it
|
||||
is not safe as an implicit per-selection dependency.
|
||||
7. Expand persisted raster partition manifests beyond the regression regions
|
||||
only where repeated use justifies caching; bounded acquisition remains the
|
||||
default for one-off selections.
|
||||
|
||||
+5
-3
@@ -63,9 +63,11 @@ geen open productroadmap meer.
|
||||
- [x] Maak UrbIS Land Cover begrensd operationeel voor Brussel: alle Blocks als
|
||||
landbedekking, FO/GB als bos en park en WB als permanent water, met echte
|
||||
PostGIS-oppervlaktemetrics en broncodes.
|
||||
- [ ] Implementeer begrensde WALOUS 2018/2020/2023 rasteracquisitie in
|
||||
EPSG:3812 met officiële klassen, vergelijkbaarheidscontract en pixelbudget.
|
||||
- [ ] Implementeer de actuele Waalse overstromingsgevaarkaart als afzonderlijk
|
||||
- [x] Implementeer begrensde WALOUS 2020/2023 rasteracquisitie in EPSG:3812
|
||||
met officiële klassen, vergelijkbaarheidscontract, pixelbudget, automatische
|
||||
tijdreeksmaterialisatie en evolutiemetrics. WALOUS 2018 blijft bewust open
|
||||
tot SPW opnieuw een stabiel checksum-verifieerbaar direct artifact aanbiedt.
|
||||
- [x] Implementeer de actuele Waalse overstromingsgevaarkaart als afzonderlijk
|
||||
scenario-/juridisch contract; gebruik WMS alleen als context tenzij
|
||||
analytische pixels of vectorgeometrie officieel beschikbaar zijn.
|
||||
- [ ] Bouw een Belgische AI-validatiematrix met gelabelde golden AOIs in elk
|
||||
|
||||
@@ -763,6 +763,19 @@ agriculture use thematic raster analysis; nature value and soil use the
|
||||
persisted-vector GeoJSON pattern. The browser calls only the GeoIntel API and
|
||||
never contacts WCS, WFS or OGC providers directly.
|
||||
|
||||
## Walloon current and historical land cover
|
||||
|
||||
For a bounded Walloon selection, `Landbedekking` resolves the newest configured
|
||||
WALOUS product and automatically persists the other configured comparable
|
||||
edition for the same rectangle. Current analysis shows semantic hectare
|
||||
metrics and an 11-class MapLibre image overlay. After dataset refresh,
|
||||
`Evolutie` offers 2020 versus 2023 through the same period selector and trend
|
||||
chart used by other temporal sources. Individual object-change counts remain
|
||||
unavailable for categorical rasters and are not simulated.
|
||||
|
||||
The source card remains `Op aanvraag` when the official source files are not
|
||||
provisioned and never contacts SPW directly from the browser.
|
||||
|
||||
## Belgium and Belgian North Sea coverage
|
||||
|
||||
When `Belgium and North Sea Workbench` exists with ready reference data, it is
|
||||
|
||||
@@ -9,7 +9,7 @@ import { getDatasetDisplayName, getDatasetSourceDisplayName } from '../../lib/da
|
||||
import { TemporalTrendChart } from './TemporalTrendChart'
|
||||
import { terrainImageUrl } from '../../lib/terrainImage'
|
||||
import { floodHazardImageUrl } from '../../lib/floodHazardImage'
|
||||
import { thematicRasterImageUrl } from '../../lib/thematicRaster'
|
||||
import { thematicRasterImageUrl, walousRasterImageUrl } from '../../lib/thematicRaster'
|
||||
import { bathymetryRasterImageUrl } from '../../lib/bathymetryRaster'
|
||||
import { FLANDERS_WORKSPACE_PROJECT_NAME } from '../../config/primaryFocus'
|
||||
import {
|
||||
@@ -59,6 +59,7 @@ const EMPTY_TEMPORAL_SERIES: DatasetCreateResponse[] = []
|
||||
type DataThemeId =
|
||||
| 'administrative'
|
||||
| 'buildings'
|
||||
| 'land_cover'
|
||||
| 'space_occupation'
|
||||
| 'open_space'
|
||||
| 'population'
|
||||
@@ -110,7 +111,7 @@ function productSupportsSelection(product: OnDemandMapProduct, bbox: VectorSelec
|
||||
if (product.kind === 'dhmv' || product.kind === 'flood_hazard') {
|
||||
return dimensions.areaSquareMetres <= 280_000_000
|
||||
}
|
||||
if (product.kind === 'thematic_raster') {
|
||||
if (product.kind === 'thematic_raster' || product.kind === 'walous') {
|
||||
return dimensions.widthMetres <= 50_000
|
||||
&& dimensions.heightMetres <= 50_000
|
||||
&& dimensions.areaSquareMetres <= 2_800_000_000
|
||||
@@ -133,6 +134,13 @@ const DATA_THEMES: DataTheme[] = [
|
||||
description: 'Gebouwen en gebouwcontouren uit GRB of een andere persistente bron.',
|
||||
tokens: ['buildings', 'building', 'gebouwen', 'gebouw', 'bebouwing', 'gbg'],
|
||||
},
|
||||
{
|
||||
id: 'land_cover',
|
||||
label: 'Landbedekking',
|
||||
shortLabel: 'Landbedekking',
|
||||
description: 'Fysieke en biologische bodembedekking uit een officieel regionaal classificatieraster.',
|
||||
tokens: ['land_cover', 'land_cover_use', 'landbedekking', 'walous', 'occupation du sol'],
|
||||
},
|
||||
{
|
||||
id: 'space_occupation',
|
||||
label: 'Ruimtebeslag',
|
||||
@@ -257,6 +265,7 @@ const DATA_THEMES: DataTheme[] = [
|
||||
const COVERAGE_THEME_BY_MAP_THEME: Record<DataThemeId, string> = {
|
||||
administrative: 'admin',
|
||||
buildings: 'buildings',
|
||||
land_cover: 'land_cover_use',
|
||||
space_occupation: 'land_cover_use',
|
||||
open_space: 'land_cover_use',
|
||||
population: 'population',
|
||||
@@ -303,6 +312,7 @@ function coverageZoneLabel(zone: string): string {
|
||||
const DATA_THEME_MAP_STYLES: Record<DataThemeId, { fill: string; line: string }> = {
|
||||
administrative: { fill: '#5f6f7f', line: '#344554' },
|
||||
buildings: { fill: '#d45f3d', line: '#9f3e24' },
|
||||
land_cover: { fill: '#4f7b4f', line: '#315c39' },
|
||||
space_occupation: { fill: '#be3e33', line: '#8f2c24' },
|
||||
open_space: { fill: '#267a46', line: '#175c32' },
|
||||
population: { fill: '#7559a6', line: '#5b3f88' },
|
||||
@@ -358,6 +368,12 @@ function datasetAvailabilityLabel(
|
||||
const resolutionLabel = Number.isFinite(resolution) ? `${resolution.toLocaleString('nl-BE')} m` : 'Raster'
|
||||
return `${resolutionLabel} officiële bron${Number.isFinite(year) ? ` · ${year}` : ''}`
|
||||
}
|
||||
if (dataset.dataset_type === 'raster' && dataset.source_name === 'spw_walous_land_cover') {
|
||||
const resolution = Number(dataset.source_metadata?.['analysis_resolution_m'])
|
||||
const year = Number(dataset.source_metadata?.['observation_year'])
|
||||
const resolutionLabel = Number.isFinite(resolution) ? `${resolution.toLocaleString('nl-BE')} m` : 'Raster'
|
||||
return `${resolutionLabel} WALOUS-landbedekking${Number.isFinite(year) ? ` · ${year}` : ''}`
|
||||
}
|
||||
if (dataset.source_name === 'ngi_adminvector') {
|
||||
return `${(dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0).toLocaleString('nl-BE')} officiële bestuursgebieden`
|
||||
}
|
||||
@@ -406,6 +422,9 @@ function datasetMatchesTheme(dataset: DatasetCreateResponse, theme: DataTheme):
|
||||
if (dataset.source_name === 'department_omgeving_thematic_raster') {
|
||||
return dataset.source_metadata?.['theme'] === theme.id
|
||||
}
|
||||
if (dataset.source_name === 'spw_walous_land_cover') {
|
||||
return theme.id === 'land_cover'
|
||||
}
|
||||
if (dataset.source_name === 'dov_soil_map') {
|
||||
return theme.id === 'soil'
|
||||
}
|
||||
@@ -522,6 +541,7 @@ function pickThemeDataset(
|
||||
(dataset.source_name === 'inbo_bwk_natura2000' ? 95_000 : 0) +
|
||||
(dataset.source_name === 'agentschap_landbouw_zeevisserij_agricultural_parcels' ? 98_000 : 0) +
|
||||
(dataset.source_name === 'department_omgeving_thematic_raster' ? 5_000_000 : 0) +
|
||||
(dataset.source_name === 'spw_walous_land_cover' ? 5_000_000 : 0) +
|
||||
(dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register' ? 5_000_000 : 0) +
|
||||
(dataset.source_name === 'digitaal_vlaanderen_dhmv' ? 5_000_000 : 0) +
|
||||
(dataset.source_name === 'vmm_flood_hazard' ? 5_000_000 : 0) +
|
||||
@@ -636,6 +656,10 @@ function formatDatasetObservation(dataset: DatasetCreateResponse): string {
|
||||
return `referentiejaar ${observationYear}`
|
||||
}
|
||||
}
|
||||
if (dataset.source_name === 'spw_walous_land_cover') {
|
||||
const observationYear = Number(dataset.source_metadata?.['observation_year'])
|
||||
return Number.isFinite(observationYear) ? `WALOUS referentiejaar ${observationYear}` : 'WALOUS landbedekking'
|
||||
}
|
||||
const period = dataset.source_metadata?.['acquisition_period']
|
||||
if (typeof period === 'string' && period.trim()) {
|
||||
return `opnameperiode ${period}`
|
||||
@@ -1085,6 +1109,29 @@ export function MapWorkspace({
|
||||
})
|
||||
}
|
||||
}
|
||||
const latestWalous = officialMapProducts.walous
|
||||
.filter((product) => product.configured && productCoversZones(product.coverage_zones, effectiveZones))
|
||||
.sort((left, right) => right.observation_year - left.observation_year)[0]
|
||||
if (latestWalous) {
|
||||
result.push({
|
||||
kind: 'walous',
|
||||
productKey: latestWalous.key,
|
||||
historyProductKeys: officialMapProducts.walous
|
||||
.filter(
|
||||
(product) =>
|
||||
product.configured
|
||||
&& product.key !== latestWalous.key
|
||||
&& productCoversZones(product.coverage_zones, effectiveZones),
|
||||
)
|
||||
.map((product) => product.key),
|
||||
displayName: latestWalous.display_name,
|
||||
theme: 'land_cover',
|
||||
availabilityLabel: `${latestWalous.analysis_resolution_m ?? latestWalous.native_resolution_m} m analyse · ${latestWalous.observation_year} · automatisch bij selectie`,
|
||||
attribution: latestWalous.attribution,
|
||||
limitationMessage: latestWalous.limitation_message,
|
||||
coverageZones: latestWalous.coverage_zones,
|
||||
})
|
||||
}
|
||||
for (const product of officialMapProducts.officialVector.filter((item) =>
|
||||
productCoversZones(item.coverage_zones, effectiveZones),
|
||||
)) {
|
||||
@@ -1319,6 +1366,20 @@ export function MapWorkspace({
|
||||
: [],
|
||||
[activeThemeDataset, selectedProjectId, thematicRasterBounds],
|
||||
)
|
||||
const walousRasterBounds = activeThemeDataset?.source_name === 'spw_walous_land_cover'
|
||||
? activeThemeDataset.source_metadata?.['bbox_epsg4326']
|
||||
: null
|
||||
const walousRasterImageOverlays = useMemo(
|
||||
() => activeThemeDataset?.source_name === 'spw_walous_land_cover' && selectedProjectId && Array.isArray(walousRasterBounds) && walousRasterBounds.length === 4
|
||||
? [{
|
||||
url: walousRasterImageUrl(selectedProjectId, activeThemeDataset.id),
|
||||
bbox: walousRasterBounds.map(Number) as [number, number, number, number],
|
||||
label: getDatasetDisplayName(activeThemeDataset),
|
||||
opacity: 0.82,
|
||||
}]
|
||||
: [],
|
||||
[activeThemeDataset, selectedProjectId, walousRasterBounds],
|
||||
)
|
||||
const bathymetryRasterBounds = activeThemeDataset?.source_name === 'spw_bathymetry'
|
||||
? activeThemeDataset.source_metadata?.['bbox_epsg4326']
|
||||
: null
|
||||
@@ -1338,6 +1399,8 @@ export function MapWorkspace({
|
||||
const activeImageOverlays = useMemo(
|
||||
() => bathymetryRasterImageOverlays.length > 0
|
||||
? bathymetryRasterImageOverlays
|
||||
: walousRasterImageOverlays.length > 0
|
||||
? walousRasterImageOverlays
|
||||
: thematicRasterImageOverlays.length > 0
|
||||
? thematicRasterImageOverlays
|
||||
: floodHazardImageOverlays.length > 0
|
||||
@@ -1345,7 +1408,7 @@ export function MapWorkspace({
|
||||
: terrainImageOverlays.length > 0
|
||||
? terrainImageOverlays
|
||||
: orthophotoImageOverlay ? [orthophotoImageOverlay] : [],
|
||||
[bathymetryRasterImageOverlays, floodHazardImageOverlays, orthophotoImageOverlay, terrainImageOverlays, thematicRasterImageOverlays],
|
||||
[bathymetryRasterImageOverlays, floodHazardImageOverlays, orthophotoImageOverlay, terrainImageOverlays, thematicRasterImageOverlays, walousRasterImageOverlays],
|
||||
)
|
||||
const municipalityAreaCount = areas.filter((area) => /^Gemeente\s/i.test(area.name)).length
|
||||
const themeTemporalSeriesMap = useMemo(
|
||||
@@ -1908,6 +1971,7 @@ export function MapWorkspace({
|
||||
kind: onDemandProduct.kind,
|
||||
productKey: onDemandProduct.productKey,
|
||||
displayName: onDemandProduct.displayName,
|
||||
historyProductKeys: onDemandProduct.historyProductKeys,
|
||||
},
|
||||
acquisitionBboxes: onDemandProduct.acquisitionBboxes,
|
||||
featureLimit: resultFeatureLimit,
|
||||
@@ -2471,7 +2535,7 @@ export function MapWorkspace({
|
||||
/>
|
||||
<div className="geo-map-legend" aria-label="Kaartlegende">
|
||||
<span><i className="geo-legend-area" /> Werkgebied</span>
|
||||
{thematicRasterImageOverlays.length > 0 ? (
|
||||
{thematicRasterImageOverlays.length > 0 || walousRasterImageOverlays.length > 0 ? (
|
||||
<span className="geo-legend-thematic">
|
||||
<i className={`geo-legend-ramp geo-legend-ramp-${activeTheme.id}`} />
|
||||
<small>{thematicLegendMin} → {thematicLegendMax}</small>
|
||||
|
||||
@@ -9,6 +9,7 @@ import { bathymetryRasterSelectionToMapSelection } from '../lib/bathymetryRaster
|
||||
|
||||
export type MapThemeAcquisitionKind =
|
||||
| 'thematic_raster'
|
||||
| 'walous'
|
||||
| 'dhmv'
|
||||
| 'flood_hazard'
|
||||
| 'grb'
|
||||
@@ -19,6 +20,7 @@ export interface MapThemeAcquisition {
|
||||
kind: MapThemeAcquisitionKind
|
||||
productKey: string
|
||||
displayName: string
|
||||
historyProductKeys?: string[]
|
||||
}
|
||||
|
||||
export interface MapThemeQuery<TThemeId extends string> {
|
||||
@@ -113,7 +115,7 @@ export function useMapThemeSelectionInsights<TThemeId extends string>(
|
||||
const resultLimit = featureLimit ?? 1000
|
||||
if (acquisition) {
|
||||
const requestedBboxes = acquisitionBboxes?.length ? acquisitionBboxes : [bbox]
|
||||
const acquisitionResults = await settleWithConcurrency(requestedBboxes, 1, async (acquisitionBbox) => {
|
||||
const acquireProduct = async (acquisitionBbox: VectorSelectionBBox, productKey: string) => {
|
||||
const commonPayload = {
|
||||
bbox: acquisitionBbox,
|
||||
area_id: areaId,
|
||||
@@ -124,6 +126,11 @@ export function useMapThemeSelectionInsights<TThemeId extends string>(
|
||||
...commonPayload,
|
||||
product_key: acquisition.productKey,
|
||||
})
|
||||
: acquisition.kind === 'walous'
|
||||
? await datasetsApi.acquireWalous(selectedProjectId, {
|
||||
...commonPayload,
|
||||
product_key: productKey,
|
||||
})
|
||||
: acquisition.kind === 'dhmv'
|
||||
? await datasetsApi.acquireDhmv(selectedProjectId, {
|
||||
...commonPayload,
|
||||
@@ -152,13 +159,38 @@ export function useMapThemeSelectionInsights<TThemeId extends string>(
|
||||
)
|
||||
}
|
||||
return datasetsApi.get(selectedProjectId, acquisitionJob.output_dataset_id)
|
||||
})
|
||||
}
|
||||
const acquisitionResults = await settleWithConcurrency(
|
||||
requestedBboxes,
|
||||
1,
|
||||
(acquisitionBbox) => acquireProduct(acquisitionBbox, acquisition.productKey),
|
||||
)
|
||||
const failedAcquisition = acquisitionResults.find((item) => item.status === 'rejected')
|
||||
if (failedAcquisition?.status === 'rejected') {
|
||||
throw failedAcquisition.reason
|
||||
}
|
||||
acquiredDatasets = acquisitionResults.flatMap((item) => item.status === 'fulfilled' ? [item.value] : [])
|
||||
dataset = acquiredDatasets[0]
|
||||
const historyProductKeys = acquisition.kind === 'walous'
|
||||
? [...new Set(acquisition.historyProductKeys ?? [])].filter((key) => key !== acquisition.productKey)
|
||||
: []
|
||||
if (historyProductKeys.length > 0) {
|
||||
const historyRequests = historyProductKeys.flatMap((productKey) =>
|
||||
requestedBboxes.map((acquisitionBbox) => ({ acquisitionBbox, productKey })),
|
||||
)
|
||||
const historyResults = await settleWithConcurrency(
|
||||
historyRequests,
|
||||
1,
|
||||
({ acquisitionBbox, productKey }) => acquireProduct(acquisitionBbox, productKey),
|
||||
)
|
||||
const failedHistory = historyResults.find((item) => item.status === 'rejected')
|
||||
if (failedHistory?.status === 'rejected') {
|
||||
throw failedHistory.reason
|
||||
}
|
||||
acquiredDatasets.push(
|
||||
...historyResults.flatMap((item) => item.status === 'fulfilled' ? [item.value] : []),
|
||||
)
|
||||
}
|
||||
}
|
||||
if (!dataset) {
|
||||
throw new Error(`Geen persistente databron beschikbaar voor thema ${themeId}.`)
|
||||
@@ -197,8 +229,13 @@ export function useMapThemeSelectionInsights<TThemeId extends string>(
|
||||
: dataset.dataset_type === 'raster' && dataset.source_name === 'department_omgeving_thematic_raster'
|
||||
? thematicRasterSelectionToMapSelection(await datasetsApi.selectThematicRaster(selectedProjectId, dataset.id, {
|
||||
bbox,
|
||||
area_id: areaId,
|
||||
}))
|
||||
area_id: areaId,
|
||||
}))
|
||||
: dataset.dataset_type === 'raster' && dataset.source_name === 'spw_walous_land_cover'
|
||||
? thematicRasterSelectionToMapSelection(await datasetsApi.selectWalous(selectedProjectId, dataset.id, {
|
||||
bbox,
|
||||
area_id: areaId,
|
||||
}))
|
||||
: dataset.dataset_type === 'raster' && dataset.source_name === 'spw_bathymetry'
|
||||
? bathymetryRasterSelectionToMapSelection(await datasetsApi.selectBathymetryRaster(selectedProjectId, dataset.id, {
|
||||
bbox,
|
||||
|
||||
@@ -14,6 +14,7 @@ import type {
|
||||
|
||||
export interface OfficialMapProducts {
|
||||
thematic: ThematicRasterProductRead[]
|
||||
walous: ThematicRasterProductRead[]
|
||||
dhmv: DhmvProductRead[]
|
||||
floodHazard: FloodHazardProductRead[]
|
||||
grb: GrbProductRead[]
|
||||
@@ -23,6 +24,7 @@ export interface OfficialMapProducts {
|
||||
|
||||
const EMPTY_PRODUCTS: OfficialMapProducts = {
|
||||
thematic: [],
|
||||
walous: [],
|
||||
dhmv: [],
|
||||
floodHazard: [],
|
||||
grb: [],
|
||||
@@ -50,16 +52,18 @@ export function useOfficialMapProducts(selectedProjectId: string | null) {
|
||||
setError(null)
|
||||
void Promise.all([
|
||||
datasetsApi.listThematicRasterProducts(selectedProjectId),
|
||||
datasetsApi.listWalousProducts(selectedProjectId),
|
||||
datasetsApi.listDhmvProducts(selectedProjectId),
|
||||
datasetsApi.listFloodHazardProducts(selectedProjectId),
|
||||
datasetsApi.listGrbProducts(selectedProjectId),
|
||||
datasetsApi.listOfficialVectorProducts(selectedProjectId),
|
||||
datasetsApi.listBathymetrySources(selectedProjectId),
|
||||
])
|
||||
.then(([thematic, dhmv, floodHazard, grb, officialVector, bathymetry]) => {
|
||||
.then(([thematic, walous, dhmv, floodHazard, grb, officialVector, bathymetry]) => {
|
||||
if (!cancelled) {
|
||||
setProducts({
|
||||
thematic: thematic.items,
|
||||
walous: walous.items,
|
||||
dhmv: dhmv.items,
|
||||
floodHazard: floodHazard.items,
|
||||
grb: grb.items,
|
||||
|
||||
@@ -5,6 +5,7 @@ const NON_IMAGERY_RASTER_SOURCES = new Set([
|
||||
'digitaal_vlaanderen_dhmv',
|
||||
'vmm_flood_hazard',
|
||||
'spw_bathymetry',
|
||||
'spw_walous_land_cover',
|
||||
])
|
||||
|
||||
export function isDetectionImageryDataset(dataset: DatasetCreateResponse): boolean {
|
||||
|
||||
@@ -17,6 +17,7 @@ const DATASET_LABEL_BY_LAYER: Record<string, string> = {
|
||||
open_space: 'Open ruimte',
|
||||
accessibility: 'Knooppuntwaarde',
|
||||
services: 'Voorzieningenniveau',
|
||||
land_cover: 'Landbedekking',
|
||||
building_registry: 'Gebouwenregister',
|
||||
regional_boundary: 'Grens vervoerregio Kempen',
|
||||
municipality_boundaries: 'Gemeentegrenzen Kempen',
|
||||
@@ -37,6 +38,8 @@ const DATASET_SOURCE_LABELS: Record<string, string> = {
|
||||
vmm_flood_hazard: 'Vlaamse Milieumaatschappij',
|
||||
vmm_vha_bathymetry_profiles: 'VMM / Vlaamse Hydrografische Atlas',
|
||||
spw_bathymetry: 'Service public de Wallonie',
|
||||
spw_walous_land_cover: 'Service public de Wallonie',
|
||||
spw_flood_hazard: 'Service public de Wallonie',
|
||||
department_omgeving_thematic_raster: 'Departement Omgeving',
|
||||
dov_soil_map: 'Databank Ondergrond Vlaanderen',
|
||||
}
|
||||
@@ -65,6 +68,10 @@ export function getDatasetDisplayName(dataset: DatasetCreateResponse): string {
|
||||
const productName = dataset.source_metadata?.['product_display_name']
|
||||
return typeof productName === 'string' && productName.trim() ? productName : 'Officieel Vlaams themaraster'
|
||||
}
|
||||
if (dataset.source_name === 'spw_walous_land_cover') {
|
||||
const productName = dataset.source_metadata?.['product_display_name']
|
||||
return typeof productName === 'string' && productName.trim() ? productName : 'WALOUS landbedekking'
|
||||
}
|
||||
const layer = (dataset.reference_layer_name ?? dataset.source_metadata?.layer_name ?? dataset.source_metadata?.layer_type ?? '')
|
||||
.toString()
|
||||
.toLowerCase()
|
||||
|
||||
@@ -4,6 +4,10 @@ export function thematicRasterImageUrl(projectId: string, datasetId: string): st
|
||||
return `/api/v1/projects/${projectId}/datasets/${datasetId}/raster/thematic/image`
|
||||
}
|
||||
|
||||
export function walousRasterImageUrl(projectId: string, datasetId: string): string {
|
||||
return `/api/v1/projects/${projectId}/datasets/${datasetId}/raster/walous/image`
|
||||
}
|
||||
|
||||
export function thematicRasterSelectionToMapSelection(result: ThematicRasterSelectionResponse): VectorSelectionResponse {
|
||||
return {
|
||||
selection_bbox: result.selection_bbox,
|
||||
|
||||
@@ -220,12 +220,22 @@ export const datasetsApi = {
|
||||
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/thematic-raster/acquire`, payload),
|
||||
listThematicRasterProducts: (projectId: string): Promise<{ items: ThematicRasterProductRead[]; total: number }> =>
|
||||
apiGet<{ items: ThematicRasterProductRead[]; total: number }>(`/api/v1/projects/${projectId}/datasets/thematic-raster/products`),
|
||||
acquireWalous: (projectId: string, payload: ThematicRasterAcquireRequest): Promise<JobRead> =>
|
||||
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/walous/acquire`, payload),
|
||||
listWalousProducts: (projectId: string): Promise<{ items: ThematicRasterProductRead[]; total: number }> =>
|
||||
apiGet<{ items: ThematicRasterProductRead[]; total: number }>(`/api/v1/projects/${projectId}/datasets/walous/products`),
|
||||
selectThematicRaster: (
|
||||
projectId: string,
|
||||
datasetId: string,
|
||||
payload: { bbox: VectorSelectionRequest['bbox']; area_id?: string },
|
||||
): Promise<ThematicRasterSelectionResponse> =>
|
||||
apiPost<ThematicRasterSelectionResponse>(`/api/v1/projects/${projectId}/datasets/${datasetId}/raster/thematic/select`, payload),
|
||||
selectWalous: (
|
||||
projectId: string,
|
||||
datasetId: string,
|
||||
payload: { bbox: VectorSelectionRequest['bbox']; area_id?: string },
|
||||
): Promise<ThematicRasterSelectionResponse> =>
|
||||
apiPost<ThematicRasterSelectionResponse>(`/api/v1/projects/${projectId}/datasets/${datasetId}/raster/walous/select`, payload),
|
||||
refreshMetadata: (projectId: string, datasetId: string): Promise<DatasetCreateResponse> =>
|
||||
apiPost<DatasetCreateResponse>(`/api/v1/projects/${projectId}/datasets/${datasetId}/metadata/refresh`, {}),
|
||||
inspectRaster: (projectId: string, datasetId: string): Promise<RasterInspectResponse> =>
|
||||
|
||||
@@ -576,11 +576,12 @@ export interface ThematicRasterAcquireRequest {
|
||||
export interface ThematicRasterProductRead {
|
||||
key: string
|
||||
display_name: string
|
||||
theme: 'space_occupation' | 'open_space' | 'forest' | 'agriculture' | 'population' | 'accessibility' | 'services'
|
||||
metric_kind: 'binary_area' | 'population_density' | 'index_score' | 'normalized_score'
|
||||
theme: 'space_occupation' | 'open_space' | 'forest' | 'agriculture' | 'population' | 'accessibility' | 'services' | 'land_cover'
|
||||
metric_kind: 'binary_area' | 'population_density' | 'index_score' | 'normalized_score' | 'categorical_area'
|
||||
coverage_id: string
|
||||
native_resolution_m: number
|
||||
source_crs: 'EPSG:31370'
|
||||
analysis_resolution_m?: number | null
|
||||
source_crs: 'EPSG:31370' | 'EPSG:3812'
|
||||
source_value_unit: string
|
||||
observation_year: number
|
||||
source_version: string
|
||||
@@ -591,6 +592,9 @@ export interface ThematicRasterProductRead {
|
||||
legend_max_label: string
|
||||
included_source_values: number[]
|
||||
limitation_message: string
|
||||
coverage_zones: string[]
|
||||
configured: boolean
|
||||
status: 'configured' | 'source_not_provisioned'
|
||||
}
|
||||
|
||||
export interface ThematicRasterSelectionResponse {
|
||||
|
||||
@@ -2,6 +2,24 @@
|
||||
|
||||
Setup-, import-, demo- en maintenance-scripts voor GeoIntel.
|
||||
|
||||
## WALOUS source provisioning
|
||||
|
||||
Run the networked operator only after checking at least 2 GB of archive space
|
||||
plus room for the extracted official GeoTIFFs:
|
||||
|
||||
```bash
|
||||
python scripts/provision_walous_sources.py \
|
||||
--years 2020 2023 \
|
||||
--destination storage/source-cache/walous
|
||||
```
|
||||
|
||||
The command accepts only the hard-coded official SPW 2020/2023 archives,
|
||||
streams with a 1 GB per-archive cap, rejects changed content lengths, extracts
|
||||
only the single GeoTIFF by basename, validates the raster contract and writes
|
||||
checksums plus `provisioning-report.json`. Existing valid sources are reused;
|
||||
`--force` performs a new download. This is an operator acquisition, not an
|
||||
application startup task.
|
||||
|
||||
## Runtime verification
|
||||
|
||||
Inspect interrupted runtime state without changing it:
|
||||
|
||||
@@ -19,6 +19,7 @@ ALLOWED_NON_ENVELOPE_ENDPOINTS = {
|
||||
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/bathymetry/image"),
|
||||
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/flood-hazard/image"),
|
||||
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/thematic/image"),
|
||||
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/walous/image"),
|
||||
}
|
||||
|
||||
IGNORED_OPENAPI_PATHS = {
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Provision official WALOUS GeoTIFF source rasters for bounded runtime analysis."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
from urllib.request import Request, urlopen
|
||||
from zipfile import BadZipFile, ZipFile
|
||||
|
||||
|
||||
SOURCES = {
|
||||
2020: {
|
||||
"url": (
|
||||
"https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
|
||||
"47b348f1-6e7a-4baa-963c-0232a43c0cff/WAL_OCS_IA__2020_GEOTIFF_3812.zip"
|
||||
),
|
||||
"expected_archive_bytes": 728_244_755,
|
||||
"target": "walous_land_cover_2020_3812.tif",
|
||||
},
|
||||
2023: {
|
||||
"url": (
|
||||
"https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
|
||||
"4e780ba1-463c-478e-95df-d2f1963a150d/WAL_OCS_IA__2023_GEOTIFF_3812.zip"
|
||||
),
|
||||
"expected_archive_bytes": 876_014_572,
|
||||
"target": "walous_land_cover_2023_3812.tif",
|
||||
},
|
||||
}
|
||||
MAX_ARCHIVE_BYTES = 1_000_000_000
|
||||
MAX_EXTRACTED_BYTES = 50_000_000_000
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
while chunk := handle.read(8 * 1024 * 1024):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def download(url: str, destination: Path, expected_bytes: int) -> str:
|
||||
temporary = destination.with_suffix(destination.suffix + ".part")
|
||||
temporary.unlink(missing_ok=True)
|
||||
digest = hashlib.sha256()
|
||||
received = 0
|
||||
request = Request(url, headers={"User-Agent": "GeoIntel/1.0 WALOUS-source-provisioner"})
|
||||
try:
|
||||
with urlopen(request, timeout=300) as response, temporary.open("wb") as output:
|
||||
content_length = int(response.headers.get("Content-Length") or 0)
|
||||
if content_length and content_length != expected_bytes:
|
||||
raise RuntimeError(f"official archive size changed: expected {expected_bytes}, advertised {content_length}")
|
||||
while chunk := response.read(8 * 1024 * 1024):
|
||||
received += len(chunk)
|
||||
if received > MAX_ARCHIVE_BYTES:
|
||||
raise RuntimeError("official archive exceeds the governed 1 GB transfer limit")
|
||||
digest.update(chunk)
|
||||
output.write(chunk)
|
||||
if received % (128 * 1024 * 1024) < len(chunk):
|
||||
print(f" downloaded {received / 1024 / 1024:.0f} MiB", flush=True)
|
||||
if received != expected_bytes:
|
||||
raise RuntimeError(f"archive is incomplete: expected {expected_bytes} bytes, received {received}")
|
||||
temporary.replace(destination)
|
||||
return digest.hexdigest()
|
||||
except Exception:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def extract_single_geotiff(archive: Path, target: Path) -> None:
|
||||
try:
|
||||
with ZipFile(archive) as bundle:
|
||||
candidates = [item for item in bundle.infolist() if not item.is_dir() and item.filename.lower().endswith((".tif", ".tiff"))]
|
||||
if len(candidates) != 1:
|
||||
raise RuntimeError(f"archive must contain exactly one GeoTIFF, found {len(candidates)}")
|
||||
member = candidates[0]
|
||||
if member.file_size <= 0 or member.file_size > MAX_EXTRACTED_BYTES:
|
||||
raise RuntimeError(f"GeoTIFF uncompressed size is outside the governed limit: {member.file_size}")
|
||||
if Path(member.filename).name != member.filename.replace("\\", "/").split("/")[-1]:
|
||||
# Nested paths are accepted only by basename; extraction never trusts archive paths.
|
||||
pass
|
||||
temporary = target.with_suffix(target.suffix + ".part")
|
||||
temporary.unlink(missing_ok=True)
|
||||
with bundle.open(member) as source, temporary.open("wb") as output:
|
||||
shutil.copyfileobj(source, output, length=8 * 1024 * 1024)
|
||||
temporary.replace(target)
|
||||
except BadZipFile as exc:
|
||||
raise RuntimeError("official WALOUS archive is not a valid ZIP file") from exc
|
||||
|
||||
|
||||
def validate_raster(path: Path) -> dict:
|
||||
try:
|
||||
import numpy as np
|
||||
import rasterio
|
||||
from rasterio.enums import Resampling
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("rasterio and numpy are required to validate WALOUS sources") from exc
|
||||
with rasterio.open(path) as source:
|
||||
if source.crs is None or source.crs.to_epsg() != 3812:
|
||||
raise RuntimeError(f"WALOUS raster must use EPSG:3812, found {source.crs}")
|
||||
if source.count != 1:
|
||||
raise RuntimeError(f"WALOUS raster must have one band, found {source.count}")
|
||||
if not all(abs(abs(float(value)) - 1.0) <= 0.05 for value in source.res):
|
||||
raise RuntimeError(f"WALOUS raster must retain 1 m cells, found {source.res}")
|
||||
sample_height = min(2048, source.height)
|
||||
sample_width = min(2048, source.width)
|
||||
sample = source.read(1, out_shape=(sample_height, sample_width), masked=True, resampling=Resampling.nearest)
|
||||
values = np.unique(sample.compressed()).astype(int).tolist()
|
||||
unexpected = sorted(set(values) - set(range(1, 12)))
|
||||
if unexpected:
|
||||
raise RuntimeError(f"WALOUS sample contains classes outside 1-11: {unexpected}")
|
||||
return {
|
||||
"path": str(path),
|
||||
"crs": str(source.crs),
|
||||
"width": int(source.width),
|
||||
"height": int(source.height),
|
||||
"resolution": [float(value) for value in source.res],
|
||||
"bounds": [float(value) for value in source.bounds],
|
||||
"nodata": None if source.nodata is None else float(source.nodata),
|
||||
"sample_classes": values,
|
||||
}
|
||||
|
||||
|
||||
def provision(year: int, destination: Path, force: bool) -> dict:
|
||||
source = SOURCES[year]
|
||||
target = destination / source["target"]
|
||||
archive = destination / f"{Path(source['target']).stem}.zip"
|
||||
if target.is_file() and not force:
|
||||
print(f"WALOUS {year}: validating existing source {target}")
|
||||
validation = validate_raster(target)
|
||||
digest = sha256_file(target)
|
||||
else:
|
||||
print(f"WALOUS {year}: downloading official archive")
|
||||
archive_digest = download(source["url"], archive, source["expected_archive_bytes"])
|
||||
print(f"WALOUS {year}: archive sha256 {archive_digest}")
|
||||
extract_single_geotiff(archive, target)
|
||||
validation = validate_raster(target)
|
||||
digest = sha256_file(target)
|
||||
archive.unlink(missing_ok=True)
|
||||
checksum_path = target.with_suffix(".sha256")
|
||||
checksum_path.write_text(f"{digest} {target.name}\n", encoding="ascii")
|
||||
validation.update({"year": year, "sha256": digest, "download_url": source["url"]})
|
||||
print(f"WALOUS {year}: ready ({target.stat().st_size / 1024 / 1024:.0f} MiB)")
|
||||
return validation
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Provision official WALOUS 2020/2023 GeoTIFF sources.")
|
||||
parser.add_argument("--years", nargs="+", type=int, choices=sorted(SOURCES), default=sorted(SOURCES))
|
||||
parser.add_argument("--destination", type=Path, default=Path("storage/source-cache/walous"))
|
||||
parser.add_argument("--force", action="store_true")
|
||||
args = parser.parse_args()
|
||||
args.destination.mkdir(parents=True, exist_ok=True)
|
||||
report = [provision(year, args.destination.resolve(), args.force) for year in args.years]
|
||||
report_path = args.destination / "provisioning-report.json"
|
||||
report_path.write_text(json.dumps({"sources": report}, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"Provisioning report: {report_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except Exception as exc:
|
||||
print(f"WALOUS_PROVISIONING_FAILED: {exc}", file=sys.stderr)
|
||||
raise SystemExit(1) from exc
|
||||
@@ -81,6 +81,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_flanders_geographic_scope.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_flanders_bathymetry_profiles.py
|
||||
${PYTHON_BIN} -m py_compile scripts/probe_mdk_bathymetry.py
|
||||
${PYTHON_BIN} -m py_compile scripts/import_spw_bathymetry.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_walous_sources.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py
|
||||
|
||||
Reference in New Issue
Block a user