From 86dd1a56890f05fb6615d908d0a9ff7fc93e67a4 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 22 Jul 2026 04:30:54 +0200 Subject: [PATCH] fix: align WALOUS with official class codes --- CHANGELOG.md | 4 + backend/README.md | 14 ++- .../app/services/walous_land_cover_service.py | 85 +++++++++++-------- .../tests/test_walous_land_cover_service.py | 43 ++++++++-- docs/API_CONTRACTS.md | 10 ++- docs/CODEX_EXECUTION_LOG.md | 6 ++ docs/DATA_SOURCES.md | 21 +++++ docs/DATA_SPECIFICATION.md | 9 ++ scripts/provision_walous_sources.py | 5 +- 9 files changed, 144 insertions(+), 53 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4be2ea3..2127e6e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ temporal comparison. The map acquires comparable configured editions for the same selection so 2020-2023 evolution becomes available without manual dataset administration. +- Corrected WALOUS to the official non-contiguous raster code set + `1,2,3,4,5,6,7,8,9,80,90`, including class labels, colours, semantic area + aggregation and exact SPW observation ranges. Live provisioning now rejects + unknown values without rejecting valid low woody-cover codes 80 and 90. - Added the current legal SPW Walloon flood-hazard polygons as a bounded authoritative vector product with class-aware hectare metrics and canonical persistence. No WMS pixels or modeled depths are fabricated. diff --git a/backend/README.md b/backend/README.md index 5e0e80ef..6946e8e2 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1613,9 +1613,10 @@ docker exec geointel python /app/scripts/provision_walous_sources.py \ ``` The provisioner verifies advertised archive sizes, safe ZIP structure, -EPSG:3812, one band, 1 m cells, class values 1-11 and SHA-256 checksums. It -does not run at application startup. `GET .../datasets/walous/products` -therefore reports `source_not_provisioned` until both source files exist. +EPSG:3812, one band, 1 m cells, the official non-contiguous class codes +`1,2,3,4,5,6,7,8,9,80,90` and SHA-256 checksums. It does not run at +application startup. `GET .../datasets/walous/products` therefore reports +`source_not_provisioned` until both source files exist. For a bounded Walloon selection the browser persists the latest edition and all other configured comparable editions. `POST .../raster/walous/select` @@ -1623,6 +1624,13 @@ returns cell-area hectares; the temporal API compares the same semantic metric keys for 2020 and 2023. WALOUS is land cover, not legal land use, ownership, tree count, timber volume or water volume. +The class semantics follow the official raster codes, not display-list +positions: 1 artificial ground, 2 above-ground construction, 3 railway, 4 bare +soil, 5 surface water, 6 rotating herbaceous cover, 7 continuous herbaceous +cover, 8/9 trees above 3 m and 80/90 woody cover up to 3 m. Observation ranges +are retained from the SPW metadata rather than replaced by arbitrary year-end +dates. + Settings: `WALOUS_ENABLED`, `WALOUS_SOURCE_DIR`, `WALOUS_ANALYSIS_RESOLUTION_M`, `WALOUS_MAX_SIDE_M` and `WALOUS_MAX_PIXELS`. The SPW flood polygon adapter uses diff --git a/backend/app/services/walous_land_cover_service.py b/backend/app/services/walous_land_cover_service.py index b0ee129b..07f97676 100644 --- a/backend/app/services/walous_land_cover_service.py +++ b/backend/app/services/walous_land_cover_service.py @@ -41,13 +41,15 @@ class WalousProduct: download_url: str source_sha256_filename: str accuracy_label: str + observation_start: datetime + observation_end: datetime class WalousLandCoverService: PROVIDER = "spw_walous_land_cover" SOURCE_CRS = "EPSG:3812" SOURCE_RESOLUTION_M = 1.0 - SOURCE_VALUE_UNIT = "class_1_11" + SOURCE_VALUE_UNIT = "walous_class_code" THEME = "land_cover_use" METRIC_KIND = "categorical_area" NODATA = 255 @@ -58,31 +60,33 @@ class WalousLandCoverService: "geconfigureerde analyseresolutie. Oppervlakten zijn celgebaseerde schattingen; de kaart is landbedekking, " "geen juridisch landgebruik, eigendom, boomtelling of actuele terreinwaarneming." ) + # WALOUS has 11 semantic classes, but its official raster codes are not a + # continuous 1..11 range. Codes 80 and 90 distinguish low woody cover. CLASS_LABELS = { - 1: "Jaarlijks wisselende kruidlaag", - 2: "Jaarronde kruidlaag", - 3: "Naaldbomen hoger dan 3 m", - 4: "Loofbomen hoger dan 3 m", - 5: "Naaldbomen tot 3 m", - 6: "Loofbomen tot 3 m", - 7: "Kale bodem", - 8: "Oppervlaktewater", - 9: "Kunstmatige bodembedekking", - 10: "Spoorweg", - 11: "Kunstmatige constructies boven maaiveld", + 1: "Kunstmatige bodembedekking", + 2: "Kunstmatige constructies boven maaiveld", + 3: "Spoorweg", + 4: "Kale bodem", + 5: "Oppervlaktewater", + 6: "Jaarlijks wisselende kruidlaag", + 7: "Jaarronde kruidlaag", + 8: "Naaldbomen hoger dan 3 m", + 9: "Loofbomen hoger dan 3 m", + 80: "Naaldbomen tot 3 m", + 90: "Loofbomen tot 3 m", } CLASS_COLORS = { - 1: (236, 202, 73), - 2: (161, 201, 78), - 3: (28, 89, 51), - 4: (52, 132, 72), - 5: (78, 125, 70), - 6: (107, 164, 87), - 7: (194, 165, 119), - 8: (44, 129, 185), - 9: (155, 155, 155), - 10: (68, 68, 68), - 11: (183, 72, 67), + 1: (155, 155, 155), + 2: (183, 72, 67), + 3: (68, 68, 68), + 4: (194, 165, 119), + 5: (44, 129, 185), + 6: (236, 202, 73), + 7: (161, 201, 78), + 8: (28, 89, 51), + 9: (52, 132, 72), + 80: (78, 125, 70), + 90: (107, 164, 87), } @staticmethod @@ -101,6 +105,8 @@ class WalousLandCoverService: ), source_sha256_filename="walous_land_cover_2020_3812.sha256", accuracy_label="Officiele globale nauwkeurigheid 83,30%", + observation_start=datetime(2020, 4, 1, tzinfo=UTC), + observation_end=datetime(2020, 4, 24, 23, 59, 59, tzinfo=UTC), ), WalousProduct( key="walous_land_cover_2023", @@ -115,6 +121,8 @@ class WalousLandCoverService: ), source_sha256_filename="walous_land_cover_2023_3812.sha256", accuracy_label="Officiele globale nauwkeurigheid 87,10%", + observation_start=datetime(2023, 5, 27, tzinfo=UTC), + observation_end=datetime(2023, 6, 25, 23, 59, 59, tzinfo=UTC), ), ) return {product.key: product for product in products} @@ -145,8 +153,8 @@ class WalousLandCoverService: catalog_url=product.catalog_url, attribution=WalousLandCoverService.ATTRIBUTION, license_note=WalousLandCoverService.LICENSE_NOTE, - legend_min_label="WALOUS klasse 1", - legend_max_label="WALOUS klasse 11", + legend_min_label="WALOUS klasse 1 (kunstmatige bodem)", + legend_max_label="WALOUS klasse 90 (loofbomen tot 3 m)", included_source_values=list(WalousLandCoverService.CLASS_LABELS), limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.", coverage_zones=["wallonia"], @@ -250,7 +258,12 @@ class WalousLandCoverService: classes = set(np.unique(valid).astype(int).tolist()) unexpected = sorted(classes - set(WalousLandCoverService.CLASS_LABELS)) if unexpected: - raise AppError(code="WALOUS_SOURCE_INVALID_VALUES", message="WALOUS contains classes outside the governed 1-11 legend", details={"unexpected_classes": unexpected}, status_code=409) + raise AppError( + code="WALOUS_SOURCE_INVALID_VALUES", + message="WALOUS contains classes outside the governed 11-class code set", + details={"unexpected_classes": unexpected}, + status_code=409, + ) profile = { "driver": "GTiff", "width": width, @@ -345,7 +358,7 @@ class WalousLandCoverService: source_sha256_path = source_path.with_name(product.source_sha256_filename) source_sha256 = source_sha256_path.read_text(encoding="ascii").strip().split()[0] if source_sha256_path.is_file() else None acquired_at = datetime.now(UTC) - observed_at = datetime(product.observation_year, 12, 31, 23, 59, 59, tzinfo=UTC) + observed_at = product.observation_end spatial_series_hash = hashlib.sha256(json.dumps({"bbox": identity["bbox_epsg4326"], "area_id": identity["area_id"], "resolution": identity["analysis_resolution_m"]}, sort_keys=True).encode()).hexdigest()[:24] dataset = DatasetService.import_raster_bytes( db, @@ -357,8 +370,8 @@ class WalousLandCoverService: source_name=WalousLandCoverService.PROVIDER, temporal_series_key=f"spw:walous:land-cover:{spatial_series_hash}", observed_at=observed_at, - valid_from=datetime(product.observation_year, 1, 1, tzinfo=UTC), - valid_to=observed_at, + valid_from=product.observation_start, + valid_to=product.observation_end, temporal_granularity="year", source_version=product.source_version, source_metadata={ @@ -374,6 +387,8 @@ class WalousLandCoverService: "source_value_unit": WalousLandCoverService.SOURCE_VALUE_UNIT, "class_labels": WalousLandCoverService.CLASS_LABELS, "observation_year": product.observation_year, + "observation_start": product.observation_start.isoformat(), + "observation_end": product.observation_end.isoformat(), "valid_pixel_count": validation["valid_pixel_count"], "classes_present": validation["classes_present"], "bbox_epsg4326": bbox_4326, @@ -480,12 +495,12 @@ class WalousLandCoverService: metric_specs = [ ("land_cover_observed_area_ha", "Gekarteerde landbedekking", set(WalousLandCoverService.CLASS_LABELS)), - ("forest_cover_area_ha", "Boom- en bosbedekking", {3, 4, 5, 6}), - ("surface_water_area_ha", "Oppervlaktewater", {8}), - ("artificial_cover_area_ha", "Kunstmatige bedekking en constructies", {9, 10, 11}), - ("annual_herbaceous_cover_area_ha", "Jaarlijks wisselende kruidlaag", {1}), - ("permanent_herbaceous_cover_area_ha", "Jaarronde kruidlaag", {2}), - ("bare_soil_area_ha", "Kale bodem", {7}), + ("forest_cover_area_ha", "Boom- en bosbedekking", {8, 9, 80, 90}), + ("surface_water_area_ha", "Oppervlaktewater", {5}), + ("artificial_cover_area_ha", "Kunstmatige bedekking en constructies", {1, 2, 3}), + ("annual_herbaceous_cover_area_ha", "Jaarlijks wisselende kruidlaag", {6}), + ("permanent_herbaceous_cover_area_ha", "Jaarronde kruidlaag", {7}), + ("bare_soil_area_ha", "Kale bodem", {4}), ] metrics = [ ThematicRasterMetric( diff --git a/backend/tests/test_walous_land_cover_service.py b/backend/tests/test_walous_land_cover_service.py index c28a22e4..c3a47ef8 100644 --- a/backend/tests/test_walous_land_cover_service.py +++ b/backend/tests/test_walous_land_cover_service.py @@ -1,6 +1,7 @@ from __future__ import annotations from datetime import datetime, timezone +import importlib.util from pathlib import Path from types import SimpleNamespace from uuid import uuid4 @@ -22,6 +23,15 @@ from app.services.temporal_analysis_service import TemporalAnalysisService from app.services.walous_land_cover_service import WalousLandCoverService +def load_provisioner(): + path = Path(__file__).resolve().parents[2] / "scripts" / "provision_walous_sources.py" + spec = importlib.util.spec_from_file_location("walous_source_provisioner_test", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + class FakeQuery: def filter(self, *_args): return self @@ -70,16 +80,15 @@ def make_source(path: Path) -> tuple[list[float], np.ndarray]: to_4326 = Transformer.from_crs("EPSG:3812", "EPSG:4326", always_xy=True) x, y = to_3812.transform(4.85, 50.45) transform = from_origin(x, y + 100, 1, 1) - values = np.ones((100, 100), dtype="uint8") - values[:, 20:40] = 4 - values[:, 40:50] = 8 - values[:, 50:70] = 9 - values[:, 70:] = 2 + class_codes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90] + values = np.empty((100, len(class_codes) * 20), dtype="uint8") + for index, class_code in enumerate(class_codes): + values[:, index * 20 : (index + 1) * 20] = class_code with rasterio.open( path, "w", driver="GTiff", - width=100, + width=values.shape[1], height=100, count=1, dtype="uint8", @@ -89,7 +98,7 @@ def make_source(path: Path) -> tuple[list[float], np.ndarray]: ) as target: target.write(values, 1) min_lon, min_lat = to_4326.transform(x, y) - max_lon, max_lat = to_4326.transform(x + 100, y + 100) + max_lon, max_lat = to_4326.transform(x + values.shape[1], y + values.shape[0]) return [min_lon, min_lat, max_lon, max_lat], values @@ -113,6 +122,17 @@ def test_walous_registry_reports_real_provisioning_state(tmp_path: Path) -> None assert after["walous_land_cover_2023"]["native_resolution_m"] == 1.0 assert after["walous_land_cover_2023"]["analysis_resolution_m"] == 10.0 assert after["walous_land_cover_2023"]["coverage_zones"] == ["wallonia"] + assert after["walous_land_cover_2023"]["included_source_values"] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90] + assert after["walous_land_cover_2023"]["source_value_unit"] == "walous_class_code" + + +def test_walous_provisioner_accepts_official_non_contiguous_class_codes(tmp_path: Path) -> None: + source_path = tmp_path / "walous_land_cover_2023_3812.tif" + make_source(source_path) + + validation = load_provisioner().validate_raster(source_path) + + assert validation["sample_classes"] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90] def test_walous_acquisition_reads_real_classes_and_persists_provenance(tmp_path: Path, monkeypatch) -> None: @@ -141,10 +161,12 @@ def test_walous_acquisition_reads_real_classes_and_persists_provenance(tmp_path: assert result["output_dataset_id"] == str(output_id) assert result["resolution_m"] == 10 assert captured["source_name"] == "spw_walous_land_cover" - assert captured["source_metadata"]["classes_present"] == [1, 2, 4, 8, 9] + assert captured["source_metadata"]["classes_present"] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90] assert captured["provenance_metadata"]["resampling"] == "nearest" assert captured["temporal_series_key"].startswith("spw:walous:land-cover:") - assert captured["observed_at"].year == 2023 + assert captured["observed_at"].date().isoformat() == "2023-06-25" + assert captured["valid_from"].date().isoformat() == "2023-05-27" + assert captured["valid_to"] == captured["observed_at"] def test_walous_analysis_returns_semantic_area_metrics(tmp_path: Path, monkeypatch) -> None: @@ -193,6 +215,9 @@ def test_walous_analysis_returns_semantic_area_metrics(tmp_path: Path, monkeypat assert metrics["forest_cover_area_ha"] > 0 assert metrics["surface_water_area_ha"] > 0 assert metrics["artificial_cover_area_ha"] > 0 + assert metrics["annual_herbaceous_cover_area_ha"] > 0 + assert metrics["permanent_herbaceous_cover_area_ha"] > 0 + assert metrics["bare_soil_area_ha"] > 0 assert "water_volume" in result["unsupported_metrics"] diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 3302e397..0bb4de2d 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -494,10 +494,12 @@ becomes true only when the checksum-validated source GeoTIFF exists below Reads a bounded window from one provisioned official 1 m WALOUS source, applies nearest-neighbour resampling to the configured analysis resolution, -masks `bbox intersect Area`, validates class values 1-11 and persists a normal -raster Dataset through `DatasetService`. URLs, paths, classes and resolutions -are not caller-controlled. Equal spatial requests for 2020 and 2023 share one -temporal series key. +masks `bbox intersect Area`, validates the official non-contiguous class-code +set `1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90` and persists a normal raster Dataset +through `DatasetService`. URLs, paths, classes and resolutions are not +caller-controlled. Equal spatial requests for 2020 and 2023 share one temporal +series key. The exact official observation ranges are retained as +2020-04-01/2020-04-24 and 2023-05-27/2023-06-25. ### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/walous/select` diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 94bc405c..10442687 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -3654,6 +3654,12 @@ Validation: ## Post-V1 national coverage completion: Wallonia (2026-07-22) +- Live Tower provisioning exposed an incorrect assumption that 11 WALOUS + classes implied numeric codes 1 through 11. The official SPW legend and the + downloaded 2020 raster confirm codes `1,2,3,4,5,6,7,8,9,80,90`. Corrected + validation, class semantics, colours and all hectare aggregations; added a + provisioner regression containing codes 80/90 and retained exact source + observation ranges. - Validated the official WALOUS 2020 and 2023 archives, their EPSG:3812 1 m raster contract, 11 classes, CC BY 4.0 attribution and published edition accuracy. Added a fail-closed operator provisioner with archive-size, diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 1336dbe6..70df3aa3 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -790,6 +790,27 @@ Areas. They persist only through `DatasetService`; the browser never contacts either provider directly. Cross-zone selections remain split by authority and metric semantics. +## Wallonia WALOUS land cover + +The official SPW `WAL_OCS_IA__2020` and `WAL_OCS_IA__2023` GeoTIFF archives +provide comparable Walloon land-cover observations at native 1 m resolution in +EPSG:3812. Runtime analysis reads only bounded windows from operator- +provisioned, checksum-recorded source files and uses nearest-neighbour +resampling for the governed 10 m analysis derivative. + +The 11 semantic classes use the non-contiguous source codes `1, 2, 3, 4, 5, 6, +7, 8, 9, 80, 90`. In order these mean artificial ground, above-ground +construction, railway, bare soil, surface water, rotating herbaceous cover, +continuous herbaceous cover, conifer trees above 3 m, deciduous trees above 3 +m, conifer woody cover up to 3 m and deciduous woody cover up to 3 m. Codes 80 +and 90 must never be normalized to invented classes 10 and 11. + +The official temporal extents are 2020-04-01 through 2020-04-24 and 2023-05-27 +through 2023-06-25. Metrics are estimated hectares from classified cells. They +are not legal land use, ownership, individual tree counts, timber volume or +water volume. The official catalogue reports overall accuracy per edition and +also warns that accuracy varies by class and place. + ## Bathymetry, inland profiles and maritime scope The official VHA Digital Atlas profile-point layer is the first operational diff --git a/docs/DATA_SPECIFICATION.md b/docs/DATA_SPECIFICATION.md index 17448ed3..9fa810b9 100644 --- a/docs/DATA_SPECIFICATION.md +++ b/docs/DATA_SPECIFICATION.md @@ -308,6 +308,15 @@ review hashes, persists sampled official flight dates as the temporal evidence range and creates a new Dataset/DatasetVersion through DatasetService. It does not retroactively rewrite or delete legacy rows. +WALOUS land-cover rasters retain EPSG:3812, their native 1 m source resolution, +the derived analysis resolution, source checksum and exact observation range. +The governed class domain is `{1,2,3,4,5,6,7,8,9,80,90}`. Codes 80 and 90 are +valid official low-woody-cover classes; `10` and `11` are not substitutes. +Bounded derivatives use `uint8`, nodata 255 and nearest-neighbour resampling. +Area metrics group forest/tree cover as `{8,9,80,90}`, water as `{5}`, +artificial cover as `{1,2,3}`, rotating herbaceous cover as `{6}`, continuous +herbaceous cover as `{7}` and bare soil as `{4}`. + ### Hydrological station observations Waterinfo observations are persisted as EPSG:4326 Point features, one station diff --git a/scripts/provision_walous_sources.py b/scripts/provision_walous_sources.py index 2dab8343..c9be618a 100644 --- a/scripts/provision_walous_sources.py +++ b/scripts/provision_walous_sources.py @@ -33,6 +33,7 @@ SOURCES = { } MAX_ARCHIVE_BYTES = 1_000_000_000 MAX_EXTRACTED_BYTES = 50_000_000_000 +WALOUS_CLASS_CODES = {1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90} def sha256_file(path: Path) -> str: @@ -110,9 +111,9 @@ def validate_raster(path: Path) -> dict: sample_width = min(2048, source.width) sample = source.read(1, out_shape=(sample_height, sample_width), masked=True, resampling=Resampling.nearest) values = np.unique(sample.compressed()).astype(int).tolist() - unexpected = sorted(set(values) - set(range(1, 12))) + unexpected = sorted(set(values) - WALOUS_CLASS_CODES) if unexpected: - raise RuntimeError(f"WALOUS sample contains classes outside 1-11: {unexpected}") + raise RuntimeError(f"WALOUS sample contains classes outside the official 11-class code set: {unexpected}") return { "path": str(path), "crs": str(source.crs),