fix: align WALOUS with official class codes
This commit is contained in:
@@ -15,6 +15,10 @@
|
||||
temporal comparison. The map acquires comparable configured editions for
|
||||
the same selection so 2020-2023 evolution becomes available without manual
|
||||
dataset administration.
|
||||
- Corrected WALOUS to the official non-contiguous raster code set
|
||||
`1,2,3,4,5,6,7,8,9,80,90`, including class labels, colours, semantic area
|
||||
aggregation and exact SPW observation ranges. Live provisioning now rejects
|
||||
unknown values without rejecting valid low woody-cover codes 80 and 90.
|
||||
- Added the current legal SPW Walloon flood-hazard polygons as a bounded
|
||||
authoritative vector product with class-aware hectare metrics and canonical
|
||||
persistence. No WMS pixels or modeled depths are fabricated.
|
||||
|
||||
+11
-3
@@ -1613,9 +1613,10 @@ docker exec geointel python /app/scripts/provision_walous_sources.py \
|
||||
```
|
||||
|
||||
The provisioner verifies advertised archive sizes, safe ZIP structure,
|
||||
EPSG:3812, one band, 1 m cells, class values 1-11 and SHA-256 checksums. It
|
||||
does not run at application startup. `GET .../datasets/walous/products`
|
||||
therefore reports `source_not_provisioned` until both source files exist.
|
||||
EPSG:3812, one band, 1 m cells, the official non-contiguous class codes
|
||||
`1,2,3,4,5,6,7,8,9,80,90` and SHA-256 checksums. It does not run at
|
||||
application startup. `GET .../datasets/walous/products` therefore reports
|
||||
`source_not_provisioned` until both source files exist.
|
||||
|
||||
For a bounded Walloon selection the browser persists the latest edition and
|
||||
all other configured comparable editions. `POST .../raster/walous/select`
|
||||
@@ -1623,6 +1624,13 @@ returns cell-area hectares; the temporal API compares the same semantic metric
|
||||
keys for 2020 and 2023. WALOUS is land cover, not legal land use, ownership,
|
||||
tree count, timber volume or water volume.
|
||||
|
||||
The class semantics follow the official raster codes, not display-list
|
||||
positions: 1 artificial ground, 2 above-ground construction, 3 railway, 4 bare
|
||||
soil, 5 surface water, 6 rotating herbaceous cover, 7 continuous herbaceous
|
||||
cover, 8/9 trees above 3 m and 80/90 woody cover up to 3 m. Observation ranges
|
||||
are retained from the SPW metadata rather than replaced by arbitrary year-end
|
||||
dates.
|
||||
|
||||
Settings: `WALOUS_ENABLED`, `WALOUS_SOURCE_DIR`,
|
||||
`WALOUS_ANALYSIS_RESOLUTION_M`, `WALOUS_MAX_SIDE_M` and
|
||||
`WALOUS_MAX_PIXELS`. The SPW flood polygon adapter uses
|
||||
|
||||
@@ -41,13 +41,15 @@ class WalousProduct:
|
||||
download_url: str
|
||||
source_sha256_filename: str
|
||||
accuracy_label: str
|
||||
observation_start: datetime
|
||||
observation_end: datetime
|
||||
|
||||
|
||||
class WalousLandCoverService:
|
||||
PROVIDER = "spw_walous_land_cover"
|
||||
SOURCE_CRS = "EPSG:3812"
|
||||
SOURCE_RESOLUTION_M = 1.0
|
||||
SOURCE_VALUE_UNIT = "class_1_11"
|
||||
SOURCE_VALUE_UNIT = "walous_class_code"
|
||||
THEME = "land_cover_use"
|
||||
METRIC_KIND = "categorical_area"
|
||||
NODATA = 255
|
||||
@@ -58,31 +60,33 @@ class WalousLandCoverService:
|
||||
"geconfigureerde analyseresolutie. Oppervlakten zijn celgebaseerde schattingen; de kaart is landbedekking, "
|
||||
"geen juridisch landgebruik, eigendom, boomtelling of actuele terreinwaarneming."
|
||||
)
|
||||
# WALOUS has 11 semantic classes, but its official raster codes are not a
|
||||
# continuous 1..11 range. Codes 80 and 90 distinguish low woody cover.
|
||||
CLASS_LABELS = {
|
||||
1: "Jaarlijks wisselende kruidlaag",
|
||||
2: "Jaarronde kruidlaag",
|
||||
3: "Naaldbomen hoger dan 3 m",
|
||||
4: "Loofbomen hoger dan 3 m",
|
||||
5: "Naaldbomen tot 3 m",
|
||||
6: "Loofbomen tot 3 m",
|
||||
7: "Kale bodem",
|
||||
8: "Oppervlaktewater",
|
||||
9: "Kunstmatige bodembedekking",
|
||||
10: "Spoorweg",
|
||||
11: "Kunstmatige constructies boven maaiveld",
|
||||
1: "Kunstmatige bodembedekking",
|
||||
2: "Kunstmatige constructies boven maaiveld",
|
||||
3: "Spoorweg",
|
||||
4: "Kale bodem",
|
||||
5: "Oppervlaktewater",
|
||||
6: "Jaarlijks wisselende kruidlaag",
|
||||
7: "Jaarronde kruidlaag",
|
||||
8: "Naaldbomen hoger dan 3 m",
|
||||
9: "Loofbomen hoger dan 3 m",
|
||||
80: "Naaldbomen tot 3 m",
|
||||
90: "Loofbomen tot 3 m",
|
||||
}
|
||||
CLASS_COLORS = {
|
||||
1: (236, 202, 73),
|
||||
2: (161, 201, 78),
|
||||
3: (28, 89, 51),
|
||||
4: (52, 132, 72),
|
||||
5: (78, 125, 70),
|
||||
6: (107, 164, 87),
|
||||
7: (194, 165, 119),
|
||||
8: (44, 129, 185),
|
||||
9: (155, 155, 155),
|
||||
10: (68, 68, 68),
|
||||
11: (183, 72, 67),
|
||||
1: (155, 155, 155),
|
||||
2: (183, 72, 67),
|
||||
3: (68, 68, 68),
|
||||
4: (194, 165, 119),
|
||||
5: (44, 129, 185),
|
||||
6: (236, 202, 73),
|
||||
7: (161, 201, 78),
|
||||
8: (28, 89, 51),
|
||||
9: (52, 132, 72),
|
||||
80: (78, 125, 70),
|
||||
90: (107, 164, 87),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
@@ -101,6 +105,8 @@ class WalousLandCoverService:
|
||||
),
|
||||
source_sha256_filename="walous_land_cover_2020_3812.sha256",
|
||||
accuracy_label="Officiele globale nauwkeurigheid 83,30%",
|
||||
observation_start=datetime(2020, 4, 1, tzinfo=UTC),
|
||||
observation_end=datetime(2020, 4, 24, 23, 59, 59, tzinfo=UTC),
|
||||
),
|
||||
WalousProduct(
|
||||
key="walous_land_cover_2023",
|
||||
@@ -115,6 +121,8 @@ class WalousLandCoverService:
|
||||
),
|
||||
source_sha256_filename="walous_land_cover_2023_3812.sha256",
|
||||
accuracy_label="Officiele globale nauwkeurigheid 87,10%",
|
||||
observation_start=datetime(2023, 5, 27, tzinfo=UTC),
|
||||
observation_end=datetime(2023, 6, 25, 23, 59, 59, tzinfo=UTC),
|
||||
),
|
||||
)
|
||||
return {product.key: product for product in products}
|
||||
@@ -145,8 +153,8 @@ class WalousLandCoverService:
|
||||
catalog_url=product.catalog_url,
|
||||
attribution=WalousLandCoverService.ATTRIBUTION,
|
||||
license_note=WalousLandCoverService.LICENSE_NOTE,
|
||||
legend_min_label="WALOUS klasse 1",
|
||||
legend_max_label="WALOUS klasse 11",
|
||||
legend_min_label="WALOUS klasse 1 (kunstmatige bodem)",
|
||||
legend_max_label="WALOUS klasse 90 (loofbomen tot 3 m)",
|
||||
included_source_values=list(WalousLandCoverService.CLASS_LABELS),
|
||||
limitation_message=f"{WalousLandCoverService.LIMITATION} {product.accuracy_label}.",
|
||||
coverage_zones=["wallonia"],
|
||||
@@ -250,7 +258,12 @@ class WalousLandCoverService:
|
||||
classes = set(np.unique(valid).astype(int).tolist())
|
||||
unexpected = sorted(classes - set(WalousLandCoverService.CLASS_LABELS))
|
||||
if unexpected:
|
||||
raise AppError(code="WALOUS_SOURCE_INVALID_VALUES", message="WALOUS contains classes outside the governed 1-11 legend", details={"unexpected_classes": unexpected}, status_code=409)
|
||||
raise AppError(
|
||||
code="WALOUS_SOURCE_INVALID_VALUES",
|
||||
message="WALOUS contains classes outside the governed 11-class code set",
|
||||
details={"unexpected_classes": unexpected},
|
||||
status_code=409,
|
||||
)
|
||||
profile = {
|
||||
"driver": "GTiff",
|
||||
"width": width,
|
||||
@@ -345,7 +358,7 @@ class WalousLandCoverService:
|
||||
source_sha256_path = source_path.with_name(product.source_sha256_filename)
|
||||
source_sha256 = source_sha256_path.read_text(encoding="ascii").strip().split()[0] if source_sha256_path.is_file() else None
|
||||
acquired_at = datetime.now(UTC)
|
||||
observed_at = datetime(product.observation_year, 12, 31, 23, 59, 59, tzinfo=UTC)
|
||||
observed_at = product.observation_end
|
||||
spatial_series_hash = hashlib.sha256(json.dumps({"bbox": identity["bbox_epsg4326"], "area_id": identity["area_id"], "resolution": identity["analysis_resolution_m"]}, sort_keys=True).encode()).hexdigest()[:24]
|
||||
dataset = DatasetService.import_raster_bytes(
|
||||
db,
|
||||
@@ -357,8 +370,8 @@ class WalousLandCoverService:
|
||||
source_name=WalousLandCoverService.PROVIDER,
|
||||
temporal_series_key=f"spw:walous:land-cover:{spatial_series_hash}",
|
||||
observed_at=observed_at,
|
||||
valid_from=datetime(product.observation_year, 1, 1, tzinfo=UTC),
|
||||
valid_to=observed_at,
|
||||
valid_from=product.observation_start,
|
||||
valid_to=product.observation_end,
|
||||
temporal_granularity="year",
|
||||
source_version=product.source_version,
|
||||
source_metadata={
|
||||
@@ -374,6 +387,8 @@ class WalousLandCoverService:
|
||||
"source_value_unit": WalousLandCoverService.SOURCE_VALUE_UNIT,
|
||||
"class_labels": WalousLandCoverService.CLASS_LABELS,
|
||||
"observation_year": product.observation_year,
|
||||
"observation_start": product.observation_start.isoformat(),
|
||||
"observation_end": product.observation_end.isoformat(),
|
||||
"valid_pixel_count": validation["valid_pixel_count"],
|
||||
"classes_present": validation["classes_present"],
|
||||
"bbox_epsg4326": bbox_4326,
|
||||
@@ -480,12 +495,12 @@ class WalousLandCoverService:
|
||||
|
||||
metric_specs = [
|
||||
("land_cover_observed_area_ha", "Gekarteerde landbedekking", set(WalousLandCoverService.CLASS_LABELS)),
|
||||
("forest_cover_area_ha", "Boom- en bosbedekking", {3, 4, 5, 6}),
|
||||
("surface_water_area_ha", "Oppervlaktewater", {8}),
|
||||
("artificial_cover_area_ha", "Kunstmatige bedekking en constructies", {9, 10, 11}),
|
||||
("annual_herbaceous_cover_area_ha", "Jaarlijks wisselende kruidlaag", {1}),
|
||||
("permanent_herbaceous_cover_area_ha", "Jaarronde kruidlaag", {2}),
|
||||
("bare_soil_area_ha", "Kale bodem", {7}),
|
||||
("forest_cover_area_ha", "Boom- en bosbedekking", {8, 9, 80, 90}),
|
||||
("surface_water_area_ha", "Oppervlaktewater", {5}),
|
||||
("artificial_cover_area_ha", "Kunstmatige bedekking en constructies", {1, 2, 3}),
|
||||
("annual_herbaceous_cover_area_ha", "Jaarlijks wisselende kruidlaag", {6}),
|
||||
("permanent_herbaceous_cover_area_ha", "Jaarronde kruidlaag", {7}),
|
||||
("bare_soil_area_ha", "Kale bodem", {4}),
|
||||
]
|
||||
metrics = [
|
||||
ThematicRasterMetric(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
@@ -22,6 +23,15 @@ from app.services.temporal_analysis_service import TemporalAnalysisService
|
||||
from app.services.walous_land_cover_service import WalousLandCoverService
|
||||
|
||||
|
||||
def load_provisioner():
|
||||
path = Path(__file__).resolve().parents[2] / "scripts" / "provision_walous_sources.py"
|
||||
spec = importlib.util.spec_from_file_location("walous_source_provisioner_test", path)
|
||||
assert spec and spec.loader
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
@@ -70,16 +80,15 @@ def make_source(path: Path) -> tuple[list[float], np.ndarray]:
|
||||
to_4326 = Transformer.from_crs("EPSG:3812", "EPSG:4326", always_xy=True)
|
||||
x, y = to_3812.transform(4.85, 50.45)
|
||||
transform = from_origin(x, y + 100, 1, 1)
|
||||
values = np.ones((100, 100), dtype="uint8")
|
||||
values[:, 20:40] = 4
|
||||
values[:, 40:50] = 8
|
||||
values[:, 50:70] = 9
|
||||
values[:, 70:] = 2
|
||||
class_codes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90]
|
||||
values = np.empty((100, len(class_codes) * 20), dtype="uint8")
|
||||
for index, class_code in enumerate(class_codes):
|
||||
values[:, index * 20 : (index + 1) * 20] = class_code
|
||||
with rasterio.open(
|
||||
path,
|
||||
"w",
|
||||
driver="GTiff",
|
||||
width=100,
|
||||
width=values.shape[1],
|
||||
height=100,
|
||||
count=1,
|
||||
dtype="uint8",
|
||||
@@ -89,7 +98,7 @@ def make_source(path: Path) -> tuple[list[float], np.ndarray]:
|
||||
) as target:
|
||||
target.write(values, 1)
|
||||
min_lon, min_lat = to_4326.transform(x, y)
|
||||
max_lon, max_lat = to_4326.transform(x + 100, y + 100)
|
||||
max_lon, max_lat = to_4326.transform(x + values.shape[1], y + values.shape[0])
|
||||
return [min_lon, min_lat, max_lon, max_lat], values
|
||||
|
||||
|
||||
@@ -113,6 +122,17 @@ def test_walous_registry_reports_real_provisioning_state(tmp_path: Path) -> None
|
||||
assert after["walous_land_cover_2023"]["native_resolution_m"] == 1.0
|
||||
assert after["walous_land_cover_2023"]["analysis_resolution_m"] == 10.0
|
||||
assert after["walous_land_cover_2023"]["coverage_zones"] == ["wallonia"]
|
||||
assert after["walous_land_cover_2023"]["included_source_values"] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90]
|
||||
assert after["walous_land_cover_2023"]["source_value_unit"] == "walous_class_code"
|
||||
|
||||
|
||||
def test_walous_provisioner_accepts_official_non_contiguous_class_codes(tmp_path: Path) -> None:
|
||||
source_path = tmp_path / "walous_land_cover_2023_3812.tif"
|
||||
make_source(source_path)
|
||||
|
||||
validation = load_provisioner().validate_raster(source_path)
|
||||
|
||||
assert validation["sample_classes"] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90]
|
||||
|
||||
|
||||
def test_walous_acquisition_reads_real_classes_and_persists_provenance(tmp_path: Path, monkeypatch) -> None:
|
||||
@@ -141,10 +161,12 @@ def test_walous_acquisition_reads_real_classes_and_persists_provenance(tmp_path:
|
||||
assert result["output_dataset_id"] == str(output_id)
|
||||
assert result["resolution_m"] == 10
|
||||
assert captured["source_name"] == "spw_walous_land_cover"
|
||||
assert captured["source_metadata"]["classes_present"] == [1, 2, 4, 8, 9]
|
||||
assert captured["source_metadata"]["classes_present"] == [1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90]
|
||||
assert captured["provenance_metadata"]["resampling"] == "nearest"
|
||||
assert captured["temporal_series_key"].startswith("spw:walous:land-cover:")
|
||||
assert captured["observed_at"].year == 2023
|
||||
assert captured["observed_at"].date().isoformat() == "2023-06-25"
|
||||
assert captured["valid_from"].date().isoformat() == "2023-05-27"
|
||||
assert captured["valid_to"] == captured["observed_at"]
|
||||
|
||||
|
||||
def test_walous_analysis_returns_semantic_area_metrics(tmp_path: Path, monkeypatch) -> None:
|
||||
@@ -193,6 +215,9 @@ def test_walous_analysis_returns_semantic_area_metrics(tmp_path: Path, monkeypat
|
||||
assert metrics["forest_cover_area_ha"] > 0
|
||||
assert metrics["surface_water_area_ha"] > 0
|
||||
assert metrics["artificial_cover_area_ha"] > 0
|
||||
assert metrics["annual_herbaceous_cover_area_ha"] > 0
|
||||
assert metrics["permanent_herbaceous_cover_area_ha"] > 0
|
||||
assert metrics["bare_soil_area_ha"] > 0
|
||||
assert "water_volume" in result["unsupported_metrics"]
|
||||
|
||||
|
||||
|
||||
@@ -494,10 +494,12 @@ becomes true only when the checksum-validated source GeoTIFF exists below
|
||||
|
||||
Reads a bounded window from one provisioned official 1 m WALOUS source,
|
||||
applies nearest-neighbour resampling to the configured analysis resolution,
|
||||
masks `bbox intersect Area`, validates class values 1-11 and persists a normal
|
||||
raster Dataset through `DatasetService`. URLs, paths, classes and resolutions
|
||||
are not caller-controlled. Equal spatial requests for 2020 and 2023 share one
|
||||
temporal series key.
|
||||
masks `bbox intersect Area`, validates the official non-contiguous class-code
|
||||
set `1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90` and persists a normal raster Dataset
|
||||
through `DatasetService`. URLs, paths, classes and resolutions are not
|
||||
caller-controlled. Equal spatial requests for 2020 and 2023 share one temporal
|
||||
series key. The exact official observation ranges are retained as
|
||||
2020-04-01/2020-04-24 and 2023-05-27/2023-06-25.
|
||||
|
||||
### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/walous/select`
|
||||
|
||||
|
||||
@@ -3654,6 +3654,12 @@ Validation:
|
||||
|
||||
## Post-V1 national coverage completion: Wallonia (2026-07-22)
|
||||
|
||||
- Live Tower provisioning exposed an incorrect assumption that 11 WALOUS
|
||||
classes implied numeric codes 1 through 11. The official SPW legend and the
|
||||
downloaded 2020 raster confirm codes `1,2,3,4,5,6,7,8,9,80,90`. Corrected
|
||||
validation, class semantics, colours and all hectare aggregations; added a
|
||||
provisioner regression containing codes 80/90 and retained exact source
|
||||
observation ranges.
|
||||
- Validated the official WALOUS 2020 and 2023 archives, their EPSG:3812 1 m
|
||||
raster contract, 11 classes, CC BY 4.0 attribution and published edition
|
||||
accuracy. Added a fail-closed operator provisioner with archive-size,
|
||||
|
||||
@@ -790,6 +790,27 @@ Areas. They persist only through `DatasetService`; the browser never contacts
|
||||
either provider directly. Cross-zone selections remain split by authority and
|
||||
metric semantics.
|
||||
|
||||
## Wallonia WALOUS land cover
|
||||
|
||||
The official SPW `WAL_OCS_IA__2020` and `WAL_OCS_IA__2023` GeoTIFF archives
|
||||
provide comparable Walloon land-cover observations at native 1 m resolution in
|
||||
EPSG:3812. Runtime analysis reads only bounded windows from operator-
|
||||
provisioned, checksum-recorded source files and uses nearest-neighbour
|
||||
resampling for the governed 10 m analysis derivative.
|
||||
|
||||
The 11 semantic classes use the non-contiguous source codes `1, 2, 3, 4, 5, 6,
|
||||
7, 8, 9, 80, 90`. In order these mean artificial ground, above-ground
|
||||
construction, railway, bare soil, surface water, rotating herbaceous cover,
|
||||
continuous herbaceous cover, conifer trees above 3 m, deciduous trees above 3
|
||||
m, conifer woody cover up to 3 m and deciduous woody cover up to 3 m. Codes 80
|
||||
and 90 must never be normalized to invented classes 10 and 11.
|
||||
|
||||
The official temporal extents are 2020-04-01 through 2020-04-24 and 2023-05-27
|
||||
through 2023-06-25. Metrics are estimated hectares from classified cells. They
|
||||
are not legal land use, ownership, individual tree counts, timber volume or
|
||||
water volume. The official catalogue reports overall accuracy per edition and
|
||||
also warns that accuracy varies by class and place.
|
||||
|
||||
## Bathymetry, inland profiles and maritime scope
|
||||
|
||||
The official VHA Digital Atlas profile-point layer is the first operational
|
||||
|
||||
@@ -308,6 +308,15 @@ review hashes, persists sampled official flight dates as the temporal evidence
|
||||
range and creates a new Dataset/DatasetVersion through DatasetService. It does
|
||||
not retroactively rewrite or delete legacy rows.
|
||||
|
||||
WALOUS land-cover rasters retain EPSG:3812, their native 1 m source resolution,
|
||||
the derived analysis resolution, source checksum and exact observation range.
|
||||
The governed class domain is `{1,2,3,4,5,6,7,8,9,80,90}`. Codes 80 and 90 are
|
||||
valid official low-woody-cover classes; `10` and `11` are not substitutes.
|
||||
Bounded derivatives use `uint8`, nodata 255 and nearest-neighbour resampling.
|
||||
Area metrics group forest/tree cover as `{8,9,80,90}`, water as `{5}`,
|
||||
artificial cover as `{1,2,3}`, rotating herbaceous cover as `{6}`, continuous
|
||||
herbaceous cover as `{7}` and bare soil as `{4}`.
|
||||
|
||||
### Hydrological station observations
|
||||
|
||||
Waterinfo observations are persisted as EPSG:4326 Point features, one station
|
||||
|
||||
@@ -33,6 +33,7 @@ SOURCES = {
|
||||
}
|
||||
MAX_ARCHIVE_BYTES = 1_000_000_000
|
||||
MAX_EXTRACTED_BYTES = 50_000_000_000
|
||||
WALOUS_CLASS_CODES = {1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90}
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
@@ -110,9 +111,9 @@ def validate_raster(path: Path) -> dict:
|
||||
sample_width = min(2048, source.width)
|
||||
sample = source.read(1, out_shape=(sample_height, sample_width), masked=True, resampling=Resampling.nearest)
|
||||
values = np.unique(sample.compressed()).astype(int).tolist()
|
||||
unexpected = sorted(set(values) - set(range(1, 12)))
|
||||
unexpected = sorted(set(values) - WALOUS_CLASS_CODES)
|
||||
if unexpected:
|
||||
raise RuntimeError(f"WALOUS sample contains classes outside 1-11: {unexpected}")
|
||||
raise RuntimeError(f"WALOUS sample contains classes outside the official 11-class code set: {unexpected}")
|
||||
return {
|
||||
"path": str(path),
|
||||
"crs": str(source.crs),
|
||||
|
||||
Reference in New Issue
Block a user