feat: complete Wallonia land cover and terrain sources

scripts/README.md

@@ -4,22 +4,39 @@ Setup-, import-, demo- en maintenance-scripts voor GeoIntel.
 
 ## WALOUS source provisioning
 
-Run the networked operator only after checking at least 2 GB of archive space
+Run the networked operator only after checking at least 3 GB of archive space
 plus room for the extracted official GeoTIFFs:
 
 ```bash
 python scripts/provision_walous_sources.py \
-  --years 2020 2023 \
+  --years 2018 2020 2023 \
   --destination storage/source-cache/walous
 ```
 
-The command accepts only the hard-coded official SPW 2020/2023 archives,
-streams with a 1 GB per-archive cap, rejects changed content lengths, extracts
+The command accepts only the hard-coded official SPW 2018/2020/2023 archives,
+streams with a 1.25 GB per-archive cap, rejects changed content lengths, extracts
 only the single GeoTIFF by basename, validates the raster contract and writes
 checksums plus `provisioning-report.json`. Existing valid sources are reused;
 `--force` performs a new download. This is an operator acquisition, not an
 application startup task.
 
+## SPW Wallonia terrain source provisioning
+
+The official 1 m MNT is a large operator asset, never an implicit startup
+download. Reserve at least 90 GB temporarily for archive plus extraction and
+run:
+
+```bash
+python scripts/provision_spw_terrain_source.py \
+  --destination storage/source-cache/spw-terrain
+```
+
+The provisioner accepts only the fixed official SPW artifact, enforces a
+30-60 GB archive range and a single safe GeoTIFF member, validates EPSG:3812,
+one band, native 1 m cells and representative elevation samples, writes
+source/archive SHA-256 evidence and removes the archive after successful
+extraction unless `--keep-archive` is supplied.
+
 ## Runtime verification
 
 Inspect interrupted runtime state without changing it:

scripts/provision_spw_terrain_source.py

@@ -0,0 +1,242 @@
+#!/usr/bin/env python3
+"""Provision the official Wallonia 1 m MNT for bounded GeoIntel analysis."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import sys
+from urllib.request import Request, urlopen
+from zipfile import BadZipFile, ZipFile
+
+
+SOURCE_URL = (
+    "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
+    "fe13bc84-e371-46ca-9632-8ad4139f1ee5/RELIEF_WALLONIE_MNT_1M_2021_2022_GEOTIFF_3812.zip"
+)
+TARGET_FILENAME = "spw_mnt_1m_2021_2022_3812.tif"
+ARCHIVE_FILENAME = "spw_mnt_1m_2021_2022_3812.zip"
+MIN_ARCHIVE_BYTES = 30_000_000_000
+MAX_ARCHIVE_BYTES = 60_000_000_000
+MAX_EXTRACTED_BYTES = 80_000_000_000
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        while chunk := handle.read(16 * 1024 * 1024):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def download(destination: Path) -> str:
+    temporary = destination.with_suffix(destination.suffix + ".part")
+    temporary.unlink(missing_ok=True)
+    digest = hashlib.sha256()
+    received = 0
+    request = Request(
+        SOURCE_URL,
+        headers={"User-Agent": "GeoIntel/1.0 SPW-terrain-source-provisioner"},
+    )
+    try:
+        with urlopen(request, timeout=300) as response, temporary.open("wb") as output:
+            content_length = int(response.headers.get("Content-Length") or 0)
+            if (
+                content_length
+                and not MIN_ARCHIVE_BYTES <= content_length <= MAX_ARCHIVE_BYTES
+            ):
+                raise RuntimeError(
+                    f"official archive size is outside the governed range: {content_length}"
+                )
+            while chunk := response.read(16 * 1024 * 1024):
+                received += len(chunk)
+                if received > MAX_ARCHIVE_BYTES:
+                    raise RuntimeError(
+                        "official archive exceeds the governed 60 GB transfer limit"
+                    )
+                digest.update(chunk)
+                output.write(chunk)
+                if received % (1024 * 1024 * 1024) < len(chunk):
+                    print(
+                        f"  downloaded {received / 1024 / 1024 / 1024:.1f} GiB",
+                        flush=True,
+                    )
+        if received < MIN_ARCHIVE_BYTES:
+            raise RuntimeError(
+                f"official archive is unexpectedly small: {received} bytes"
+            )
+        temporary.replace(destination)
+        return digest.hexdigest()
+    except Exception:
+        temporary.unlink(missing_ok=True)
+        raise
+
+
+def extract_single_geotiff(archive: Path, target: Path) -> None:
+    try:
+        with ZipFile(archive) as bundle:
+            candidates = [
+                item
+                for item in bundle.infolist()
+                if not item.is_dir()
+                and item.filename.lower().endswith((".tif", ".tiff"))
+            ]
+            if len(candidates) != 1:
+                raise RuntimeError(
+                    f"archive must contain exactly one GeoTIFF, found {len(candidates)}"
+                )
+            member = candidates[0]
+            if member.file_size <= 0 or member.file_size > MAX_EXTRACTED_BYTES:
+                raise RuntimeError(
+                    f"GeoTIFF uncompressed size is outside the governed limit: {member.file_size}"
+                )
+            temporary = target.with_suffix(target.suffix + ".part")
+            temporary.unlink(missing_ok=True)
+            with bundle.open(member) as source, temporary.open("wb") as output:
+                shutil.copyfileobj(source, output, length=16 * 1024 * 1024)
+            temporary.replace(target)
+    except BadZipFile as exc:
+        raise RuntimeError("official SPW MNT archive is not a valid ZIP file") from exc
+
+
+def validate_raster(path: Path) -> dict:
+    try:
+        import numpy as np
+        import rasterio
+        from rasterio.windows import Window
+    except ImportError as exc:
+        raise RuntimeError(
+            "rasterio and numpy are required to validate the SPW MNT source"
+        ) from exc
+    with rasterio.open(path) as source:
+        if source.crs is None or source.crs.to_epsg() != 3812:
+            raise RuntimeError(f"SPW MNT must use EPSG:3812, found {source.crs}")
+        if source.count != 1:
+            raise RuntimeError(f"SPW MNT must have one band, found {source.count}")
+        if not all(abs(abs(float(value)) - 1.0) <= 0.05 for value in source.res):
+            raise RuntimeError(f"SPW MNT must retain 1 m cells, found {source.res}")
+        sample_windows = []
+        sample_size = 512
+        for x_fraction, y_fraction in (
+            (0.1, 0.1),
+            (0.5, 0.5),
+            (0.9, 0.9),
+            (0.1, 0.9),
+            (0.9, 0.1),
+        ):
+            col = max(
+                0,
+                min(
+                    source.width - sample_size,
+                    round(source.width * x_fraction - sample_size / 2),
+                ),
+            )
+            row = max(
+                0,
+                min(
+                    source.height - sample_size,
+                    round(source.height * y_fraction - sample_size / 2),
+                ),
+            )
+            sample_windows.append(
+                Window(
+                    col,
+                    row,
+                    min(sample_size, source.width),
+                    min(sample_size, source.height),
+                )
+            )
+        samples = [
+            source.read(1, window=window, masked=True).compressed().astype("float64")
+            for window in sample_windows
+        ]
+        values = np.concatenate([sample for sample in samples if sample.size])
+        values = values[np.isfinite(values)]
+        if not values.size:
+            raise RuntimeError(
+                "SPW MNT validation samples contain no finite elevation values"
+            )
+        if float(values.min()) < -100.0 or float(values.max()) > 1000.0:
+            raise RuntimeError(
+                f"SPW MNT samples contain implausible values: {values.min()}..{values.max()}"
+            )
+        return {
+            "path": str(path),
+            "crs": str(source.crs),
+            "width": int(source.width),
+            "height": int(source.height),
+            "resolution": [float(value) for value in source.res],
+            "bounds": [float(value) for value in source.bounds],
+            "nodata": None if source.nodata is None else float(source.nodata),
+            "dtype": source.dtypes[0],
+            "sample_min_m": float(values.min()),
+            "sample_max_m": float(values.max()),
+        }
+
+
+def provision(destination: Path, force: bool, keep_archive: bool) -> dict:
+    destination.mkdir(parents=True, exist_ok=True)
+    target = destination / TARGET_FILENAME
+    archive = destination / ARCHIVE_FILENAME
+    archive_digest = None
+    if target.is_file() and not force:
+        print(f"SPW MNT: validating existing source {target}")
+    else:
+        if archive.is_file():
+            archive_size = archive.stat().st_size
+            if not MIN_ARCHIVE_BYTES <= archive_size <= MAX_ARCHIVE_BYTES:
+                raise RuntimeError(
+                    f"existing archive size is outside the governed range: {archive_size}"
+                )
+            print(f"SPW MNT: using existing archive {archive}")
+            archive_digest = sha256_file(archive)
+        else:
+            print("SPW MNT: downloading official archive")
+            archive_digest = download(archive)
+        print(f"SPW MNT: archive sha256 {archive_digest}")
+        extract_single_geotiff(archive, target)
+        if not keep_archive:
+            archive.unlink(missing_ok=True)
+    validation = validate_raster(target)
+    source_digest = sha256_file(target)
+    target.with_suffix(".sha256").write_text(
+        f"{source_digest}  {target.name}\n", encoding="ascii"
+    )
+    validation.update(
+        {
+            "source_sha256": source_digest,
+            "archive_sha256": archive_digest,
+            "download_url": SOURCE_URL,
+            "catalog_url": "https://geoportail.wallonie.be/catalogue/fe13bc84-e371-46ca-9632-8ad4139f1ee5.html",
+        }
+    )
+    report_path = destination / "provisioning-report.json"
+    report_path.write_text(json.dumps(validation, indent=2) + "\n", encoding="utf-8")
+    print(f"SPW MNT: ready ({target.stat().st_size / 1024 / 1024 / 1024:.1f} GiB)")
+    print(f"Provisioning report: {report_path}")
+    return validation
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Provision the official Wallonia 1 m MNT 2021-2022 GeoTIFF."
+    )
+    parser.add_argument(
+        "--destination", type=Path, default=Path("storage/source-cache/spw-terrain")
+    )
+    parser.add_argument("--force", action="store_true")
+    parser.add_argument("--keep-archive", action="store_true")
+    args = parser.parse_args()
+    provision(args.destination.resolve(), args.force, args.keep_archive)
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception as exc:
+        print(f"SPW_TERRAIN_PROVISIONING_FAILED: {exc}", file=sys.stderr)
+        raise SystemExit(1) from exc

scripts/provision_walous_sources.py

@@ -14,6 +14,14 @@ from zipfile import BadZipFile, ZipFile
 
 
 SOURCES = {
+    2018: {
+        "url": (
+            "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
+            "a0ad23a1-1845-4bd5-8c2f-0f62d3f1ec75/WALOUS_OCS__2018_GEOTIFF_3812.zip"
+        ),
+        "expected_archive_bytes": 1_122_785_133,
+        "target": "walous_land_cover_2018_3812.tif",
+    },
     2020: {
         "url": (
             "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
@@ -31,9 +39,46 @@ SOURCES = {
         "target": "walous_land_cover_2023_3812.tif",
     },
 }
-MAX_ARCHIVE_BYTES = 1_000_000_000
+MAX_ARCHIVE_BYTES = 1_250_000_000
 MAX_EXTRACTED_BYTES = 50_000_000_000
-WALOUS_CLASS_CODES = {1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90}
+WALOUS_CLASS_CODES = {
+    0,
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    11,
+    15,
+    18,
+    19,
+    28,
+    29,
+    31,
+    38,
+    39,
+    51,
+    55,
+    58,
+    59,
+    62,
+    71,
+    73,
+    75,
+    80,
+    81,
+    83,
+    85,
+    90,
+    91,
+    93,
+    95,
+}
+WALOUS_CANONICAL_CLASS_CODES = {1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90}
 
 
 def sha256_file(path: Path) -> str:
@@ -49,22 +94,30 @@ def download(url: str, destination: Path, expected_bytes: int) -> str:
     temporary.unlink(missing_ok=True)
     digest = hashlib.sha256()
     received = 0
-    request = Request(url, headers={"User-Agent": "GeoIntel/1.0 WALOUS-source-provisioner"})
+    request = Request(
+        url, headers={"User-Agent": "GeoIntel/1.0 WALOUS-source-provisioner"}
+    )
     try:
         with urlopen(request, timeout=300) as response, temporary.open("wb") as output:
             content_length = int(response.headers.get("Content-Length") or 0)
             if content_length and content_length != expected_bytes:
-                raise RuntimeError(f"official archive size changed: expected {expected_bytes}, advertised {content_length}")
+                raise RuntimeError(
+                    f"official archive size changed: expected {expected_bytes}, advertised {content_length}"
+                )
             while chunk := response.read(8 * 1024 * 1024):
                 received += len(chunk)
                 if received > MAX_ARCHIVE_BYTES:
-                    raise RuntimeError("official archive exceeds the governed 1 GB transfer limit")
+                    raise RuntimeError(
+                        "official archive exceeds the governed 1 GB transfer limit"
+                    )
                 digest.update(chunk)
                 output.write(chunk)
                 if received % (128 * 1024 * 1024) < len(chunk):
                     print(f"  downloaded {received / 1024 / 1024:.0f} MiB", flush=True)
         if received != expected_bytes:
-            raise RuntimeError(f"archive is incomplete: expected {expected_bytes} bytes, received {received}")
+            raise RuntimeError(
+                f"archive is incomplete: expected {expected_bytes} bytes, received {received}"
+            )
         temporary.replace(destination)
         return digest.hexdigest()
     except Exception:
@@ -75,13 +128,25 @@ def download(url: str, destination: Path, expected_bytes: int) -> str:
 def extract_single_geotiff(archive: Path, target: Path) -> None:
     try:
         with ZipFile(archive) as bundle:
-            candidates = [item for item in bundle.infolist() if not item.is_dir() and item.filename.lower().endswith((".tif", ".tiff"))]
+            candidates = [
+                item
+                for item in bundle.infolist()
+                if not item.is_dir()
+                and item.filename.lower().endswith((".tif", ".tiff"))
+            ]
             if len(candidates) != 1:
-                raise RuntimeError(f"archive must contain exactly one GeoTIFF, found {len(candidates)}")
+                raise RuntimeError(
+                    f"archive must contain exactly one GeoTIFF, found {len(candidates)}"
+                )
             member = candidates[0]
             if member.file_size <= 0 or member.file_size > MAX_EXTRACTED_BYTES:
-                raise RuntimeError(f"GeoTIFF uncompressed size is outside the governed limit: {member.file_size}")
-            if Path(member.filename).name != member.filename.replace("\\", "/").split("/")[-1]:
+                raise RuntimeError(
+                    f"GeoTIFF uncompressed size is outside the governed limit: {member.file_size}"
+                )
+            if (
+                Path(member.filename).name
+                != member.filename.replace("\\", "/").split("/")[-1]
+            ):
                 # Nested paths are accepted only by basename; extraction never trusts archive paths.
                 pass
             temporary = target.with_suffix(target.suffix + ".part")
@@ -99,21 +164,37 @@ def validate_raster(path: Path) -> dict:
         import rasterio
         from rasterio.enums import Resampling
     except ImportError as exc:
-        raise RuntimeError("rasterio and numpy are required to validate WALOUS sources") from exc
+        raise RuntimeError(
+            "rasterio and numpy are required to validate WALOUS sources"
+        ) from exc
     with rasterio.open(path) as source:
         if source.crs is None or source.crs.to_epsg() != 3812:
             raise RuntimeError(f"WALOUS raster must use EPSG:3812, found {source.crs}")
         if source.count != 1:
-            raise RuntimeError(f"WALOUS raster must have one band, found {source.count}")
+            raise RuntimeError(
+                f"WALOUS raster must have one band, found {source.count}"
+            )
         if not all(abs(abs(float(value)) - 1.0) <= 0.05 for value in source.res):
-            raise RuntimeError(f"WALOUS raster must retain 1 m cells, found {source.res}")
+            raise RuntimeError(
+                f"WALOUS raster must retain 1 m cells, found {source.res}"
+            )
         sample_height = min(2048, source.height)
         sample_width = min(2048, source.width)
-        sample = source.read(1, out_shape=(sample_height, sample_width), masked=True, resampling=Resampling.nearest)
+        sample = source.read(
+            1,
+            out_shape=(sample_height, sample_width),
+            masked=True,
+            resampling=Resampling.nearest,
+        )
         values = np.unique(sample.compressed()).astype(int).tolist()
-        unexpected = sorted(set(values) - WALOUS_CLASS_CODES)
+        allowed_codes = (
+            WALOUS_CLASS_CODES if "2018" in path.name else WALOUS_CANONICAL_CLASS_CODES
+        )
+        unexpected = sorted(set(values) - allowed_codes)
         if unexpected:
-            raise RuntimeError(f"WALOUS sample contains classes outside the official 11-class code set: {unexpected}")
+            raise RuntimeError(
+                f"WALOUS sample contains classes outside the official 11-class code set: {unexpected}"
+            )
         return {
             "path": str(path),
             "crs": str(source.crs),
@@ -123,6 +204,9 @@ def validate_raster(path: Path) -> dict:
             "bounds": [float(value) for value in source.bounds],
             "nodata": None if source.nodata is None else float(source.nodata),
             "sample_classes": values,
+            "implicit_source_nodata_values": [0]
+            if "2018" in path.name and 0 in values
+            else [],
         }
 
 
@@ -135,8 +219,17 @@ def provision(year: int, destination: Path, force: bool) -> dict:
         validation = validate_raster(target)
         digest = sha256_file(target)
     else:
-        print(f"WALOUS {year}: downloading official archive")
-        archive_digest = download(source["url"], archive, source["expected_archive_bytes"])
+        if (
+            archive.is_file()
+            and archive.stat().st_size == source["expected_archive_bytes"]
+        ):
+            print(f"WALOUS {year}: using existing official archive {archive}")
+            archive_digest = sha256_file(archive)
+        else:
+            print(f"WALOUS {year}: downloading official archive")
+            archive_digest = download(
+                source["url"], archive, source["expected_archive_bytes"]
+            )
         print(f"WALOUS {year}: archive sha256 {archive_digest}")
         extract_single_geotiff(archive, target)
         validation = validate_raster(target)
@@ -150,15 +243,40 @@ def provision(year: int, destination: Path, force: bool) -> dict:
 
 
 def main() -> int:
-    parser = argparse.ArgumentParser(description="Provision official WALOUS 2020/2023 GeoTIFF sources.")
-    parser.add_argument("--years", nargs="+", type=int, choices=sorted(SOURCES), default=sorted(SOURCES))
-    parser.add_argument("--destination", type=Path, default=Path("storage/source-cache/walous"))
+    parser = argparse.ArgumentParser(
+        description="Provision official WALOUS 2018/2020/2023 GeoTIFF sources."
+    )
+    parser.add_argument(
+        "--years", nargs="+", type=int, choices=sorted(SOURCES), default=sorted(SOURCES)
+    )
+    parser.add_argument(
+        "--destination", type=Path, default=Path("storage/source-cache/walous")
+    )
     parser.add_argument("--force", action="store_true")
     args = parser.parse_args()
     args.destination.mkdir(parents=True, exist_ok=True)
-    report = [provision(year, args.destination.resolve(), args.force) for year in args.years]
+    report = [
+        provision(year, args.destination.resolve(), args.force) for year in args.years
+    ]
     report_path = args.destination / "provisioning-report.json"
-    report_path.write_text(json.dumps({"sources": report}, indent=2) + "\n", encoding="utf-8")
+    retained: dict[int, dict] = {}
+    if report_path.is_file():
+        try:
+            retained = {
+                int(item["year"]): item
+                for item in json.loads(report_path.read_text(encoding="utf-8")).get(
+                    "sources", []
+                )
+                if isinstance(item, dict) and item.get("year") in SOURCES
+            }
+        except (OSError, ValueError, TypeError):
+            retained = {}
+    retained.update({int(item["year"]): item for item in report})
+    report_path.write_text(
+        json.dumps({"sources": [retained[year] for year in sorted(retained)]}, indent=2)
+        + "\n",
+        encoding="utf-8",
+    )
     print(f"Provisioning report: {report_path}")
     return 0