feat: complete governed Walloon coverage sources

scripts/provision_walous_sources.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""Provision official WALOUS GeoTIFF source rasters for bounded runtime analysis."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import sys
+from urllib.request import Request, urlopen
+from zipfile import BadZipFile, ZipFile
+
+
+SOURCES = {
+    2020: {
+        "url": (
+            "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
+            "47b348f1-6e7a-4baa-963c-0232a43c0cff/WAL_OCS_IA__2020_GEOTIFF_3812.zip"
+        ),
+        "expected_archive_bytes": 728_244_755,
+        "target": "walous_land_cover_2020_3812.tif",
+    },
+    2023: {
+        "url": (
+            "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
+            "4e780ba1-463c-478e-95df-d2f1963a150d/WAL_OCS_IA__2023_GEOTIFF_3812.zip"
+        ),
+        "expected_archive_bytes": 876_014_572,
+        "target": "walous_land_cover_2023_3812.tif",
+    },
+}
+MAX_ARCHIVE_BYTES = 1_000_000_000
+MAX_EXTRACTED_BYTES = 50_000_000_000
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        while chunk := handle.read(8 * 1024 * 1024):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def download(url: str, destination: Path, expected_bytes: int) -> str:
+    temporary = destination.with_suffix(destination.suffix + ".part")
+    temporary.unlink(missing_ok=True)
+    digest = hashlib.sha256()
+    received = 0
+    request = Request(url, headers={"User-Agent": "GeoIntel/1.0 WALOUS-source-provisioner"})
+    try:
+        with urlopen(request, timeout=300) as response, temporary.open("wb") as output:
+            content_length = int(response.headers.get("Content-Length") or 0)
+            if content_length and content_length != expected_bytes:
+                raise RuntimeError(f"official archive size changed: expected {expected_bytes}, advertised {content_length}")
+            while chunk := response.read(8 * 1024 * 1024):
+                received += len(chunk)
+                if received > MAX_ARCHIVE_BYTES:
+                    raise RuntimeError("official archive exceeds the governed 1 GB transfer limit")
+                digest.update(chunk)
+                output.write(chunk)
+                if received % (128 * 1024 * 1024) < len(chunk):
+                    print(f"  downloaded {received / 1024 / 1024:.0f} MiB", flush=True)
+        if received != expected_bytes:
+            raise RuntimeError(f"archive is incomplete: expected {expected_bytes} bytes, received {received}")
+        temporary.replace(destination)
+        return digest.hexdigest()
+    except Exception:
+        temporary.unlink(missing_ok=True)
+        raise
+
+
+def extract_single_geotiff(archive: Path, target: Path) -> None:
+    try:
+        with ZipFile(archive) as bundle:
+            candidates = [item for item in bundle.infolist() if not item.is_dir() and item.filename.lower().endswith((".tif", ".tiff"))]
+            if len(candidates) != 1:
+                raise RuntimeError(f"archive must contain exactly one GeoTIFF, found {len(candidates)}")
+            member = candidates[0]
+            if member.file_size <= 0 or member.file_size > MAX_EXTRACTED_BYTES:
+                raise RuntimeError(f"GeoTIFF uncompressed size is outside the governed limit: {member.file_size}")
+            if Path(member.filename).name != member.filename.replace("\\", "/").split("/")[-1]:
+                # Nested paths are accepted only by basename; extraction never trusts archive paths.
+                pass
+            temporary = target.with_suffix(target.suffix + ".part")
+            temporary.unlink(missing_ok=True)
+            with bundle.open(member) as source, temporary.open("wb") as output:
+                shutil.copyfileobj(source, output, length=8 * 1024 * 1024)
+            temporary.replace(target)
+    except BadZipFile as exc:
+        raise RuntimeError("official WALOUS archive is not a valid ZIP file") from exc
+
+
+def validate_raster(path: Path) -> dict:
+    try:
+        import numpy as np
+        import rasterio
+        from rasterio.enums import Resampling
+    except ImportError as exc:
+        raise RuntimeError("rasterio and numpy are required to validate WALOUS sources") from exc
+    with rasterio.open(path) as source:
+        if source.crs is None or source.crs.to_epsg() != 3812:
+            raise RuntimeError(f"WALOUS raster must use EPSG:3812, found {source.crs}")
+        if source.count != 1:
+            raise RuntimeError(f"WALOUS raster must have one band, found {source.count}")
+        if not all(abs(abs(float(value)) - 1.0) <= 0.05 for value in source.res):
+            raise RuntimeError(f"WALOUS raster must retain 1 m cells, found {source.res}")
+        sample_height = min(2048, source.height)
+        sample_width = min(2048, source.width)
+        sample = source.read(1, out_shape=(sample_height, sample_width), masked=True, resampling=Resampling.nearest)
+        values = np.unique(sample.compressed()).astype(int).tolist()
+        unexpected = sorted(set(values) - set(range(1, 12)))
+        if unexpected:
+            raise RuntimeError(f"WALOUS sample contains classes outside 1-11: {unexpected}")
+        return {
+            "path": str(path),
+            "crs": str(source.crs),
+            "width": int(source.width),
+            "height": int(source.height),
+            "resolution": [float(value) for value in source.res],
+            "bounds": [float(value) for value in source.bounds],
+            "nodata": None if source.nodata is None else float(source.nodata),
+            "sample_classes": values,
+        }
+
+
+def provision(year: int, destination: Path, force: bool) -> dict:
+    source = SOURCES[year]
+    target = destination / source["target"]
+    archive = destination / f"{Path(source['target']).stem}.zip"
+    if target.is_file() and not force:
+        print(f"WALOUS {year}: validating existing source {target}")
+        validation = validate_raster(target)
+        digest = sha256_file(target)
+    else:
+        print(f"WALOUS {year}: downloading official archive")
+        archive_digest = download(source["url"], archive, source["expected_archive_bytes"])
+        print(f"WALOUS {year}: archive sha256 {archive_digest}")
+        extract_single_geotiff(archive, target)
+        validation = validate_raster(target)
+        digest = sha256_file(target)
+        archive.unlink(missing_ok=True)
+    checksum_path = target.with_suffix(".sha256")
+    checksum_path.write_text(f"{digest}  {target.name}\n", encoding="ascii")
+    validation.update({"year": year, "sha256": digest, "download_url": source["url"]})
+    print(f"WALOUS {year}: ready ({target.stat().st_size / 1024 / 1024:.0f} MiB)")
+    return validation
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Provision official WALOUS 2020/2023 GeoTIFF sources.")
+    parser.add_argument("--years", nargs="+", type=int, choices=sorted(SOURCES), default=sorted(SOURCES))
+    parser.add_argument("--destination", type=Path, default=Path("storage/source-cache/walous"))
+    parser.add_argument("--force", action="store_true")
+    args = parser.parse_args()
+    args.destination.mkdir(parents=True, exist_ok=True)
+    report = [provision(year, args.destination.resolve(), args.force) for year in args.years]
+    report_path = args.destination / "provisioning-report.json"
+    report_path.write_text(json.dumps({"sources": report}, indent=2) + "\n", encoding="utf-8")
+    print(f"Provisioning report: {report_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception as exc:
+        print(f"WALOUS_PROVISIONING_FAILED: {exc}", file=sys.stderr)
+        raise SystemExit(1) from exc