Initial public release

scripts/provision_walous_sources.py

@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+"""Provision official WALOUS GeoTIFF source rasters for bounded runtime analysis."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import shutil
+import sys
+from urllib.request import Request, urlopen
+from zipfile import BadZipFile, ZipFile
+
+
+SOURCES = {
+    2018: {
+        "url": (
+            "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
+            "a0ad23a1-1845-4bd5-8c2f-0f62d3f1ec75/WALOUS_OCS__2018_GEOTIFF_3812.zip"
+        ),
+        "expected_archive_bytes": 1_122_785_133,
+        "target": "walous_land_cover_2018_3812.tif",
+    },
+    2020: {
+        "url": (
+            "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
+            "47b348f1-6e7a-4baa-963c-0232a43c0cff/WAL_OCS_IA__2020_GEOTIFF_3812.zip"
+        ),
+        "expected_archive_bytes": 728_244_755,
+        "target": "walous_land_cover_2020_3812.tif",
+    },
+    2023: {
+        "url": (
+            "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
+            "4e780ba1-463c-478e-95df-d2f1963a150d/WAL_OCS_IA__2023_GEOTIFF_3812.zip"
+        ),
+        "expected_archive_bytes": 876_014_572,
+        "target": "walous_land_cover_2023_3812.tif",
+    },
+}
+MAX_ARCHIVE_BYTES = 1_250_000_000
+MAX_EXTRACTED_BYTES = 50_000_000_000
+WALOUS_CLASS_CODES = {
+    0,
+    1,
+    2,
+    3,
+    4,
+    5,
+    6,
+    7,
+    8,
+    9,
+    11,
+    15,
+    18,
+    19,
+    28,
+    29,
+    31,
+    38,
+    39,
+    51,
+    55,
+    58,
+    59,
+    62,
+    71,
+    73,
+    75,
+    80,
+    81,
+    83,
+    85,
+    90,
+    91,
+    93,
+    95,
+}
+WALOUS_CANONICAL_CLASS_CODES = {1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90}
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        while chunk := handle.read(8 * 1024 * 1024):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def download(url: str, destination: Path, expected_bytes: int) -> str:
+    temporary = destination.with_suffix(destination.suffix + ".part")
+    temporary.unlink(missing_ok=True)
+    digest = hashlib.sha256()
+    received = 0
+    request = Request(
+        url, headers={"User-Agent": "GeoIntel/1.0 WALOUS-source-provisioner"}
+    )
+    try:
+        with urlopen(request, timeout=300) as response, temporary.open("wb") as output:
+            content_length = int(response.headers.get("Content-Length") or 0)
+            if content_length and content_length != expected_bytes:
+                raise RuntimeError(
+                    f"official archive size changed: expected {expected_bytes}, advertised {content_length}"
+                )
+            while chunk := response.read(8 * 1024 * 1024):
+                received += len(chunk)
+                if received > MAX_ARCHIVE_BYTES:
+                    raise RuntimeError(
+                        "official archive exceeds the governed 1 GB transfer limit"
+                    )
+                digest.update(chunk)
+                output.write(chunk)
+                if received % (128 * 1024 * 1024) < len(chunk):
+                    print(f"  downloaded {received / 1024 / 1024:.0f} MiB", flush=True)
+        if received != expected_bytes:
+            raise RuntimeError(
+                f"archive is incomplete: expected {expected_bytes} bytes, received {received}"
+            )
+        temporary.replace(destination)
+        return digest.hexdigest()
+    except Exception:
+        temporary.unlink(missing_ok=True)
+        raise
+
+
+def extract_single_geotiff(archive: Path, target: Path) -> None:
+    try:
+        with ZipFile(archive) as bundle:
+            candidates = [
+                item
+                for item in bundle.infolist()
+                if not item.is_dir()
+                and item.filename.lower().endswith((".tif", ".tiff"))
+            ]
+            if len(candidates) != 1:
+                raise RuntimeError(
+                    f"archive must contain exactly one GeoTIFF, found {len(candidates)}"
+                )
+            member = candidates[0]
+            if member.file_size <= 0 or member.file_size > MAX_EXTRACTED_BYTES:
+                raise RuntimeError(
+                    f"GeoTIFF uncompressed size is outside the governed limit: {member.file_size}"
+                )
+            if (
+                Path(member.filename).name
+                != member.filename.replace("\\", "/").split("/")[-1]
+            ):
+                # Nested paths are accepted only by basename; extraction never trusts archive paths.
+                pass
+            temporary = target.with_suffix(target.suffix + ".part")
+            temporary.unlink(missing_ok=True)
+            with bundle.open(member) as source, temporary.open("wb") as output:
+                shutil.copyfileobj(source, output, length=8 * 1024 * 1024)
+            temporary.replace(target)
+    except BadZipFile as exc:
+        raise RuntimeError("official WALOUS archive is not a valid ZIP file") from exc
+
+
+def validate_raster(path: Path) -> dict:
+    try:
+        import numpy as np
+        import rasterio
+        from rasterio.enums import Resampling
+    except ImportError as exc:
+        raise RuntimeError(
+            "rasterio and numpy are required to validate WALOUS sources"
+        ) from exc
+    with rasterio.open(path) as source:
+        if source.crs is None or source.crs.to_epsg() != 3812:
+            raise RuntimeError(f"WALOUS raster must use EPSG:3812, found {source.crs}")
+        if source.count != 1:
+            raise RuntimeError(
+                f"WALOUS raster must have one band, found {source.count}"
+            )
+        if not all(abs(abs(float(value)) - 1.0) <= 0.05 for value in source.res):
+            raise RuntimeError(
+                f"WALOUS raster must retain 1 m cells, found {source.res}"
+            )
+        sample_height = min(2048, source.height)
+        sample_width = min(2048, source.width)
+        sample = source.read(
+            1,
+            out_shape=(sample_height, sample_width),
+            masked=True,
+            resampling=Resampling.nearest,
+        )
+        values = np.unique(sample.compressed()).astype(int).tolist()
+        allowed_codes = (
+            WALOUS_CLASS_CODES if "2018" in path.name else WALOUS_CANONICAL_CLASS_CODES
+        )
+        unexpected = sorted(set(values) - allowed_codes)
+        if unexpected:
+            raise RuntimeError(
+                f"WALOUS sample contains classes outside the official 11-class code set: {unexpected}"
+            )
+        return {
+            "path": str(path),
+            "crs": str(source.crs),
+            "width": int(source.width),
+            "height": int(source.height),
+            "resolution": [float(value) for value in source.res],
+            "bounds": [float(value) for value in source.bounds],
+            "nodata": None if source.nodata is None else float(source.nodata),
+            "sample_classes": values,
+            "implicit_source_nodata_values": [0]
+            if "2018" in path.name and 0 in values
+            else [],
+        }
+
+
+def provision(year: int, destination: Path, force: bool) -> dict:
+    source = SOURCES[year]
+    target = destination / source["target"]
+    archive = destination / f"{Path(source['target']).stem}.zip"
+    if target.is_file() and not force:
+        print(f"WALOUS {year}: validating existing source {target}")
+        validation = validate_raster(target)
+        digest = sha256_file(target)
+    else:
+        if (
+            archive.is_file()
+            and archive.stat().st_size == source["expected_archive_bytes"]
+        ):
+            print(f"WALOUS {year}: using existing official archive {archive}")
+            archive_digest = sha256_file(archive)
+        else:
+            print(f"WALOUS {year}: downloading official archive")
+            archive_digest = download(
+                source["url"], archive, source["expected_archive_bytes"]
+            )
+        print(f"WALOUS {year}: archive sha256 {archive_digest}")
+        extract_single_geotiff(archive, target)
+        validation = validate_raster(target)
+        digest = sha256_file(target)
+        archive.unlink(missing_ok=True)
+    checksum_path = target.with_suffix(".sha256")
+    checksum_path.write_text(f"{digest}  {target.name}\n", encoding="ascii")
+    validation.update({"year": year, "sha256": digest, "download_url": source["url"]})
+    print(f"WALOUS {year}: ready ({target.stat().st_size / 1024 / 1024:.0f} MiB)")
+    return validation
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Provision official WALOUS 2018/2020/2023 GeoTIFF sources."
+    )
+    parser.add_argument(
+        "--years", nargs="+", type=int, choices=sorted(SOURCES), default=sorted(SOURCES)
+    )
+    parser.add_argument(
+        "--destination", type=Path, default=Path("storage/source-cache/walous")
+    )
+    parser.add_argument("--force", action="store_true")
+    args = parser.parse_args()
+    args.destination.mkdir(parents=True, exist_ok=True)
+    report = [
+        provision(year, args.destination.resolve(), args.force) for year in args.years
+    ]
+    report_path = args.destination / "provisioning-report.json"
+    retained: dict[int, dict] = {}
+    if report_path.is_file():
+        try:
+            retained = {
+                int(item["year"]): item
+                for item in json.loads(report_path.read_text(encoding="utf-8")).get(
+                    "sources", []
+                )
+                if isinstance(item, dict) and item.get("year") in SOURCES
+            }
+        except (OSError, ValueError, TypeError):
+            retained = {}
+    retained.update({int(item["year"]): item for item in report})
+    report_path.write_text(
+        json.dumps({"sources": [retained[year] for year in sorted(retained)]}, indent=2)
+        + "\n",
+        encoding="utf-8",
+    )
+    print(f"Provisioning report: {report_path}")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception as exc:
+        print(f"WALOUS_PROVISIONING_FAILED: {exc}", file=sys.stderr)
+        raise SystemExit(1) from exc