geointel/scripts/provision_walous_sources.py

#!/usr/bin/env python3
"""Provision official WALOUS GeoTIFF source rasters for bounded runtime analysis."""

from __future__ import annotations

import argparse
import hashlib
import json
from pathlib import Path
import shutil
import sys
from urllib.request import Request, urlopen
from zipfile import BadZipFile, ZipFile


SOURCES = {
    2018: {
        "url": (
            "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
            "a0ad23a1-1845-4bd5-8c2f-0f62d3f1ec75/WALOUS_OCS__2018_GEOTIFF_3812.zip"
        ),
        "expected_archive_bytes": 1_122_785_133,
        "target": "walous_land_cover_2018_3812.tif",
    },
    2020: {
        "url": (
            "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
            "47b348f1-6e7a-4baa-963c-0232a43c0cff/WAL_OCS_IA__2020_GEOTIFF_3812.zip"
        ),
        "expected_archive_bytes": 728_244_755,
        "target": "walous_land_cover_2020_3812.tif",
    },
    2023: {
        "url": (
            "https://geoservices.wallonie.be/geotraitement/spwdatadownload/results/"
            "4e780ba1-463c-478e-95df-d2f1963a150d/WAL_OCS_IA__2023_GEOTIFF_3812.zip"
        ),
        "expected_archive_bytes": 876_014_572,
        "target": "walous_land_cover_2023_3812.tif",
    },
}
MAX_ARCHIVE_BYTES = 1_250_000_000
MAX_EXTRACTED_BYTES = 50_000_000_000
WALOUS_CLASS_CODES = {
    0,
    1,
    2,
    3,
    4,
    5,
    6,
    7,
    8,
    9,
    11,
    15,
    18,
    19,
    28,
    29,
    31,
    38,
    39,
    51,
    55,
    58,
    59,
    62,
    71,
    73,
    75,
    80,
    81,
    83,
    85,
    90,
    91,
    93,
    95,
}
WALOUS_CANONICAL_CLASS_CODES = {1, 2, 3, 4, 5, 6, 7, 8, 9, 80, 90}


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        while chunk := handle.read(8 * 1024 * 1024):
            digest.update(chunk)
    return digest.hexdigest()


def download(url: str, destination: Path, expected_bytes: int) -> str:
    temporary = destination.with_suffix(destination.suffix + ".part")
    temporary.unlink(missing_ok=True)
    digest = hashlib.sha256()
    received = 0
    request = Request(
        url, headers={"User-Agent": "GeoIntel/1.0 WALOUS-source-provisioner"}
    )
    try:
        with urlopen(request, timeout=300) as response, temporary.open("wb") as output:
            content_length = int(response.headers.get("Content-Length") or 0)
            if content_length and content_length != expected_bytes:
                raise RuntimeError(
                    f"official archive size changed: expected {expected_bytes}, advertised {content_length}"
                )
            while chunk := response.read(8 * 1024 * 1024):
                received += len(chunk)
                if received > MAX_ARCHIVE_BYTES:
                    raise RuntimeError(
                        "official archive exceeds the governed 1 GB transfer limit"
                    )
                digest.update(chunk)
                output.write(chunk)
                if received % (128 * 1024 * 1024) < len(chunk):
                    print(f"  downloaded {received / 1024 / 1024:.0f} MiB", flush=True)
        if received != expected_bytes:
            raise RuntimeError(
                f"archive is incomplete: expected {expected_bytes} bytes, received {received}"
            )
        temporary.replace(destination)
        return digest.hexdigest()
    except Exception:
        temporary.unlink(missing_ok=True)
        raise


def extract_single_geotiff(archive: Path, target: Path) -> None:
    try:
        with ZipFile(archive) as bundle:
            candidates = [
                item
                for item in bundle.infolist()
                if not item.is_dir()
                and item.filename.lower().endswith((".tif", ".tiff"))
            ]
            if len(candidates) != 1:
                raise RuntimeError(
                    f"archive must contain exactly one GeoTIFF, found {len(candidates)}"
                )
            member = candidates[0]
            if member.file_size <= 0 or member.file_size > MAX_EXTRACTED_BYTES:
                raise RuntimeError(
                    f"GeoTIFF uncompressed size is outside the governed limit: {member.file_size}"
                )
            if (
                Path(member.filename).name
                != member.filename.replace("\\", "/").split("/")[-1]
            ):
                # Nested paths are accepted only by basename; extraction never trusts archive paths.
                pass
            temporary = target.with_suffix(target.suffix + ".part")
            temporary.unlink(missing_ok=True)
            with bundle.open(member) as source, temporary.open("wb") as output:
                shutil.copyfileobj(source, output, length=8 * 1024 * 1024)
            temporary.replace(target)
    except BadZipFile as exc:
        raise RuntimeError("official WALOUS archive is not a valid ZIP file") from exc


def validate_raster(path: Path) -> dict:
    try:
        import numpy as np
        import rasterio
        from rasterio.enums import Resampling
    except ImportError as exc:
        raise RuntimeError(
            "rasterio and numpy are required to validate WALOUS sources"
        ) from exc
    with rasterio.open(path) as source:
        if source.crs is None or source.crs.to_epsg() != 3812:
            raise RuntimeError(f"WALOUS raster must use EPSG:3812, found {source.crs}")
        if source.count != 1:
            raise RuntimeError(
                f"WALOUS raster must have one band, found {source.count}"
            )
        if not all(abs(abs(float(value)) - 1.0) <= 0.05 for value in source.res):
            raise RuntimeError(
                f"WALOUS raster must retain 1 m cells, found {source.res}"
            )
        sample_height = min(2048, source.height)
        sample_width = min(2048, source.width)
        sample = source.read(
            1,
            out_shape=(sample_height, sample_width),
            masked=True,
            resampling=Resampling.nearest,
        )
        values = np.unique(sample.compressed()).astype(int).tolist()
        allowed_codes = (
            WALOUS_CLASS_CODES if "2018" in path.name else WALOUS_CANONICAL_CLASS_CODES
        )
        unexpected = sorted(set(values) - allowed_codes)
        if unexpected:
            raise RuntimeError(
                f"WALOUS sample contains classes outside the official 11-class code set: {unexpected}"
            )
        return {
            "path": str(path),
            "crs": str(source.crs),
            "width": int(source.width),
            "height": int(source.height),
            "resolution": [float(value) for value in source.res],
            "bounds": [float(value) for value in source.bounds],
            "nodata": None if source.nodata is None else float(source.nodata),
            "sample_classes": values,
            "implicit_source_nodata_values": [0]
            if "2018" in path.name and 0 in values
            else [],
        }


def provision(year: int, destination: Path, force: bool) -> dict:
    source = SOURCES[year]
    target = destination / source["target"]
    archive = destination / f"{Path(source['target']).stem}.zip"
    if target.is_file() and not force:
        print(f"WALOUS {year}: validating existing source {target}")
        validation = validate_raster(target)
        digest = sha256_file(target)
    else:
        if (
            archive.is_file()
            and archive.stat().st_size == source["expected_archive_bytes"]
        ):
            print(f"WALOUS {year}: using existing official archive {archive}")
            archive_digest = sha256_file(archive)
        else:
            print(f"WALOUS {year}: downloading official archive")
            archive_digest = download(
                source["url"], archive, source["expected_archive_bytes"]
            )
        print(f"WALOUS {year}: archive sha256 {archive_digest}")
        extract_single_geotiff(archive, target)
        validation = validate_raster(target)
        digest = sha256_file(target)
        archive.unlink(missing_ok=True)
    checksum_path = target.with_suffix(".sha256")
    checksum_path.write_text(f"{digest}  {target.name}\n", encoding="ascii")
    validation.update({"year": year, "sha256": digest, "download_url": source["url"]})
    print(f"WALOUS {year}: ready ({target.stat().st_size / 1024 / 1024:.0f} MiB)")
    return validation


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Provision official WALOUS 2018/2020/2023 GeoTIFF sources."
    )
    parser.add_argument(
        "--years", nargs="+", type=int, choices=sorted(SOURCES), default=sorted(SOURCES)
    )
    parser.add_argument(
        "--destination", type=Path, default=Path("storage/source-cache/walous")
    )
    parser.add_argument("--force", action="store_true")
    args = parser.parse_args()
    args.destination.mkdir(parents=True, exist_ok=True)
    report = [
        provision(year, args.destination.resolve(), args.force) for year in args.years
    ]
    report_path = args.destination / "provisioning-report.json"
    retained: dict[int, dict] = {}
    if report_path.is_file():
        try:
            retained = {
                int(item["year"]): item
                for item in json.loads(report_path.read_text(encoding="utf-8")).get(
                    "sources", []
                )
                if isinstance(item, dict) and item.get("year") in SOURCES
            }
        except (OSError, ValueError, TypeError):
            retained = {}
    retained.update({int(item["year"]): item for item in report})
    report_path.write_text(
        json.dumps({"sources": [retained[year] for year in sorted(retained)]}, indent=2)
        + "\n",
        encoding="utf-8",
    )
    print(f"Provisioning report: {report_path}")
    return 0


if __name__ == "__main__":
    try:
        raise SystemExit(main())
    except Exception as exc:
        print(f"WALOUS_PROVISIONING_FAILED: {exc}", file=sys.stderr)
        raise SystemExit(1) from exc