Add governed ALZ release promotion

scripts/provision_agricultural_parcel_history.py

@@ -10,17 +10,19 @@ directly and never uses the provisional current-campaign snapshot.
 from __future__ import annotations
 
 import argparse
+from dataclasses import dataclass
 import hashlib
 import json
 import math
 import os
-import shutil
+import re
 import sys
 import tempfile
 import zipfile
 from datetime import date, datetime, timezone
 from pathlib import Path
 from typing import Any, Iterable
+from urllib.parse import urlsplit
 
 import requests
 from pyproj import Transformer
@@ -51,6 +53,10 @@ SOURCE_CRS = "EPSG:31370"
 OUTPUT_CRS = "EPSG:4326"
 SCHEMA_VERSION = 1
 MAX_ARCHIVE_BYTES = 250 * 1024 * 1024
+MAX_EXTRACTED_BYTES = 2 * 1024 * 1024 * 1024
+MAX_ARCHIVE_MEMBERS = 32
+ARCHIVE_HOST = "www.landbouwvlaanderen.be"
+ARCHIVE_PATH_PATTERN = re.compile(r"^/bestanden/gis/agpa_(20[0-9]{2})_(20[0-9]{2}-[0-9]{2}-[0-9]{2})_public\.zip$")
 
 ARCHIVE_URLS = {
     2008: "https://www.landbouwvlaanderen.be/bestanden/gis/agpa_2008_2022-03-23_public.zip",
@@ -74,6 +80,16 @@ ARCHIVE_URLS = {
 }
 SUPPORTED_YEARS = tuple(ARCHIVE_URLS)
 
+
+@dataclass(frozen=True)
+class AgriculturalReleaseConfig:
+    year: int
+    archive_url: str
+
+    @property
+    def definitive_version(self) -> str:
+        return f"{self.year}-v3"
+
 STABLE_REQUIRED_FIELDS = {
     "agpakey",
     "parcelnumber",
@@ -131,6 +147,10 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
     parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE_KEY)
     parser.add_argument("--years", default=",".join(str(year) for year in SUPPORTED_YEARS))
+    parser.add_argument(
+        "--archive-url",
+        help="Exact official archive URL for one explicitly confirmed future definitive edition.",
+    )
     parser.add_argument("--output-root", type=Path, default=Path(os.environ.get("GEOINTEL_AGRICULTURE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)))
     parser.add_argument("--request-timeout", type=int, default=900)
     parser.add_argument("--import-timeout", type=int, default=3600)
@@ -201,21 +221,29 @@ def response_data(response: requests.Response) -> Any:
 def api_items(session: requests.Session, url: str, timeout: int) -> list[dict[str, Any]]:
     items: list[dict[str, Any]] = []
     offset = 0
-    while True:
+    expected_total: int | None = None
+    while expected_total is None or offset < expected_total:
         response = session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout)
         data = response_data(response)
         if isinstance(data, dict):
             page = data.get("items") or data.get("results") or []
-            total = int(data.get("total") or len(page))
+            page_total = int(data.get("total") if data.get("total") is not None else len(page))
         else:
             page = data
-            total = len(page) if isinstance(page, list) else 0
+            page_total = len(page) if isinstance(page, list) else 0
         if not isinstance(page, list):
             raise RuntimeError(f"Expected a list response from {url}")
+        if expected_total is None:
+            expected_total = page_total
+        elif page_total != expected_total:
+            raise RuntimeError("GeoIntel pagination total changed while reading the agricultural workspace")
         items.extend(item for item in page if isinstance(item, dict))
-        if not page or len(items) >= total:
-            return items
+        if not page:
+            break
         offset += len(page)
+    if expected_total is not None and len(items) != expected_total:
+        raise RuntimeError(f"GeoIntel list response for {url} returned {len(items)} of {expected_total} items")
+    return items
 
 
 def locate_workspace(
@@ -251,6 +279,53 @@ def parse_years(raw: str) -> list[int]:
     return years
 
 
+def validate_archive_url(url: str, *, expected_year: int) -> str:
+    parsed = urlsplit(url)
+    match = ARCHIVE_PATH_PATTERN.fullmatch(parsed.path)
+    if (
+        parsed.scheme != "https"
+        or parsed.hostname != ARCHIVE_HOST
+        or parsed.port not in {None, 443}
+        or parsed.username
+        or parsed.password
+        or parsed.query
+        or parsed.fragment
+        or not match
+        or int(match.group(1)) != expected_year
+    ):
+        raise ValueError("Agricultural release archive is outside the official ALZ URL contract")
+    try:
+        datetime.strptime(match.group(2), "%Y-%m-%d")
+    except ValueError as exc:
+        raise ValueError("Agricultural release archive contains an invalid publication date") from exc
+    return url
+
+
+def resolve_release_config(year: int, *, archive_url: str | None = None) -> AgriculturalReleaseConfig:
+    known_url = ARCHIVE_URLS.get(year)
+    if known_url is not None:
+        if archive_url is not None and archive_url != known_url:
+            raise ValueError(f"The official retained archive identity for {year} may not be overridden")
+        return AgriculturalReleaseConfig(year=year, archive_url=known_url)
+    if year <= max(SUPPORTED_YEARS) or not archive_url:
+        raise ValueError(
+            f"One future definitive edition after {max(SUPPORTED_YEARS)} may be supplied with --archive-url"
+        )
+    return AgriculturalReleaseConfig(year=year, archive_url=validate_archive_url(archive_url, expected_year=year))
+
+
+def resolve_release_configs(raw_years: str, *, archive_url: str | None = None) -> list[AgriculturalReleaseConfig]:
+    if archive_url is None:
+        return [resolve_release_config(year) for year in parse_years(raw_years)]
+    try:
+        years = sorted({int(value.strip()) for value in raw_years.split(",") if value.strip()})
+    except ValueError as exc:
+        raise ValueError("Years must be a comma-separated list of integers") from exc
+    if len(years) != 1:
+        raise ValueError("--archive-url requires exactly one explicitly selected definitive year")
+    return [resolve_release_config(years[0], archive_url=archive_url)]
+
+
 def polygonal_geometry(geometry):
     if geometry is None or geometry.is_empty:
         return None
@@ -308,6 +383,10 @@ def download_archive(
     temporary.unlink(missing_ok=True)
     response = session.get(url, timeout=timeout, stream=True)
     response.raise_for_status()
+    final_url = str(getattr(response, "url", "") or url)
+    expected_year = int(ARCHIVE_PATH_PATTERN.fullmatch(urlsplit(url).path).group(1))
+    if validate_archive_url(final_url, expected_year=expected_year) != url:
+        raise RuntimeError("Official archive download redirected to a different release identity")
     content_length = int(response.headers.get("content-length") or 0)
     if content_length > max_bytes:
         raise RuntimeError(f"Official archive exceeds the configured {max_bytes // (1024 * 1024)} MiB limit")
@@ -331,7 +410,12 @@ def download_archive(
 
 def archive_geopackage_member(path: Path) -> str:
     with zipfile.ZipFile(path) as archive:
-        members = [item.filename for item in archive.infolist() if not item.is_dir() and item.filename.lower().endswith(".gpkg")]
+        entries = archive.infolist()
+        if len(entries) > MAX_ARCHIVE_MEMBERS:
+            raise RuntimeError(f"Official archive contains more than {MAX_ARCHIVE_MEMBERS} members")
+        if sum(item.file_size for item in entries if not item.is_dir()) > MAX_EXTRACTED_BYTES:
+            raise RuntimeError("Official archive exceeds the extracted-size safety limit")
+        members = [item.filename for item in entries if not item.is_dir() and item.filename.lower().endswith(".gpkg")]
     if len(members) != 1:
         raise RuntimeError(f"Official archive must contain exactly one GeoPackage; found {len(members)}")
     member = members[0]
@@ -351,7 +435,12 @@ def extract_geopackage(archive_path: Path, destination_dir: Path) -> Path:
     member = validate_archive(archive_path)
     destination = destination_dir / Path(member).name
     with zipfile.ZipFile(archive_path) as archive, archive.open(member) as source, destination.open("wb") as target:
-        shutil.copyfileobj(source, target, length=1024 * 1024)
+        extracted_bytes = 0
+        for chunk in iter(lambda: source.read(1024 * 1024), b""):
+            extracted_bytes += len(chunk)
+            if extracted_bytes > MAX_EXTRACTED_BYTES:
+                raise RuntimeError("Official GeoPackage exceeded the extracted-size safety limit while streaming")
+            target.write(chunk)
     if destination.stat().st_size == 0:
         raise RuntimeError("Extracted official GeoPackage is empty")
     return destination
@@ -470,23 +559,39 @@ def normalize_frame(frame, *, year: int, boundary_lambert72, max_features: int)
     }
 
 
-def artifact_paths(output_root: Path, scope_key: str, year: int) -> dict[str, Path]:
+def artifact_paths(
+    output_root: Path,
+    scope_key: str,
+    year: int,
+    *,
+    archive_url: str | None = None,
+) -> dict[str, Path]:
+    release = resolve_release_config(year, archive_url=archive_url)
     directory = output_root / scope_key / str(year)
     return {
         "directory": directory,
-        "archive": directory / Path(ARCHIVE_URLS[year]).name,
+        "archive": directory / Path(urlsplit(release.archive_url).path).name,
         "artifact": directory / f"agricultural_use_parcels_{year}_{scope_key}.geojson",
         "codelist": directory / f"agricultural_use_parcels_{year}_crop_codes.json",
         "manifest": directory / f"agricultural_use_parcels_{year}_{scope_key}.manifest.json",
     }
 
 
-def reusable_artifact(paths: dict[str, Path], *, year: int, scope_key: str) -> dict[str, Any] | None:
+def reusable_artifact(
+    paths: dict[str, Path],
+    *,
+    year: int,
+    scope_key: str,
+    archive_url: str | None = None,
+) -> dict[str, Any] | None:
     if not all(paths[key].is_file() for key in ("archive", "artifact", "codelist", "manifest")):
         return None
     manifest = json.loads(paths["manifest"].read_text(encoding="utf-8"))
     if manifest.get("schema_version") != SCHEMA_VERSION or manifest.get("year") != year or manifest.get("scope_key") != scope_key:
         return None
+    release = resolve_release_config(year, archive_url=archive_url)
+    if manifest.get("source_url") != release.archive_url:
+        return None
     if manifest.get("source_archive_sha256") != sha256_file(paths["archive"]):
         return None
     if manifest.get("artifact_sha256") != sha256_file(paths["artifact"]):
@@ -508,16 +613,18 @@ def prepare_year(
     max_archive_bytes: int,
     max_features: int,
     force: bool,
+    archive_url: str | None = None,
 ) -> tuple[dict[str, Path], dict[str, Any]]:
-    paths = artifact_paths(output_root, scope.key, year)
+    release = resolve_release_config(year, archive_url=archive_url)
+    paths = artifact_paths(output_root, scope.key, year, archive_url=release.archive_url)
     paths["directory"].mkdir(parents=True, exist_ok=True)
     if not force:
-        reused = reusable_artifact(paths, year=year, scope_key=scope.key)
+        reused = reusable_artifact(paths, year=year, scope_key=scope.key, archive_url=release.archive_url)
         if reused is not None:
             return paths, reused
     download = download_archive(
         session,
-        ARCHIVE_URLS[year],
+        release.archive_url,
         paths["archive"],
         timeout=request_timeout,
         max_bytes=max_archive_bytes,
@@ -546,7 +653,7 @@ def prepare_year(
         "scope_name": scope.display_name,
         "scope_type": scope.scope_type,
         "member_nis_codes": list(scope.nis_codes),
-        "source_url": ARCHIVE_URLS[year],
+        "source_url": release.archive_url,
         "catalog_url": CATALOG_URL,
         "data_catalog_url": DATA_CATALOG_URL,
         "attribution": ATTRIBUTION,
@@ -635,7 +742,7 @@ def upload_artifact(
         "operator_tool": "provision_agricultural_parcel_history.py",
         "operator_explicit_fetch": True,
         "geometry_clipped_to_area": True,
-        "source_archive_url": ARCHIVE_URLS[year],
+        "source_archive_url": manifest["source_url"],
         "source_archive_path": str(paths["archive"]),
         "source_archive_sha256": manifest["source_archive_sha256"],
         "crop_code_list_path": str(paths["codelist"]),
@@ -684,7 +791,7 @@ def existing_dataset_for_year(datasets: list[dict[str, Any]], *, area_id: str, y
 def main() -> int:
     args = parse_args()
     try:
-        years = parse_years(args.years)
+        releases = resolve_release_configs(args.years, archive_url=args.archive_url)
         if args.max_features < 1 or args.max_archive_mb < 1:
             raise ValueError("Feature and archive safety limits must be positive")
         scope = GEOGRAPHIC_SCOPES[args.scope]
@@ -693,7 +800,8 @@ def main() -> int:
             project_id, area_id, boundary, datasets = locate_workspace(api_session, base_url, scope, args.import_timeout)
             results: list[dict[str, Any]] = []
             with build_session() as official_session:
-                for year in years:
+                for release in releases:
+                    year = release.year
                     paths, manifest = prepare_year(
                         official_session,
                         year=year,
@@ -704,6 +812,7 @@ def main() -> int:
                         max_archive_bytes=min(args.max_archive_mb * 1024 * 1024, MAX_ARCHIVE_BYTES),
                         max_features=args.max_features,
                         force=args.force,
+                        archive_url=release.archive_url,
                     )
                     existing = existing_dataset_for_year(datasets, area_id=area_id, year=year)
                     if existing is not None: