geointel/scripts/provision_mol_municipality_workspace.py

"""Provision a complete, authoritative Mol municipality map workspace.

The script is an explicit operator tool. It fetches the official Mol boundary
and GRB buildings, writes auditable artifacts, and imports them through the
existing GeoIntel API. It is never called automatically by application startup.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable

import requests
from requests.adapters import HTTPAdapter
from pyproj import Transformer
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
from shapely.ops import transform, unary_union
from shapely.validation import make_valid
from urllib3.util.retry import Retry


MUNICIPALITY_NAME = "Mol"
MUNICIPALITY_NIS_CODE = "13025"
PROJECT_NAME = "Mol Municipality Workbench"
PROJECT_REGION = "Mol, Kempen"
AREA_NAME = "Gemeente Mol - officiële grens"
BOUNDARY_FILENAME = "mol_municipality_boundary.geojson"
BUILDINGS_FILENAME = "mol_grb_gbg_buildings.geojson"
MANIFEST_FILENAME = "mol_municipality_manifest.json"
VRBG_ITEMS_URL = "https://geo.api.vlaanderen.be/VRBG/ogc/features/v1/collections/Refgem/items"
GRB_GBG_ITEMS_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
VRBG_ATTRIBUTION = "Bron: Voorlopig referentiebestand gemeentegrenzen, Digitaal Vlaanderen"
GRB_ATTRIBUTION = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen"
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/mol-municipality")
DEFAULT_API_URL = "http://127.0.0.1:8000"
DEFAULT_PAGE_LIMIT = 1000
DEFAULT_MAX_FEATURES = 100000
GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}}
RETRYABLE_SOURCE_STATUS_CODES = (429, 500, 502, 503, 504)


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        description="Fetch and provision the complete official Mol municipality workspace.",
    )
    parser.add_argument(
        "--output-dir",
        type=Path,
        default=Path(os.environ.get("MOL_MUNICIPALITY_OUTPUT_DIR", DEFAULT_OUTPUT_DIR)),
        help="Persistent directory for the boundary, GRB buildings and provenance manifest.",
    )
    parser.add_argument(
        "--base-url",
        default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL),
        help="GeoIntel backend URL. The in-container direct backend avoids reverse-proxy timeouts during large imports.",
    )
    parser.add_argument(
        "--project-name",
        default=PROJECT_NAME,
        help="Exact idempotent project name used for the municipality workspace.",
    )
    parser.add_argument(
        "--page-limit",
        type=int,
        default=int(os.environ.get("MOL_GRB_PAGE_LIMIT", str(DEFAULT_PAGE_LIMIT))),
        help="GRB OGC API page size.",
    )
    parser.add_argument(
        "--max-features",
        type=int,
        default=int(os.environ.get("MOL_GRB_MAX_FEATURES", str(DEFAULT_MAX_FEATURES))),
        help="Safety cap. The script fails instead of writing a truncated municipality dataset.",
    )
    parser.add_argument(
        "--request-timeout",
        type=int,
        default=int(os.environ.get("MOL_MUNICIPALITY_REQUEST_TIMEOUT", "180")),
        help="Timeout per official source request in seconds.",
    )
    parser.add_argument(
        "--import-timeout",
        type=int,
        default=int(os.environ.get("MOL_MUNICIPALITY_IMPORT_TIMEOUT", "1800")),
        help="Timeout for each GeoIntel API import request in seconds.",
    )
    parser.add_argument("--force", action="store_true", help="Refetch official source artifacts even when complete local artifacts exist.")
    parser.add_argument("--fetch-only", action="store_true", help="Prepare artifacts without creating or updating the GeoIntel workspace.")
    return parser.parse_args()


def utc_now() -> str:
    return datetime.now(timezone.utc).isoformat()


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def write_json(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> None:
    path.write_text(
        json.dumps(
            payload,
            ensure_ascii=False,
            indent=2 if pretty else None,
            separators=None if pretty else (",", ":"),
            sort_keys=pretty,
        ),
        encoding="utf-8",
    )


def build_source_session() -> requests.Session:
    retry = Retry(
        total=5,
        connect=5,
        read=5,
        status=5,
        backoff_factor=1.0,
        status_forcelist=RETRYABLE_SOURCE_STATUS_CODES,
        allowed_methods=frozenset({"GET"}),
        raise_on_status=True,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session = requests.Session()
    session.headers.update({"User-Agent": "GeoIntel-Mol-Municipality-Operator/1.0"})
    session.mount("https://", adapter)
    session.mount("http://", adapter)
    return session


def next_page_url(payload: dict[str, Any]) -> str | None:
    links = payload.get("links") or []
    for link in links:
        if link.get("rel") == "next" and "geo+json" in str(link.get("type", "")).lower():
            return str(link["href"])
    for link in links:
        if link.get("rel") == "next" and link.get("href"):
            return str(link["href"])
    return None


def normalize_polygonal(geometry):
    if geometry.is_empty:
        return None
    if not geometry.is_valid:
        geometry = make_valid(geometry)
    if isinstance(geometry, (Polygon, MultiPolygon)):
        return geometry
    if isinstance(geometry, GeometryCollection):
        polygons = [part for part in geometry.geoms if isinstance(part, (Polygon, MultiPolygon)) and not part.is_empty]
        if polygons:
            merged = unary_union(polygons)
            return merged if isinstance(merged, (Polygon, MultiPolygon)) and not merged.is_empty else None
    return None


def fetch_mol_boundary(session: requests.Session, timeout: int) -> tuple[dict[str, Any], Any, str]:
    params = {
        "f": "application/geo+json",
        "limit": "10",
        "filter": "NAAM='Mol'",
        "filter-lang": "cql2-text",
    }
    response = session.get(VRBG_ITEMS_URL, params=params, timeout=timeout)
    response.raise_for_status()
    payload = response.json()
    matches = [
        feature
        for feature in payload.get("features") or []
        if str((feature.get("properties") or {}).get("NAAM", "")).casefold() == MUNICIPALITY_NAME.casefold()
    ]
    if len(matches) != 1:
        raise RuntimeError(f"Expected exactly one official Mol boundary, received {len(matches)}")

    source_feature = matches[0]
    properties = dict(source_feature.get("properties") or {})
    if str(properties.get("NISCODE")) != MUNICIPALITY_NIS_CODE:
        raise RuntimeError(
            f"Official Mol boundary NIS code drifted: expected {MUNICIPALITY_NIS_CODE}, received {properties.get('NISCODE')}"
        )
    boundary = normalize_polygonal(shape(source_feature.get("geometry")))
    if boundary is None or not boundary.is_valid:
        raise RuntimeError("Official Mol boundary is empty, non-polygonal or invalid")

    source_url = response.url
    properties.update(
        {
            "source_name": "vrbg",
            "source_feature_id": str(source_feature.get("id") or MUNICIPALITY_NIS_CODE),
            "layer_type": "municipality_boundary",
            "authority_level": "authoritative",
            "coverage_scope": "municipality",
            "municipality": MUNICIPALITY_NAME,
            "nis_code": MUNICIPALITY_NIS_CODE,
            "attribution": VRBG_ATTRIBUTION,
            "source_url": source_url,
        }
    )
    feature = {
        "type": "Feature",
        "id": str(source_feature.get("id") or f"Refgem.{MUNICIPALITY_NIS_CODE}"),
        "geometry": mapping(boundary),
        "properties": properties,
    }
    return feature, boundary, source_url


def iter_grb_pages(
    session: requests.Session,
    boundary_bounds: tuple[float, float, float, float],
    *,
    page_limit: int,
    timeout: int,
) -> Iterable[tuple[dict[str, Any], str]]:
    params = {
        "f": "application/geo+json",
        "limit": str(page_limit),
        "bbox": ",".join(f"{value:.8f}" for value in boundary_bounds),
    }
    url: str | None = GRB_GBG_ITEMS_URL
    seen_urls: set[str] = set()
    first_request = True
    while url:
        if url in seen_urls:
            raise RuntimeError(f"GRB pagination loop detected: {url}")
        seen_urls.add(url)
        response = session.get(url, params=params if first_request else None, timeout=timeout)
        first_request = False
        response.raise_for_status()
        payload = response.json()
        yield payload, response.url
        url = next_page_url(payload)


def build_municipality_buildings(
    pages: Iterable[tuple[dict[str, Any], str]],
    boundary,
    *,
    max_features: int,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
    features: list[dict[str, Any]] = []
    source_urls: list[str] = []
    seen_ids: set[str] = set()
    bbox_feature_count = 0
    outside_boundary_count = 0
    clipped_at_boundary_count = 0

    for page, source_url in pages:
        source_urls.append(source_url)
        for source_feature in page.get("features") or []:
            feature_id = str(source_feature.get("id") or "")
            if not feature_id:
                feature_id = hashlib.sha256(
                    json.dumps(source_feature.get("geometry"), sort_keys=True).encode("utf-8")
                ).hexdigest()
            if feature_id in seen_ids:
                continue
            seen_ids.add(feature_id)
            bbox_feature_count += 1

            source_geometry = normalize_polygonal(shape(source_feature.get("geometry")))
            if source_geometry is None or not source_geometry.intersects(boundary):
                outside_boundary_count += 1
                continue
            within_boundary = source_geometry.within(boundary)
            clipped_geometry = source_geometry if within_boundary else normalize_polygonal(source_geometry.intersection(boundary))
            if clipped_geometry is None:
                outside_boundary_count += 1
                continue
            if not within_boundary:
                clipped_at_boundary_count += 1
            if len(features) >= max_features:
                raise RuntimeError(
                    f"Mol contains more than the configured {max_features} GRB features; refusing a truncated municipality dataset"
                )

            properties = dict(source_feature.get("properties") or {})
            properties.update(
                {
                    "source_name": "grb",
                    "source_feature_id": feature_id,
                    "reference_layer_name": "buildings",
                    "layer_type": "building",
                    "authority_level": "authoritative",
                    "coverage_scope": "municipality",
                    "municipality": MUNICIPALITY_NAME,
                    "nis_code": MUNICIPALITY_NIS_CODE,
                    "clipped_to_municipality": not within_boundary,
                    "attribution": GRB_ATTRIBUTION,
                }
            )
            features.append(
                {
                    "type": "Feature",
                    "id": feature_id,
                    "geometry": mapping(clipped_geometry),
                    "properties": properties,
                }
            )

    if not features:
        raise RuntimeError("No GRB buildings intersect the official Mol municipality boundary")
    return features, {
        "pages_fetched": len(source_urls),
        "source_urls": source_urls,
        "bbox_feature_count": bbox_feature_count,
        "municipality_feature_count": len(features),
        "outside_boundary_count": outside_boundary_count,
        "clipped_at_boundary_count": clipped_at_boundary_count,
        "reference_truncated": False,
    }


def municipality_area_km2(boundary) -> float:
    transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
    return float(transform(transformer.transform, boundary).area / 1_000_000)


def prepare_artifacts(args: argparse.Namespace) -> tuple[Path, Path, dict[str, Any]]:
    output_dir: Path = args.output_dir
    output_dir.mkdir(parents=True, exist_ok=True)
    boundary_path = output_dir / BOUNDARY_FILENAME
    buildings_path = output_dir / BUILDINGS_FILENAME
    manifest_path = output_dir / MANIFEST_FILENAME

    if not args.force and boundary_path.exists() and buildings_path.exists() and manifest_path.exists():
        manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
        if manifest.get("status") == "complete" and int(manifest.get("municipality_feature_count") or 0) > 0:
            return boundary_path, buildings_path, manifest

    if args.page_limit <= 0 or args.max_features <= 0:
        raise RuntimeError("--page-limit and --max-features must be positive integers")

    with build_source_session() as session:
        boundary_feature, boundary, boundary_source_url = fetch_mol_boundary(session, args.request_timeout)
        buildings, building_summary = build_municipality_buildings(
            iter_grb_pages(
                session,
                boundary.bounds,
                page_limit=args.page_limit,
                timeout=args.request_timeout,
            ),
            boundary,
            max_features=args.max_features,
        )

    generated_at = utc_now()
    boundary_payload = {
        "type": "FeatureCollection",
        "name": "Official municipality boundary - Mol",
        "crs": GEOJSON_CRS,
        "features": [boundary_feature],
        "source": "Digitaal Vlaanderen VRBG OGC API Features collection Refgem",
        "source_url": boundary_source_url,
        "attribution": VRBG_ATTRIBUTION,
        "municipality": MUNICIPALITY_NAME,
        "nis_code": MUNICIPALITY_NIS_CODE,
        "coverage_scope": "municipality",
        "generated_at": generated_at,
    }
    buildings_payload = {
        "type": "FeatureCollection",
        "name": "GRB buildings - complete municipality Mol",
        "crs": GEOJSON_CRS,
        "features": buildings,
        "source": "Digitaal Vlaanderen GRB OGC API Features collection GBG",
        "source_url": building_summary["source_urls"][0],
        "attribution": GRB_ATTRIBUTION,
        "municipality": MUNICIPALITY_NAME,
        "nis_code": MUNICIPALITY_NIS_CODE,
        "coverage_scope": "municipality",
        "reference_truncated": False,
        "generated_at": generated_at,
    }
    write_json(boundary_path, boundary_payload)
    write_json(buildings_path, buildings_payload)

    manifest = {
        "schema_version": 1,
        "status": "complete",
        "generated_at": generated_at,
        "municipality": MUNICIPALITY_NAME,
        "nis_code": MUNICIPALITY_NIS_CODE,
        "area_km2": municipality_area_km2(boundary),
        "wgs84_bbox": list(boundary.bounds),
        "boundary_path": str(boundary_path),
        "boundary_sha256": sha256_file(boundary_path),
        "buildings_path": str(buildings_path),
        "buildings_sha256": sha256_file(buildings_path),
        "page_limit": args.page_limit,
        "max_features": args.max_features,
        "boundary_source_url": boundary_source_url,
        "attribution": {"boundary": VRBG_ATTRIBUTION, "buildings": GRB_ATTRIBUTION},
        **building_summary,
    }
    write_json(manifest_path, manifest, pretty=True)
    return boundary_path, buildings_path, manifest


def response_data(response: requests.Response) -> Any:
    try:
        payload = response.json()
    except ValueError as exc:
        raise RuntimeError(f"GeoIntel API returned non-JSON response ({response.status_code}): {response.text[:500]}") from exc
    if not response.ok:
        raise RuntimeError(f"GeoIntel API request failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:1000]}")
    if not isinstance(payload, dict) or "data" not in payload:
        raise RuntimeError("GeoIntel API response does not use the canonical data envelope")
    return payload["data"]


def find_or_create_project(session: requests.Session, base_url: str, project_name: str, timeout: int) -> dict[str, Any]:
    projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=timeout))
    existing = next((item for item in projects.get("items") or [] if item.get("name") == project_name), None)
    if existing:
        return existing
    return response_data(
        session.post(
            f"{base_url}/api/v1/projects",
            json={
                "name": project_name,
                "description": (
                    "Complete municipality workspace for Mol using the official Digitaal Vlaanderen municipality boundary "
                    "and the full GRB GBG building reference layer clipped to NIS 13025."
                ),
                "region": PROJECT_REGION,
            },
            timeout=timeout,
        )
    )


def find_or_create_area(
    session: requests.Session,
    base_url: str,
    project_id: str,
    boundary_path: Path,
    timeout: int,
) -> dict[str, Any]:
    areas = response_data(
        session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=timeout)
    )
    existing = next((item for item in areas.get("items") or [] if item.get("name") == AREA_NAME), None)
    if existing:
        return existing
    boundary_payload = json.loads(boundary_path.read_text(encoding="utf-8"))
    geometry = boundary_payload["features"][0]["geometry"]
    return response_data(
        session.post(
            f"{base_url}/api/v1/projects/{project_id}/areas",
            json={"name": AREA_NAME, "crs": "EPSG:4326", "geometry": geometry},
            timeout=timeout,
        )
    )


def upload_dataset(
    session: requests.Session,
    base_url: str,
    project_id: str,
    area_id: str,
    path: Path,
    *,
    dataset_role: str,
    source_name: str,
    reference_layer_name: str | None,
    source_metadata: dict[str, Any],
    provenance_metadata: dict[str, Any],
    temporal_series_key: str,
    observed_at: str,
    timeout: int,
) -> dict[str, Any]:
    form = {
        "dataset_type": "vector",
        "source": "operator_official_import",
        "dataset_role": dataset_role,
        "source_name": source_name,
        "source_metadata_json": json.dumps(source_metadata, ensure_ascii=False),
        "provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False),
        "area_id": area_id,
        "temporal_series_key": temporal_series_key,
        "observed_at": observed_at,
        "temporal_granularity": "snapshot",
        "source_version": observed_at[:10],
    }
    if reference_layer_name:
        form["reference_layer_name"] = reference_layer_name
    with path.open("rb") as handle:
        response = session.post(
            f"{base_url}/api/v1/projects/{project_id}/datasets/upload",
            data=form,
            files={"file": (path.name, handle, "application/geo+json")},
            timeout=timeout,
        )
    return response_data(response)


def ensure_temporal_metadata(
    session: requests.Session,
    base_url: str,
    project_id: str,
    dataset: dict[str, Any],
    *,
    temporal_series_key: str,
    observed_at: str,
    timeout: int,
) -> dict[str, Any]:
    if dataset.get("temporal_series_key") == temporal_series_key and dataset.get("observed_at"):
        return dataset
    return response_data(
        session.patch(
            f"{base_url}/api/v1/projects/{project_id}/datasets/{dataset['id']}/temporal",
            json={
                "temporal_series_key": temporal_series_key,
                "observed_at": observed_at,
                "temporal_granularity": "snapshot",
                "source_version": observed_at[:10],
            },
            timeout=timeout,
        )
    )


def provision_workspace(
    args: argparse.Namespace,
    boundary_path: Path,
    buildings_path: Path,
    manifest: dict[str, Any],
) -> dict[str, Any]:
    base_url = args.base_url.rstrip("/")
    with requests.Session() as session:
        project = find_or_create_project(session, base_url, args.project_name, args.import_timeout)
        project_id = str(project["id"])
        area = find_or_create_area(session, base_url, project_id, boundary_path, args.import_timeout)
        area_id = str(area["id"])
        dataset_list = response_data(
            session.get(
                f"{base_url}/api/v1/projects/{project_id}/datasets",
                params={"limit": 200},
                timeout=args.import_timeout,
            )
        )
        datasets = list(dataset_list.get("items") or [])

        common_provenance = {
            "operator_tool": "provision_mol_municipality_workspace.py",
            "operator_explicit_fetch": True,
            "municipality": MUNICIPALITY_NAME,
            "nis_code": MUNICIPALITY_NIS_CODE,
            "coverage_scope": "municipality",
            "manifest_path": str(args.output_dir / MANIFEST_FILENAME),
            "manifest_generated_at": manifest["generated_at"],
            "reference_truncated": False,
        }

        building_dataset = next((item for item in datasets if item.get("original_filename") == buildings_path.name), None)
        if not building_dataset:
            building_dataset = upload_dataset(
                session,
                base_url,
                project_id,
                area_id,
                buildings_path,
                dataset_role="reference",
                source_name="grb",
                reference_layer_name="buildings",
                source_metadata={
                    "provider": "Digitaal Vlaanderen",
                    "collection": "GRB/GBG",
                    "authority_level": "authoritative",
                    "coverage_scope": "municipality",
                    "municipality": MUNICIPALITY_NAME,
                    "nis_code": MUNICIPALITY_NIS_CODE,
                    "feature_count": manifest["municipality_feature_count"],
                    "pages_fetched": manifest["pages_fetched"],
                    "attribution": GRB_ATTRIBUTION,
                },
                provenance_metadata={
                    **common_provenance,
                    "source_url": GRB_GBG_ITEMS_URL,
                    "artifact_sha256": manifest["buildings_sha256"],
                },
                temporal_series_key="grb:buildings:mol",
                observed_at=manifest["generated_at"],
                timeout=args.import_timeout,
            )
        else:
            building_dataset = ensure_temporal_metadata(
                session,
                base_url,
                project_id,
                building_dataset,
                temporal_series_key="grb:buildings:mol",
                observed_at=manifest["generated_at"],
                timeout=args.import_timeout,
            )

        boundary_dataset = next((item for item in datasets if item.get("original_filename") == boundary_path.name), None)
        if not boundary_dataset:
            boundary_dataset = upload_dataset(
                session,
                base_url,
                project_id,
                area_id,
                boundary_path,
                dataset_role="source",
                source_name="vrbg",
                reference_layer_name=None,
                source_metadata={
                    "provider": "Digitaal Vlaanderen",
                    "collection": "VRBG/Refgem",
                    "authority_level": "authoritative",
                    "coverage_scope": "municipality",
                    "layer_type": "municipality_boundary",
                    "municipality": MUNICIPALITY_NAME,
                    "nis_code": MUNICIPALITY_NIS_CODE,
                    "attribution": VRBG_ATTRIBUTION,
                },
                provenance_metadata={
                    **common_provenance,
                    "source_url": manifest["boundary_source_url"],
                    "artifact_sha256": manifest["boundary_sha256"],
                },
                temporal_series_key="vrbg:municipality-boundary:mol",
                observed_at=manifest["generated_at"],
                timeout=args.import_timeout,
            )
        else:
            boundary_dataset = ensure_temporal_metadata(
                session,
                base_url,
                project_id,
                boundary_dataset,
                temporal_series_key="vrbg:municipality-boundary:mol",
                observed_at=manifest["generated_at"],
                timeout=args.import_timeout,
            )

    return {
        "project_id": project_id,
        "project_name": project.get("name"),
        "area_id": area_id,
        "area_name": area.get("name"),
        "boundary_dataset_id": str(boundary_dataset["id"]),
        "boundary_feature_count": boundary_dataset.get("feature_count"),
        "building_dataset_id": str(building_dataset["id"]),
        "building_feature_count": building_dataset.get("feature_count"),
    }


def main() -> int:
    args = parse_args()
    try:
        boundary_path, buildings_path, manifest = prepare_artifacts(args)
        workspace = None if args.fetch_only else provision_workspace(args, boundary_path, buildings_path, manifest)
    except (OSError, RuntimeError, requests.RequestException, ValueError, KeyError) as exc:
        print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
        return 1

    result = {
        "status": "ok",
        "mode": "fetch_only" if args.fetch_only else "provisioned",
        "manifest_path": str(args.output_dir / MANIFEST_FILENAME),
        "municipality": MUNICIPALITY_NAME,
        "nis_code": MUNICIPALITY_NIS_CODE,
        "area_km2": manifest["area_km2"],
        "wgs84_bbox": manifest["wgs84_bbox"],
        "municipality_feature_count": manifest["municipality_feature_count"],
        "pages_fetched": manifest["pages_fetched"],
        "reference_truncated": manifest["reference_truncated"],
        "workspace": workspace,
    }
    print(json.dumps(result, ensure_ascii=False, indent=2, default=str))
    return 0


if __name__ == "__main__":
    sys.exit(main())