feat: provision regional Kempen buildings

backend/app/services/vector_feature_service.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
-from typing import Any
+import json
+from pathlib import Path
+from typing import Any, Iterable
 from uuid import UUID
 
 from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope
@@ -17,6 +19,37 @@ from app.models import Dataset, VectorFeature
 
 
 class VectorFeatureService:
+    @staticmethod
+    def _feature_row(dataset_id: UUID, feature: dict[str, Any], index: int, feature_class: str | None) -> VectorFeature | None:
+        geometry_payload = feature.get("geometry")
+        if geometry_payload is None:
+            return None
+        try:
+            geometry = shape(geometry_payload)
+        except Exception as exc:
+            raise AppError(code="INVALID_GEOJSON", message=f"Invalid feature geometry at index {index}", status_code=400) from exc
+        if geometry.is_empty:
+            return None
+        if not geometry.is_valid:
+            geometry = make_valid(geometry)
+        if geometry.is_empty or not geometry.is_valid:
+            raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
+        if geometry.has_z:
+            geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
+
+        properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
+        source_feature_id = feature.get("id")
+        if source_feature_id is None:
+            source_feature_id = properties.get("id") or properties.get("source_feature_id")
+
+        return VectorFeature(
+            dataset_id=dataset_id,
+            feature_class=feature_class,
+            source_feature_id=str(source_feature_id) if source_feature_id is not None else None,
+            properties_json=properties,
+            geometry=from_shape(geometry, srid=4326),
+        )
+
     @staticmethod
     def _normalize_selection_bbox(bbox: dict[str, Any]) -> dict[str, float | str]:
         try:
@@ -263,34 +296,9 @@ class VectorFeatureService:
         for index, feature in enumerate(features):
             if not isinstance(feature, dict):
                 raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
-            geometry_payload = feature.get("geometry")
-            if geometry_payload is None:
+            row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
+            if row is None:
                 continue
-            try:
-                geometry = shape(geometry_payload)
-            except Exception as exc:
-                raise AppError(code="INVALID_GEOJSON", message=f"Invalid feature geometry at index {index}", status_code=400) from exc
-            if geometry.is_empty:
-                continue
-            if not geometry.is_valid:
-                geometry = make_valid(geometry)
-            if geometry.is_empty or not geometry.is_valid:
-                raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
-            if geometry.has_z:
-                geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
-
-            properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
-            source_feature_id = feature.get("id")
-            if source_feature_id is None:
-                source_feature_id = properties.get("id") or properties.get("source_feature_id")
-
-            row = VectorFeature(
-                dataset_id=dataset_id,
-                feature_class=feature_class,
-                source_feature_id=str(source_feature_id) if source_feature_id is not None else None,
-                properties_json=properties,
-                geometry=from_shape(geometry, srid=4326),
-            )
             db.add(row)
             persisted.append(row)
 
@@ -298,3 +306,69 @@ class VectorFeatureService:
             db.flush()
             db.commit()
         return persisted
+
+    @staticmethod
+    def persist_geojson_partitions(
+        db,
+        dataset_id: UUID,
+        partition_paths: Iterable[str | Path],
+        feature_class: str | None = None,
+        *,
+        batch_size: int = 1000,
+    ) -> int:
+        if batch_size <= 0:
+            raise ValueError("batch_size must be positive")
+
+        persisted_count = 0
+        source_feature_ids: set[str] = set()
+        for partition_path in partition_paths:
+            path = Path(partition_path)
+            try:
+                payload = json.loads(path.read_text(encoding="utf-8"))
+            except (OSError, json.JSONDecodeError) as exc:
+                raise AppError(
+                    code="INVALID_GEOJSON_PARTITION",
+                    message=f"Could not read GeoJSON partition {path.name}",
+                    status_code=400,
+                ) from exc
+            features = payload.get("features")
+            if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
+                raise AppError(
+                    code="INVALID_GEOJSON_PARTITION",
+                    message=f"GeoJSON partition {path.name} must be a FeatureCollection",
+                    status_code=400,
+                )
+
+            batch: list[VectorFeature] = []
+            for index, feature in enumerate(features):
+                if not isinstance(feature, dict):
+                    raise AppError(
+                        code="INVALID_GEOJSON_PARTITION",
+                        message=f"Feature {index} in {path.name} must be an object",
+                        status_code=400,
+                    )
+                row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
+                if row is None:
+                    continue
+                if row.source_feature_id:
+                    if row.source_feature_id in source_feature_ids:
+                        raise AppError(
+                            code="DUPLICATE_SOURCE_FEATURE",
+                            message=f"Duplicate source feature {row.source_feature_id} across regional partitions",
+                            status_code=400,
+                        )
+                    source_feature_ids.add(row.source_feature_id)
+                db.add(row)
+                batch.append(row)
+                persisted_count += 1
+                if len(batch) >= batch_size:
+                    db.flush()
+                    for persisted in batch:
+                        db.expunge(persisted)
+                    batch.clear()
+            if batch:
+                db.flush()
+                for persisted in batch:
+                    db.expunge(persisted)
+
+        return persisted_count