feat: provision regional Kempen buildings

backend/app/services/dataset_service.py

@@ -12,7 +12,7 @@ from fastapi import UploadFile
 from sqlalchemy.orm import Session
 
 from app.core.errors import AppError
-from app.models import Dataset, DatasetVersion, Project
+from app.models import Area, Dataset, DatasetVersion, Project
 from app.schemas.dataset import (
     DatasetCreateResponse,
     DatasetStorageResponse,
@@ -410,6 +410,136 @@ class DatasetService:
 
         return DatasetService._to_response(dataset)
 
+    @staticmethod
+    def import_partitioned_vector_artifact(
+        db: Session,
+        *,
+        project_id: UUID,
+        area_id: UUID,
+        artifact_path: str | Path,
+        partition_paths: list[str | Path],
+        original_filename: str,
+        source: str,
+        dataset_role: str,
+        source_name: str,
+        reference_layer_name: str | None,
+        metadata_json: dict[str, Any],
+        source_metadata: dict[str, Any],
+        provenance_metadata: dict[str, Any],
+        temporal_series_key: str,
+        observed_at: datetime,
+        temporal_granularity: str = "snapshot",
+        source_version: str | None = None,
+        batch_size: int = 1000,
+    ) -> DatasetCreateResponse:
+        if not db.get(Project, project_id):
+            raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
+        area = db.get(Area, area_id)
+        if not area:
+            raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
+        if area.project_id != project_id:
+            raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
+        if not partition_paths:
+            raise AppError(
+                code="INVALID_GEOJSON_PARTITIONS",
+                message="At least one GeoJSON partition is required",
+                status_code=400,
+            )
+
+        filename = DatasetService._validate_upload_filename(original_filename)
+        if DatasetService._extension_for_path(filename) not in DatasetService.VECTOR_EXTENSIONS:
+            raise AppError(code="INVALID_UPLOAD", message="Vector artifacts require .geojson or .json files", status_code=415)
+        normalized_role = DatasetService._normalize_dataset_role(dataset_role)
+        temporal = DatasetService._validate_temporal_metadata(
+            temporal_series_key=temporal_series_key,
+            observed_at=observed_at,
+            valid_from=observed_at,
+            valid_to=None,
+            temporal_granularity=temporal_granularity,
+            source_version=source_version,
+        )
+        metadata = dict(metadata_json)
+        expected_feature_count = int(metadata.get("feature_count") or 0)
+        if expected_feature_count <= 0:
+            raise AppError(
+                code="INVALID_GEOJSON_PARTITIONS",
+                message="Partition metadata must declare a positive feature_count",
+                status_code=400,
+            )
+
+        dataset_id = uuid.uuid4()
+        storage_info = StorageService.persist_dataset_file_from_path(
+            project_id=str(project_id),
+            dataset_id=str(dataset_id),
+            dataset_type="vector",
+            original_filename=filename,
+            source_path=artifact_path,
+            content_type="application/geo+json",
+        )
+        dataset = Dataset(
+            id=dataset_id,
+            project_id=project_id,
+            area_id=area_id,
+            name=filename,
+            dataset_type="vector",
+            source=source,
+            dataset_role=normalized_role,
+            source_name=source_name,
+            reference_layer_name=reference_layer_name if normalized_role == "reference" else None,
+            source_metadata=source_metadata,
+            provenance_metadata=provenance_metadata,
+            imported_at=datetime.now(timezone.utc),
+            **temporal,
+            storage_path=storage_info["storage_path"],
+            original_filename=storage_info["original_filename"],
+            stored_filename=storage_info["stored_filename"],
+            content_type=storage_info["content_type"],
+            size_bytes=storage_info["size_bytes"],
+            checksum_sha256=storage_info["checksum_sha256"],
+            crs=str(metadata.get("crs") or "EPSG:4326"),
+            bounds_json=metadata.get("bounds_json"),
+            metadata_json=metadata,
+            status="ready",
+        )
+        try:
+            db.add(dataset)
+            db.add(
+                DatasetVersion(
+                    dataset_id=dataset.id,
+                    version=1,
+                    storage_path=dataset.storage_path,
+                    source_version=dataset.source_version,
+                    observed_at=dataset.observed_at,
+                    valid_from=dataset.valid_from,
+                    checksum_sha256=dataset.checksum_sha256,
+                    source_metadata=dataset.source_metadata,
+                    provenance_metadata=dataset.provenance_metadata,
+                )
+            )
+            persisted_count = VectorFeatureService.persist_geojson_partitions(
+                db,
+                dataset.id,
+                partition_paths,
+                feature_class=reference_layer_name if normalized_role == "reference" else None,
+                batch_size=batch_size,
+            )
+            if persisted_count != expected_feature_count:
+                raise AppError(
+                    code="PARTITION_FEATURE_COUNT_MISMATCH",
+                    message=(
+                        f"Regional artifact declares {expected_feature_count} features but "
+                        f"{persisted_count} queryable features were indexed"
+                    ),
+                    status_code=400,
+                )
+            db.commit()
+            db.refresh(dataset)
+        except Exception:
+            db.rollback()
+            StorageService.remove_dataset_file(storage_info["storage_path"])
+            raise
+        return DatasetService._to_response(dataset)
+
     @staticmethod
     def refresh_metadata(db: Session, dataset_id: UUID) -> DatasetCreateResponse:
         dataset = DatasetService._get_dataset(db, dataset_id)

backend/app/services/storage_service.py

@@ -95,6 +95,39 @@ class StorageService:
         }
         return metadata
 
+    @staticmethod
+    def persist_dataset_file_from_path(
+        project_id: str,
+        dataset_id: str,
+        dataset_type: str,
+        original_filename: str,
+        source_path: str | Path,
+        content_type: str | None,
+    ) -> dict[str, Any]:
+        source = Path(source_path).resolve()
+        if not source.is_file():
+            raise FileNotFoundError(f"Dataset source artifact does not exist: {source}")
+
+        normalized_type = StorageService.normalize_dataset_type(dataset_type)
+        file_path = Path(StorageService.dataset_file_path(project_id, dataset_id, normalized_type, original_filename))
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+        digest = hashlib.sha256()
+        size_bytes = 0
+        with source.open("rb") as input_stream, file_path.open("wb") as output_stream:
+            for chunk in iter(lambda: input_stream.read(8 * 1024 * 1024), b""):
+                output_stream.write(chunk)
+                digest.update(chunk)
+                size_bytes += len(chunk)
+
+        return {
+            "original_filename": StorageService._safe_filename(original_filename),
+            "stored_filename": file_path.name,
+            "content_type": content_type or "application/octet-stream",
+            "size_bytes": size_bytes,
+            "checksum_sha256": digest.hexdigest(),
+            "storage_path": str(file_path),
+        }
+
     @staticmethod
     def persist_file(
         storage_path: str,

backend/app/services/vector_feature_service.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
-from typing import Any
+import json
+from pathlib import Path
+from typing import Any, Iterable
 from uuid import UUID
 
 from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope
@@ -17,6 +19,37 @@ from app.models import Dataset, VectorFeature
 
 
 class VectorFeatureService:
+    @staticmethod
+    def _feature_row(dataset_id: UUID, feature: dict[str, Any], index: int, feature_class: str | None) -> VectorFeature | None:
+        geometry_payload = feature.get("geometry")
+        if geometry_payload is None:
+            return None
+        try:
+            geometry = shape(geometry_payload)
+        except Exception as exc:
+            raise AppError(code="INVALID_GEOJSON", message=f"Invalid feature geometry at index {index}", status_code=400) from exc
+        if geometry.is_empty:
+            return None
+        if not geometry.is_valid:
+            geometry = make_valid(geometry)
+        if geometry.is_empty or not geometry.is_valid:
+            raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
+        if geometry.has_z:
+            geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
+
+        properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
+        source_feature_id = feature.get("id")
+        if source_feature_id is None:
+            source_feature_id = properties.get("id") or properties.get("source_feature_id")
+
+        return VectorFeature(
+            dataset_id=dataset_id,
+            feature_class=feature_class,
+            source_feature_id=str(source_feature_id) if source_feature_id is not None else None,
+            properties_json=properties,
+            geometry=from_shape(geometry, srid=4326),
+        )
+
     @staticmethod
     def _normalize_selection_bbox(bbox: dict[str, Any]) -> dict[str, float | str]:
         try:
@@ -263,34 +296,9 @@ class VectorFeatureService:
         for index, feature in enumerate(features):
             if not isinstance(feature, dict):
                 raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
-            geometry_payload = feature.get("geometry")
-            if geometry_payload is None:
+            row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
+            if row is None:
                 continue
-            try:
-                geometry = shape(geometry_payload)
-            except Exception as exc:
-                raise AppError(code="INVALID_GEOJSON", message=f"Invalid feature geometry at index {index}", status_code=400) from exc
-            if geometry.is_empty:
-                continue
-            if not geometry.is_valid:
-                geometry = make_valid(geometry)
-            if geometry.is_empty or not geometry.is_valid:
-                raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
-            if geometry.has_z:
-                geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
-
-            properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
-            source_feature_id = feature.get("id")
-            if source_feature_id is None:
-                source_feature_id = properties.get("id") or properties.get("source_feature_id")
-
-            row = VectorFeature(
-                dataset_id=dataset_id,
-                feature_class=feature_class,
-                source_feature_id=str(source_feature_id) if source_feature_id is not None else None,
-                properties_json=properties,
-                geometry=from_shape(geometry, srid=4326),
-            )
             db.add(row)
             persisted.append(row)
 
@@ -298,3 +306,69 @@ class VectorFeatureService:
             db.flush()
             db.commit()
         return persisted
+
+    @staticmethod
+    def persist_geojson_partitions(
+        db,
+        dataset_id: UUID,
+        partition_paths: Iterable[str | Path],
+        feature_class: str | None = None,
+        *,
+        batch_size: int = 1000,
+    ) -> int:
+        if batch_size <= 0:
+            raise ValueError("batch_size must be positive")
+
+        persisted_count = 0
+        source_feature_ids: set[str] = set()
+        for partition_path in partition_paths:
+            path = Path(partition_path)
+            try:
+                payload = json.loads(path.read_text(encoding="utf-8"))
+            except (OSError, json.JSONDecodeError) as exc:
+                raise AppError(
+                    code="INVALID_GEOJSON_PARTITION",
+                    message=f"Could not read GeoJSON partition {path.name}",
+                    status_code=400,
+                ) from exc
+            features = payload.get("features")
+            if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
+                raise AppError(
+                    code="INVALID_GEOJSON_PARTITION",
+                    message=f"GeoJSON partition {path.name} must be a FeatureCollection",
+                    status_code=400,
+                )
+
+            batch: list[VectorFeature] = []
+            for index, feature in enumerate(features):
+                if not isinstance(feature, dict):
+                    raise AppError(
+                        code="INVALID_GEOJSON_PARTITION",
+                        message=f"Feature {index} in {path.name} must be an object",
+                        status_code=400,
+                    )
+                row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
+                if row is None:
+                    continue
+                if row.source_feature_id:
+                    if row.source_feature_id in source_feature_ids:
+                        raise AppError(
+                            code="DUPLICATE_SOURCE_FEATURE",
+                            message=f"Duplicate source feature {row.source_feature_id} across regional partitions",
+                            status_code=400,
+                        )
+                    source_feature_ids.add(row.source_feature_id)
+                db.add(row)
+                batch.append(row)
+                persisted_count += 1
+                if len(batch) >= batch_size:
+                    db.flush()
+                    for persisted in batch:
+                        db.expunge(persisted)
+                    batch.clear()
+            if batch:
+                db.flush()
+                for persisted in batch:
+                    db.expunge(persisted)
+
+        return persisted_count