fix(platform): govern geospatial analysis and raster handoffs

backend/app/services/dataset_service.py

@@ -17,6 +17,7 @@ from shapely.geometry import MultiPoint, shape
 from sqlalchemy.orm import Session
 
 from app.core.errors import AppError
+from app.core.config import get_settings
 from app.models import Area, Dataset, DatasetVersion, Project
 from app.services.data_contract_validation import (
     ContractKind,
@@ -476,6 +477,29 @@ class DatasetService:
             content_type=content_type,
         )
 
+    @classmethod
+    def _persist_vector_source_evidence_from_path(
+        cls,
+        *,
+        project_id: UUID,
+        dataset_id: UUID,
+        original_filename: str,
+        source_path: str | Path,
+        content_type: str | None,
+    ) -> dict[str, Any]:
+        safe_filename = StorageService._safe_filename(original_filename)
+        evidence_path = (
+            StorageService.dataset_root(str(project_id), str(dataset_id), "vector")
+            / "provenance"
+            / f"{dataset_id}_source_{safe_filename}"
+        )
+        return StorageService.persist_file_from_path(
+            str(evidence_path),
+            source_path,
+            original_filename=safe_filename,
+            content_type=content_type,
+        )
+
     @classmethod
     def _record_vector_source_evidence(
         cls,
@@ -853,6 +877,52 @@ class DatasetService:
             raise AppError(code="INVALID_UPLOAD", message="Missing file name", status_code=400)
         return filename
 
+    @staticmethod
+    async def _stage_upload(
+        *,
+        project_id: UUID,
+        dataset_id: uuid.UUID,
+        dataset_type: str,
+        filename: str,
+        file: UploadFile,
+    ) -> dict[str, Any]:
+        settings = get_settings()
+        max_upload_mb = int(settings.max_upload_mb)
+        if DatasetService._canonical_dataset_type(dataset_type) == "vector":
+            max_upload_mb = min(max_upload_mb, int(settings.max_in_memory_vector_mb))
+        return await StorageService.persist_upload_file(
+            project_id=str(project_id),
+            dataset_id=str(dataset_id),
+            dataset_type=dataset_type,
+            original_filename=filename,
+            upload=file,
+            content_type=file.content_type,
+            max_bytes=max_upload_mb * 1024 * 1024,
+        )
+
+    @staticmethod
+    def _read_staged_vector_bytes(storage_info: dict[str, Any]) -> bytes:
+        settings = get_settings()
+        max_bytes = min(
+            int(settings.max_upload_mb),
+            int(settings.max_in_memory_vector_mb),
+        ) * 1024 * 1024
+        path = Path(str(storage_info["storage_path"]))
+        with path.open("rb") as stream:
+            content = stream.read(max_bytes + 1)
+        if len(content) > max_bytes:
+            StorageService.remove_dataset_file(str(path))
+            raise AppError(
+                code="UPLOAD_TOO_LARGE",
+                message="Vector upload exceeds the bounded in-memory parsing limit.",
+                details={
+                    "max_bytes": max_bytes,
+                    "max_in_memory_vector_mb": max_bytes // (1024 * 1024),
+                },
+                status_code=413,
+            )
+        return content
+
     @staticmethod
     def list_datasets(db: Session, project_id: UUID, limit: int = 50, offset: int = 0) -> tuple[list[DatasetCreateResponse], int]:
         total = db.query(Dataset).filter(Dataset.project_id == project_id).count()
@@ -1135,15 +1205,15 @@ class DatasetService:
                 status_code=415,
             )
 
-        raw = await file.read()
-        storage_info = StorageService.persist_dataset_file(
-            project_id=str(project_id),
-            dataset_id=str(dataset_id := uuid.uuid4()),
+        dataset_id = uuid.uuid4()
+        storage_info = await DatasetService._stage_upload(
+            project_id=project_id,
+            dataset_id=dataset_id,
             dataset_type=canonical_type,
-            original_filename=filename,
-            content=raw,
-            content_type=file.content_type,
+            filename=filename,
+            file=file,
         )
+        raw = DatasetService._read_staged_vector_bytes(storage_info) if canonical_type == "vector" else None
 
         metadata: dict[str, Any] = {}
         vector_payload: dict[str, Any] | None = None
@@ -1151,6 +1221,7 @@ class DatasetService:
         try:
             status = "validating"
             if canonical_type == "vector":
+                assert raw is not None
                 try:
                     text = raw.decode("utf-8")
                 except UnicodeDecodeError as exc:
@@ -1319,8 +1390,15 @@ class DatasetService:
             temporal_granularity=temporal_granularity,
             source_version=source_version,
         )
-        raw = await file.read()
-        checksum_sha256 = StorageService.calculate_checksum_sha256(raw)
+        dataset_id = uuid.uuid4()
+        storage_info = await DatasetService._stage_upload(
+            project_id=project_id,
+            dataset_id=dataset_id,
+            dataset_type=canonical_type,
+            filename=filename,
+            file=file,
+        )
+        checksum_sha256 = storage_info["checksum_sha256"]
         ingest_key = DatasetService._ingest_key(
             project_id=project_id,
             source_key="manual",
@@ -1333,6 +1411,7 @@ class DatasetService:
         )
         existing = DatasetService._find_existing_ingest(db, project_id, ingest_key)
         if existing is not None:
+            StorageService.remove_dataset_file(storage_info["storage_path"])
             return DatasetService._to_response(existing)
 
         raw_source_metadata = dict(source_metadata or {})
@@ -1361,8 +1440,7 @@ class DatasetService:
             }
         )
 
-        dataset_id = uuid.uuid4()
-        storage_info: dict[str, Any] | None = None
+        raw = DatasetService._read_staged_vector_bytes(storage_info) if canonical_type == "vector" else None
         storage_content = raw
         source_evidence: dict[str, Any] | None = None
         imported_at = datetime.now(timezone.utc)
@@ -1372,6 +1450,7 @@ class DatasetService:
         parser_error: tuple[str, str] | None = None
         try:
             if canonical_type == "vector":
+                assert raw is not None
                 try:
                     payload = json.loads(raw.decode("utf-8"))
                 except UnicodeDecodeError as exc:
@@ -1394,22 +1473,22 @@ class DatasetService:
                 )
                 if DatasetService._vector_storage_requires_canonicalization(source_crs):
                     storage_content = DatasetService._canonical_vector_storage_bytes(canonical_vector_payload)
-                    source_evidence = DatasetService._persist_vector_source_evidence(
+                    source_evidence = DatasetService._persist_vector_source_evidence_from_path(
                         project_id=project_id,
                         dataset_id=dataset_id,
                         original_filename=filename,
-                        content=raw,
+                        source_path=storage_info["storage_path"],
+                        content_type=file.content_type,
+                    )
+                    storage_info = StorageService.persist_dataset_file(
+                        project_id=str(project_id),
+                        dataset_id=str(dataset_id),
+                        dataset_type=canonical_type,
+                        original_filename=filename,
+                        content=storage_content,
                         content_type=file.content_type,
                     )
             else:
-                storage_info = StorageService.persist_dataset_file(
-                    project_id=str(project_id),
-                    dataset_id=str(dataset_id),
-                    dataset_type=canonical_type,
-                    original_filename=filename,
-                    content=raw,
-                    content_type=file.content_type,
-                )
                 metadata = extract_raster_metadata(storage_info["storage_path"])
                 metadata["dataset_type"] = "raster"
                 source_crs = metadata.get("crs")
@@ -1422,16 +1501,7 @@ class DatasetService:
                 "processing_code": code,
             }
 
-        if storage_info is None:
-            storage_info = StorageService.persist_dataset_file(
-                project_id=str(project_id),
-                dataset_id=str(dataset_id),
-                dataset_type=canonical_type,
-                original_filename=filename,
-                content=storage_content,
-                content_type=file.content_type,
-            )
-        computed_storage_checksum_sha256 = StorageService.calculate_checksum_sha256(storage_content)
+        computed_storage_checksum_sha256 = storage_info["checksum_sha256"]
         if source_evidence is not None:
             resolved_source_crs = source_crs or DatasetService.CANONICAL_VECTOR_CRS
             DatasetService._record_vector_source_evidence(
@@ -1502,7 +1572,7 @@ class DatasetService:
                         feature_collection=canonical_vector_payload or {"type": "FeatureCollection", "features": []},
                         checksum_sha256=storage_info["checksum_sha256"],
                         computed_checksum_sha256=computed_storage_checksum_sha256,
-                        content=storage_content,
+                        content=None,
                         source_registry_id=str(source_registry.id),
                         source_snapshot_id=str(source_snapshot.id),
                         imported_at=imported_at,
@@ -1541,8 +1611,8 @@ class DatasetService:
                     bounds=DatasetService._extract_raster_bounds_json(metadata),
                     resolution=resolution,
                     checksum_sha256=storage_info["checksum_sha256"],
-                    computed_checksum_sha256=checksum_sha256,
-                    content=raw,
+                    computed_checksum_sha256=storage_info["checksum_sha256"],
+                    content=None,
                     source_registry_id=str(source_registry.id),
                     source_snapshot_id=str(source_snapshot.id),
                     imported_at=imported_at,