Aggregate regional bathymetry partitions

2026-07-17 17:08:35 +02:00
parent 24247746c1
commit 4fd6c06f5c
17 changed files with 661 additions and 29 deletions
@@ -137,6 +137,8 @@ SELECTION_FILTER_VALUE_ALIASES: dict[tuple[str, str], tuple[str, ...]] = {


 class VectorFeatureService:
+    MAX_VECTOR_PARTITIONS = 500
+
    @staticmethod
    def _expanded_selection_filter_values(filter_property: str, filter_values: list[Any]) -> list[str]:
        expanded: list[str] = []
@@ -307,6 +309,178 @@ class VectorFeatureService:

        return {"min_x": min_x, "min_y": min_y, "max_x": max_x, "max_y": max_y, "crs": "EPSG:4326"}

+    @staticmethod
+    def _dataset_bbox_intersects(
+        dataset: Dataset,
+        bbox: dict[str, float | str],
+    ) -> bool:
+        source_metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
+        source_bbox = source_metadata.get("bbox_epsg4326")
+        if not isinstance(source_bbox, list) or len(source_bbox) != 4:
+            return True
+        try:
+            min_x, min_y, max_x, max_y = (float(value) for value in source_bbox)
+        except (TypeError, ValueError):
+            return True
+        return not (
+            max_x <= float(bbox["min_x"])
+            or min_x >= float(bbox["max_x"])
+            or max_y <= float(bbox["min_y"])
+            or min_y >= float(bbox["max_y"])
+        )
+
+    @staticmethod
+    def _latest_complete_partition_manifest(
+        datasets: Iterable[Dataset],
+        *,
+        source_name: str,
+        partition_scope_key: str,
+    ) -> list[Dataset]:
+        groups: dict[str, list[Dataset]] = {}
+        for dataset in datasets:
+            source_metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
+            manifest_sha256 = str(source_metadata.get("partition_manifest_sha256") or "")
+            if (
+                dataset.source_name != source_name
+                or dataset.dataset_type not in {"vector", "geojson"}
+                or dataset.status != "ready"
+                or dataset.area_id is None
+                or source_metadata.get("regional_partitions_complete") is not True
+                or source_metadata.get("partition_scope_key") != partition_scope_key
+                or len(manifest_sha256) != 64
+            ):
+                continue
+            groups.setdefault(manifest_sha256, []).append(dataset)
+
+        complete_groups: list[list[Dataset]] = []
+        for group in groups.values():
+            area_ids = {dataset.area_id for dataset in group}
+            expected_data_count = max(
+                int((dataset.source_metadata or {}).get("data_partition_count") or 0)
+                for dataset in group
+            )
+            if expected_data_count > 0 and len(group) == expected_data_count and len(area_ids) == len(group):
+                complete_groups.append(group)
+
+        if not complete_groups:
+            return []
+
+        def manifest_priority(group: list[Dataset]) -> tuple[str, int, str]:
+            observed_at = max(
+                str((dataset.source_metadata or {}).get("partition_manifest_observed_at") or "")
+                for dataset in group
+            )
+            manifest_sha256 = str((group[0].source_metadata or {}).get("partition_manifest_sha256") or "")
+            return observed_at, len(group), manifest_sha256
+
+        selected = max(complete_groups, key=manifest_priority)
+        return sorted(selected, key=lambda dataset: (str(dataset.area_id), str(dataset.id)))
+
+    @staticmethod
+    def select_partitioned_features_by_bbox(
+        db,
+        *,
+        project_id: UUID,
+        source_name: str,
+        partition_scope_key: str,
+        bbox: dict[str, Any],
+        limit: int = 100,
+        selection_geometry: Any | None = None,
+        selection_area_id: UUID | None = None,
+        partition_area_id: UUID | None = None,
+    ) -> dict[str, Any]:
+        normalized_bbox = VectorFeatureService._normalize_selection_bbox(bbox)
+        safe_limit = max(1, min(int(limit), 1000))
+        project_datasets = (
+            db.query(Dataset)
+            .filter(
+                Dataset.project_id == project_id,
+                Dataset.source_name == source_name,
+                Dataset.status == "ready",
+            )
+            .all()
+        )
+        manifest_datasets = VectorFeatureService._latest_complete_partition_manifest(
+            project_datasets,
+            source_name=source_name,
+            partition_scope_key=partition_scope_key,
+        )
+        if not manifest_datasets:
+            raise AppError(
+                code="VECTOR_PARTITIONS_NOT_READY",
+                message="No complete persisted vector partition manifest is available",
+                details={"source_name": source_name, "partition_scope_key": partition_scope_key},
+                status_code=409,
+            )
+        if len(manifest_datasets) > VectorFeatureService.MAX_VECTOR_PARTITIONS:
+            raise AppError(
+                code="VECTOR_PARTITION_LIMIT_EXCEEDED",
+                message="The complete vector partition manifest exceeds the safety limit",
+                details={
+                    "partition_count": len(manifest_datasets),
+                    "max_partitions": VectorFeatureService.MAX_VECTOR_PARTITIONS,
+                },
+                status_code=422,
+            )
+
+        scoped_datasets = [
+            dataset
+            for dataset in manifest_datasets
+            if (partition_area_id is None or dataset.area_id == partition_area_id)
+            and VectorFeatureService._dataset_bbox_intersects(dataset, normalized_bbox)
+        ]
+        dataset_ids = [dataset.id for dataset in scoped_datasets]
+        representative = scoped_datasets[0] if scoped_datasets else manifest_datasets[0]
+        selection_shape = selection_geometry
+        if selection_shape is None:
+            selection_shape = ST_MakeEnvelope(
+                normalized_bbox["min_x"],
+                normalized_bbox["min_y"],
+                normalized_bbox["max_x"],
+                normalized_bbox["max_y"],
+                4326,
+            )
+
+        query = db.query(VectorFeature).filter(
+            VectorFeature.dataset_id.in_(dataset_ids),
+            ST_Intersects(VectorFeature.geometry, selection_shape),
+        )
+        total_feature_count = int(query.count())
+        rows = (
+            query.order_by(VectorFeature.created_at.asc(), VectorFeature.id.asc())
+            .limit(safe_limit + 1)
+            .all()
+        )
+        features = [
+            VectorFeatureService._row_to_geojson_feature(row)
+            for row in rows[:safe_limit]
+        ]
+        result = {
+            "selection_bbox": normalized_bbox,
+            "feature_count": len(features),
+            "total_feature_count": total_feature_count,
+            "limit": safe_limit,
+            "truncated": total_feature_count > safe_limit,
+            "geojson": {"type": "FeatureCollection", "features": features},
+            "partition_count": len(scoped_datasets),
+            "available_partition_count": len(manifest_datasets),
+            "partition_scope_key": partition_scope_key,
+            "source_name": source_name,
+            "dataset_ids": dataset_ids,
+        }
+        if selection_area_id is not None:
+            result["selection_area_id"] = str(selection_area_id)
+        if VectorFeatureService.supports_selection_summary(representative):
+            result["summary"] = VectorFeatureService.summarize_features_by_bbox(
+                db,
+                dataset=representative,
+                dataset_ids=dataset_ids,
+                bbox=normalized_bbox,
+                total_feature_count=total_feature_count,
+                selection_geometry=selection_shape,
+            )
+        return result
+
    @staticmethod
    def _row_to_geojson_feature(row: VectorFeature) -> dict[str, Any]:
        geometry_value = row.geometry
@@ -414,6 +588,7 @@ class VectorFeatureService:
        *,
        dataset: Dataset,
        bbox: dict[str, Any],
+        dataset_ids: list[UUID] | None = None,
        total_feature_count: int | None = None,
        selection_geometry: Any | None = None,
        full_dataset_area: bool = False,
@@ -429,7 +604,11 @@ class VectorFeatureService:
                normalized_bbox["max_y"],
                4326,
            )
-        selection_filter = (VectorFeature.dataset_id == dataset.id,)
+        selection_filter = (
+            (VectorFeature.dataset_id.in_(dataset_ids),)
+            if dataset_ids is not None
+            else (VectorFeature.dataset_id == dataset.id,)
+        )
        if preclipped_partition_filter is not None:
            partition_property, partition_value = preclipped_partition_filter
            selection_filter += (