Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(provenance): govern source snapshots and data inputs

Browse Source
This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/vector_feature_service.py
+130 -5
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
import math
from pathlib import Path
from typing import Any, Iterable
from uuid import UUID
@@ -8,6 +9,7 @@ from uuid import UUID
from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope
from geoalchemy2.shape import from_shape
from geoalchemy2.shape import to_shape
from pyproj import CRS, Transformer
from shapely.geometry import box, mapping, shape
from shapely.ops import transform as transform_geometry
from shapely.validation import make_valid
@@ -268,7 +270,14 @@ class VectorFeatureService:
return ("municipality", municipality) if municipality else None
@staticmethod
def _feature_row(dataset_id: UUID, feature: dict[str, Any], index: int, feature_class: str | None) -> VectorFeature | None:
def _feature_row(
dataset_id: UUID,
feature: dict[str, Any],
index: int,
feature_class: str | None,
*,
source_crs: str = "EPSG:4326",
) -> VectorFeature | None:
geometry_payload = feature.get("geometry")
if geometry_payload is None:
return None
@@ -282,8 +291,7 @@ class VectorFeatureService:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
if geometry.has_z:
geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
geometry = VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index)
properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
source_feature_id = feature.get("id")
@@ -298,6 +306,109 @@ class VectorFeatureService:
geometry=from_shape(geometry, srid=4326),
)
@staticmethod
def _canonical_geometry(geometry: Any, *, source_crs: str, index: int):
"""Transform one source geometry to canonical EPSG:4326 safely."""
if geometry.has_z:
geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
# VectorFeature is deliberately canonical WGS84 storage. Treating
# Lambert or another source CRS as EPSG:4326 produces geometries that
# look syntactically valid but are spatially wrong. All governed
# import callers therefore pass the declared source CRS; the default
# only preserves compatibility for legacy, already-WGS84 call sites.
try:
parsed_source_crs = CRS.from_user_input(source_crs)
target_crs = CRS.from_epsg(4326)
except Exception as exc:
raise AppError(
code="INVALID_DATASET_CRS",
message=f"Invalid source CRS for vector feature at index {index}",
details={"source_crs": source_crs},
status_code=400,
) from exc
if not parsed_source_crs.equals(target_crs):
try:
transformer = Transformer.from_crs(parsed_source_crs, target_crs, always_xy=True)
geometry = transform_geometry(transformer.transform, geometry)
except Exception as exc:
raise AppError(
code="VECTOR_CRS_TRANSFORMATION_FAILED",
message=f"Could not transform vector feature at index {index} to EPSG:4326",
details={"source_crs": source_crs},
status_code=400,
) from exc
if geometry.is_empty or not geometry.is_valid:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(
code="INVALID_GEOMETRY",
message=f"Invalid transformed feature geometry at index {index}",
status_code=400,
)
min_x, min_y, max_x, max_y = geometry.bounds
if (
not all(math.isfinite(value) for value in (min_x, min_y, max_x, max_y))
or min_x < -180
or max_x > 180
or min_y < -90
or max_y > 90
):
raise AppError(
code="VECTOR_GEOMETRY_OUTSIDE_EPSG4326",
message=f"Transformed feature geometry at index {index} is outside EPSG:4326 bounds",
details={"source_crs": source_crs, "bounds": [min_x, min_y, max_x, max_y]},
status_code=400,
)
return geometry
@staticmethod
def canonicalize_geojson_payload(payload: dict[str, Any], *, source_crs: str) -> dict[str, Any]:
"""Return a canonical-WGS84 feature collection without losing source attributes.
Callers use this payload for validation, spatial indexing and
map-safe consumption storage. A non-canonical source file, when
retained, belongs to explicit provenance evidence rather than the
Dataset consumption path; no implicit CRS assumption is recorded.
"""
features = payload.get("features")
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
raise AppError(code="INVALID_GEOJSON", message="GeoJSON payload must be a FeatureCollection", status_code=400)
canonical_features: list[dict[str, Any]] = []
for index, feature in enumerate(features):
if not isinstance(feature, dict):
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
canonical_feature = dict(feature)
geometry_payload = feature.get("geometry")
if geometry_payload is not None:
try:
geometry = shape(geometry_payload)
except Exception as exc:
raise AppError(
code="INVALID_GEOJSON",
message=f"Invalid feature geometry at index {index}",
status_code=400,
) from exc
if not geometry.is_empty:
if not geometry.is_valid:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(
code="INVALID_GEOMETRY",
message=f"Invalid feature geometry at index {index}",
status_code=400,
)
canonical_feature["geometry"] = mapping(
VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index)
)
canonical_features.append(canonical_feature)
return {
**{key: value for key, value in payload.items() if key not in {"crs", "features"}},
"type": "FeatureCollection",
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
"features": canonical_features,
}
@staticmethod
def _normalize_selection_bbox(bbox: dict[str, Any]) -> dict[str, float | str]:
try:
@@ -882,6 +993,7 @@ class VectorFeatureService:
feature_class: str | None = None,
*,
commit: bool = True,
source_crs: str = "EPSG:4326",
) -> list[VectorFeature]:
features = payload.get("features")
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
@@ -891,7 +1003,13 @@ class VectorFeatureService:
for index, feature in enumerate(features):
if not isinstance(feature, dict):
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
row = VectorFeatureService._feature_row(
dataset_id,
feature,
index,
feature_class,
source_crs=source_crs,
)
if row is None:
continue
db.add(row)
@@ -910,6 +1028,7 @@ class VectorFeatureService:
feature_class: str | None = None,
*,
batch_size: int = 1000,
source_crs: str = "EPSG:4326",
) -> int:
if batch_size <= 0:
raise ValueError("batch_size must be positive")
@@ -942,7 +1061,13 @@ class VectorFeatureService:
message=f"Feature {index} in {path.name} must be an object",
status_code=400,
)
row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
row = VectorFeatureService._feature_row(
dataset_id,
feature,
index,
feature_class,
source_crs=source_crs,
)
if row is None:
continue
if row.source_feature_id: