fix: make vector ingestion canonical and atomic
This commit is contained in:
@@ -376,32 +376,37 @@ class DatasetService:
|
||||
metadata_json=metadata,
|
||||
status=status,
|
||||
)
|
||||
db.add(dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
source_version=dataset.source_version,
|
||||
observed_at=dataset.observed_at,
|
||||
valid_from=dataset.valid_from,
|
||||
valid_to=dataset.valid_to,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
|
||||
if canonical_type == "vector" and vector_payload is not None and status == "ready":
|
||||
feature_class = reference_layer_name if normalized_role == "reference" else None
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
db=db,
|
||||
dataset_id=dataset.id,
|
||||
payload=vector_payload,
|
||||
feature_class=feature_class,
|
||||
try:
|
||||
db.add(dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
source_version=dataset.source_version,
|
||||
observed_at=dataset.observed_at,
|
||||
valid_from=dataset.valid_from,
|
||||
valid_to=dataset.valid_to,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
if canonical_type == "vector" and vector_payload is not None and status == "ready":
|
||||
feature_class = reference_layer_name if normalized_role == "reference" else None
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
db=db,
|
||||
dataset_id=dataset.id,
|
||||
payload=vector_payload,
|
||||
feature_class=feature_class,
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
StorageService.remove_dataset_file(storage_info["storage_path"])
|
||||
raise
|
||||
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ def parse_geojson_payload(raw_text: str | dict[str, Any]) -> dict[str, Any]:
|
||||
geometry_types: set[str] = set()
|
||||
geometries = []
|
||||
invalid_features = 0
|
||||
z_dimension_features = 0
|
||||
polygon_area_m2: float | None = None
|
||||
crs_assumed = None
|
||||
for feature in features:
|
||||
@@ -49,6 +50,8 @@ def parse_geojson_payload(raw_text: str | dict[str, Any]) -> dict[str, Any]:
|
||||
if not geom.is_valid:
|
||||
invalid_features += 1
|
||||
raise ValueError("Invalid geometry remains after repair")
|
||||
if geom.has_z:
|
||||
z_dimension_features += 1
|
||||
geometry_types.add(str(geom.geom_type))
|
||||
geometries.append(geom)
|
||||
|
||||
@@ -85,6 +88,8 @@ def parse_geojson_payload(raw_text: str | dict[str, Any]) -> dict[str, Any]:
|
||||
"bounds_json": bounds_json,
|
||||
"approximate_area_m2": polygon_area_m2,
|
||||
"invalid_features": invalid_features,
|
||||
"z_dimension_feature_count": z_dimension_features,
|
||||
"canonical_storage_dimension": "2D",
|
||||
"crs": crs,
|
||||
"crs_assumed": crs_assumed,
|
||||
"extracted_at": datetime.now(timezone.utc).isoformat(),
|
||||
|
||||
@@ -8,6 +8,7 @@ from geoalchemy2.shape import from_shape
|
||||
from geoalchemy2.shape import to_shape
|
||||
from shapely.geometry import mapping
|
||||
from shapely.geometry import shape
|
||||
from shapely.ops import transform as transform_geometry
|
||||
from shapely.validation import make_valid
|
||||
from sqlalchemy import Float, cast, func
|
||||
|
||||
@@ -265,6 +266,8 @@ class VectorFeatureService:
|
||||
geometry = make_valid(geometry)
|
||||
if geometry.is_empty or not geometry.is_valid:
|
||||
raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
|
||||
if geometry.has_z:
|
||||
geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
|
||||
|
||||
properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
|
||||
source_feature_id = feature.get("id")
|
||||
|
||||
Reference in New Issue
Block a user