diff --git a/CHANGELOG.md b/CHANGELOG.md index 92989ad3..e10be9d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ - Added explicit operator provisioners for official Statbel Mol population snapshots (2021-2025) and Digitaal Vlaanderen historical land-use snapshots (1778, 1873 and 1969); no source is fetched during application startup. - Preserved methodological honesty: partial statistical sectors are labelled area-weighted estimates, historical land-use identity changes are not fabricated and all source URLs, versions and processing limitations are persisted. - Serialized Tower startup and migration-smoke validation by waiting for container health, preventing concurrent Alembic upgrades from racing on the same PostGIS schema. +- Normalized valid source Z coordinates to the canonical 2D PostGIS vector store while retaining the original uploaded GeoJSON and reporting the source Z-feature count in metadata. +- Made vector dataset, version and feature persistence one transaction, with file cleanup on rollback, so an indexing error cannot leave a ready dataset without persisted features. ## Sprint 186 Map-first Mol geographic explorer (2026-07-14) diff --git a/backend/app/services/dataset_service.py b/backend/app/services/dataset_service.py index 35f3db52..6c171229 100644 --- a/backend/app/services/dataset_service.py +++ b/backend/app/services/dataset_service.py @@ -376,32 +376,37 @@ class DatasetService: metadata_json=metadata, status=status, ) - db.add(dataset) - db.add( - DatasetVersion( - dataset_id=dataset.id, - version=1, - storage_path=dataset.storage_path, - source_version=dataset.source_version, - observed_at=dataset.observed_at, - valid_from=dataset.valid_from, - valid_to=dataset.valid_to, - checksum_sha256=dataset.checksum_sha256, - source_metadata=dataset.source_metadata, - provenance_metadata=dataset.provenance_metadata, - ) - ) - db.commit() - db.refresh(dataset) - - if canonical_type == "vector" and vector_payload is not None and status == "ready": - feature_class = reference_layer_name if normalized_role == "reference" else None - VectorFeatureService.persist_geojson_features( - db=db, - dataset_id=dataset.id, - payload=vector_payload, - feature_class=feature_class, + try: + db.add(dataset) + db.add( + DatasetVersion( + dataset_id=dataset.id, + version=1, + storage_path=dataset.storage_path, + source_version=dataset.source_version, + observed_at=dataset.observed_at, + valid_from=dataset.valid_from, + valid_to=dataset.valid_to, + checksum_sha256=dataset.checksum_sha256, + source_metadata=dataset.source_metadata, + provenance_metadata=dataset.provenance_metadata, + ) ) + if canonical_type == "vector" and vector_payload is not None and status == "ready": + feature_class = reference_layer_name if normalized_role == "reference" else None + VectorFeatureService.persist_geojson_features( + db=db, + dataset_id=dataset.id, + payload=vector_payload, + feature_class=feature_class, + commit=False, + ) + db.commit() + db.refresh(dataset) + except Exception: + db.rollback() + StorageService.remove_dataset_file(storage_info["storage_path"]) + raise return DatasetService._to_response(dataset) diff --git a/backend/app/services/geojson_service.py b/backend/app/services/geojson_service.py index dc14c749..2eb13ccd 100644 --- a/backend/app/services/geojson_service.py +++ b/backend/app/services/geojson_service.py @@ -32,6 +32,7 @@ def parse_geojson_payload(raw_text: str | dict[str, Any]) -> dict[str, Any]: geometry_types: set[str] = set() geometries = [] invalid_features = 0 + z_dimension_features = 0 polygon_area_m2: float | None = None crs_assumed = None for feature in features: @@ -49,6 +50,8 @@ def parse_geojson_payload(raw_text: str | dict[str, Any]) -> dict[str, Any]: if not geom.is_valid: invalid_features += 1 raise ValueError("Invalid geometry remains after repair") + if geom.has_z: + z_dimension_features += 1 geometry_types.add(str(geom.geom_type)) geometries.append(geom) @@ -85,6 +88,8 @@ def parse_geojson_payload(raw_text: str | dict[str, Any]) -> dict[str, Any]: "bounds_json": bounds_json, "approximate_area_m2": polygon_area_m2, "invalid_features": invalid_features, + "z_dimension_feature_count": z_dimension_features, + "canonical_storage_dimension": "2D", "crs": crs, "crs_assumed": crs_assumed, "extracted_at": datetime.now(timezone.utc).isoformat(), diff --git a/backend/app/services/vector_feature_service.py b/backend/app/services/vector_feature_service.py index 21c19f9a..4742bba2 100644 --- a/backend/app/services/vector_feature_service.py +++ b/backend/app/services/vector_feature_service.py @@ -8,6 +8,7 @@ from geoalchemy2.shape import from_shape from geoalchemy2.shape import to_shape from shapely.geometry import mapping from shapely.geometry import shape +from shapely.ops import transform as transform_geometry from shapely.validation import make_valid from sqlalchemy import Float, cast, func @@ -265,6 +266,8 @@ class VectorFeatureService: geometry = make_valid(geometry) if geometry.is_empty or not geometry.is_valid: raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400) + if geometry.has_z: + geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry) properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {} source_feature_id = feature.get("id") diff --git a/backend/tests/test_geojson_dataset_service.py b/backend/tests/test_geojson_dataset_service.py index 42e876dc..7463debe 100644 --- a/backend/tests/test_geojson_dataset_service.py +++ b/backend/tests/test_geojson_dataset_service.py @@ -117,6 +117,24 @@ def test_parse_geojson_payload_returns_vector_metadata() -> None: assert metadata["approximate_area_m2"] >= 0.0 +def test_parse_geojson_payload_reports_z_dimension_for_canonical_2d_storage() -> None: + metadata = geojson_service.parse_geojson_payload( + { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [5.08, 51.18, 0.0]}, + "properties": {}, + } + ], + } + ) + + assert metadata["z_dimension_feature_count"] == 1 + assert metadata["canonical_storage_dimension"] == "2D" + + def test_parse_geojson_payload_rejects_invalid_geometry() -> None: payload = { "type": "FeatureCollection", diff --git a/backend/tests/test_sprint7a_persistence_foundation.py b/backend/tests/test_sprint7a_persistence_foundation.py index f55dc4dc..49abae0d 100644 --- a/backend/tests/test_sprint7a_persistence_foundation.py +++ b/backend/tests/test_sprint7a_persistence_foundation.py @@ -5,6 +5,7 @@ from pathlib import Path from uuid import uuid4 import pytest +from geoalchemy2.shape import to_shape from app.api.routes.qa import compare_candidate_with_reference from app.models import Dataset, Metric, Project, QualityCheck, VectorFeature @@ -22,6 +23,7 @@ class FakeSession: self.commits = 0 self.refreshes = [] self.flushes = 0 + self.rollbacks = 0 def get(self, model, item_id): return self.objects.get((model, item_id)) @@ -38,6 +40,9 @@ class FakeSession: def refresh(self, item) -> None: self.refreshes.append(item) + def rollback(self) -> None: + self.rollbacks += 1 + def test_vector_feature_service_persists_geojson_features_with_properties() -> None: db = FakeSession() @@ -84,6 +89,34 @@ def test_vector_feature_service_persists_geojson_features_with_properties() -> N assert db.refreshes == [] +def test_vector_feature_service_normalizes_source_z_coordinates_to_canonical_2d() -> None: + db = FakeSession() + persisted = VectorFeatureService.persist_geojson_features( + db=db, + dataset_id=uuid4(), + payload={ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "sector-3d", + "properties": {"population_total": 100}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [[5.0, 51.0, 0.0], [5.1, 51.0, 0.0], [5.1, 51.1, 0.0], [5.0, 51.0, 0.0]] + ], + }, + } + ], + }, + feature_class="population", + ) + + assert len(persisted) == 1 + assert to_shape(persisted[0].geometry).has_z is False + + def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None: project_id = uuid4() db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Geel")}) @@ -141,6 +174,53 @@ def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None: assert result.source_name == "manual" assert len(persisted_features) == 1 assert persisted_features[0].dataset_id == result.id + assert db.commits == 1 + + +def test_dataset_upload_rolls_back_dataset_and_file_when_vector_indexing_fails(monkeypatch, tmp_path) -> None: + project_id = uuid4() + db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Mol")}) + storage_path = tmp_path / "invalid.geojson" + storage_path.write_text("{}", encoding="utf-8") + + class Upload: + filename = "invalid.geojson" + content_type = "application/geo+json" + + async def read(self) -> bytes: + return b'{"type":"FeatureCollection","features":[]}' + + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_dataset_file", + lambda **_kwargs: { + "storage_path": str(storage_path), + "original_filename": "invalid.geojson", + "stored_filename": "invalid.geojson", + "content_type": "application/geo+json", + "size_bytes": 2, + "checksum_sha256": "0" * 64, + }, + ) + monkeypatch.setattr( + VectorFeatureService, + "persist_geojson_features", + lambda **_kwargs: (_ for _ in ()).throw(RuntimeError("PostGIS indexing failed")), + ) + + with pytest.raises(RuntimeError, match="PostGIS indexing failed"): + asyncio.run( + DatasetService.upload_dataset( + db=db, + project_id=project_id, + file=Upload(), + dataset_type="vector", + source="user_upload", + ) + ) + + assert db.commits == 0 + assert db.rollbacks == 1 + assert storage_path.exists() is False def test_quality_service_persists_quality_check_and_metrics() -> None: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 5265d042..0063cd14 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -7799,3 +7799,5 @@ Live deployment correction: - The first Tower rollout exposed a deployment race: all-in-one startup and `live_migration_smoke.sh` both began `alembic upgrade head` after PostgreSQL became reachable. - Startup committed head `202607140001`; the concurrent smoke transaction rolled back on a duplicate first column. Database contents and the successful migration remained healthy. - Both Tower deploy entry points now wait for the `geointel` container healthcheck, which includes completed startup migrations and backend readiness, before launching the independent migration smoke. +- The first Statbel upload exposed 3D sector coordinates (`Z=0`) against the canonical 2D PostGIS vector column. The source is valid; GeoIntel now preserves the original artifact, records the Z-feature count and explicitly drops Z only for the 2D query index. +- Vector upload persistence is now atomic across Dataset, DatasetVersion and VectorFeature rows, with storage cleanup on rollback. This prevents the failed-indexing orphan state observed during the live import. diff --git a/docs/DATASET_STRATEGY.md b/docs/DATASET_STRATEGY.md index d5d59466..8989789e 100644 --- a/docs/DATASET_STRATEGY.md +++ b/docs/DATASET_STRATEGY.md @@ -155,3 +155,8 @@ V1 dataset strategy is complete when: - Historical cartographic classes can change meaning between editions. Source classes and processing notes remain provenance, and object changes require explicit stable source identity. +- Canonical `vector_features.geometry` is 2D EPSG:4326. Valid source Z values + are removed only from the query index, while the original upload remains + unchanged and `z_dimension_feature_count` records that normalization. +- Dataset, version and vector-feature rows are committed atomically. Failed + geometry indexing rolls back all rows and removes the newly stored upload.