from __future__ import annotations from datetime import UTC, datetime from hashlib import sha256 import json from pathlib import Path from uuid import UUID, uuid4 import pytest from pyproj import Transformer from app.core.errors import AppError from app.models import ( Area, Dataset, DatasetQuarantine, DatasetVersion, Project, SourceRegistry, SourceSnapshot, VectorFeature, ) from app.services.dataset_service import DatasetService, _PartitionedGeoJsonRecords from app.services.vector_operations_service import VectorOperationsService class _Query: def __init__(self, session: "_Session", model: type) -> None: self.session = session self.model = model self.predicates = [] def filter(self, *predicates): self.predicates.extend(predicates) return self def one_or_none(self): matches = self._matches() if len(matches) > 1: raise AssertionError( f"expected one {self.model.__name__}, found {len(matches)}" ) return matches[0] if matches else None def all(self): return self._matches() def _matches(self): matches = list(self.session.rows.get(self.model, [])) for predicate in self.predicates: field_name = predicate.left.key expected = predicate.right.value operator_name = getattr(predicate.operator, "__name__", "") if operator_name == "in_op": matches = [ item for item in matches if getattr(item, field_name) in expected ] else: matches = [ item for item in matches if getattr(item, field_name) == expected ] return matches class _Session: """Small ORM-shaped harness that exercises the real governed path.""" def __init__(self, project: Project) -> None: self.rows: dict[type, list[object]] = {Project: [project]} self.commits = 0 self.rollbacks = 0 self.flushes = 0 def get(self, model: type, item_id: UUID): return next( ( item for item in self.rows.get(model, []) if getattr(item, "id", None) == item_id ), None, ) def query(self, model: type) -> _Query: return _Query(self, model) def add(self, item: object) -> None: if getattr(item, "id", None) is None: setattr(item, "id", uuid4()) self.rows.setdefault(type(item), []).append(item) def flush(self) -> None: self.flushes += 1 def commit(self) -> None: self.commits += 1 def rollback(self) -> None: self.rollbacks += 1 def refresh(self, _item: object) -> None: return None def expunge(self, _item: object) -> None: return None def _storage_info(tmp_path: Path, content: bytes) -> dict[str, object]: path = tmp_path / "grb-buildings.geojson" path.write_bytes(content) return { "storage_path": str(path), "original_filename": path.name, "stored_filename": path.name, "content_type": "application/geo+json", "size_bytes": len(content), "checksum_sha256": sha256(content).hexdigest(), } def _valid_payload() -> bytes: return json.dumps( { "type": "FeatureCollection", "crs": {"type": "name", "properties": {"name": "EPSG:4326"}}, "features": [ { "type": "Feature", "id": "gbg-1", "properties": {"id": "gbg-1"}, "geometry": { "type": "Polygon", "coordinates": [ [[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]] ], }, } ], } ).encode("utf-8") def _grb_payload_without_required_id() -> bytes: return json.dumps( { "type": "FeatureCollection", "crs": {"type": "name", "properties": {"name": "EPSG:4326"}}, "features": [ { "type": "Feature", "properties": {"unrelated": "not a GRB identity"}, "geometry": { "type": "Polygon", "coordinates": [ [[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]] ], }, } ], } ).encode("utf-8") def _lambert_grb_payload() -> tuple[bytes, tuple[float, float, float, float]]: """Create a valid GRB-shaped source artifact in its declared native CRS.""" longitude, latitude = 4.70, 51.10 max_longitude, max_latitude = 4.7001, 51.1001 to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) lambert_ring = [ to_lambert.transform(longitude, latitude), to_lambert.transform(max_longitude, latitude), to_lambert.transform(max_longitude, max_latitude), to_lambert.transform(longitude, latitude), ] return ( json.dumps( { "type": "FeatureCollection", "crs": {"type": "name", "properties": {"name": "EPSG:31370"}}, "features": [ { "type": "Feature", "id": "GBG.lambert.1", "properties": {"id": "GBG.lambert.1"}, "geometry": {"type": "Polygon", "coordinates": [lambert_ring]}, } ], } ).encode("utf-8"), (longitude, latitude, max_longitude, max_latitude), ) def test_governed_vector_import_persists_snapshot_contract_and_queryable_features( monkeypatch, tmp_path: Path ) -> None: project = Project(id=uuid4(), name="Phase 2 governed ingest") db = _Session(project) raw = _valid_payload() monkeypatch.setattr( "app.services.dataset_service.StorageService.persist_dataset_file", lambda **_kwargs: _storage_info(tmp_path, raw), ) result = DatasetService.import_vector_bytes( db, project_id=project.id, filename="grb-buildings.geojson", content=raw, source="grb_wfs", source_name="grb", dataset_role="reference", reference_layer_name="buildings", source_metadata={ "license": "Open data", "source_url": "https://example.invalid/grb", }, provenance_metadata={"adapter": "test"}, temporal_series_key="grb:2026-08", observed_at=datetime(2026, 8, 1, tzinfo=UTC), source_version="2026-08-01", temporal_granularity="snapshot", ) dataset = next(item for item in db.rows[Dataset] if item.id == result.id) snapshot = db.rows[SourceSnapshot][0] source = db.rows[SourceRegistry][0] assert result.status == "ready" assert dataset.source_name == "grb" assert dataset.source_registry_id == source.id assert dataset.source_snapshot_id == snapshot.id assert dataset.validation_status == "passed" assert dataset.provenance_status == "complete" assert dataset.lineage_status == "complete" assert dataset.quarantine_status == "not_quarantined" assert dataset.crs == "EPSG:4326" assert snapshot.checksum_sha256 == sha256(raw).hexdigest() assert len(db.rows[VectorFeature]) == 1 assert db.commits == 1 # A retry with identical governed evidence is idempotent and does not # create a second source snapshot, dataset or vector feature. repeated = DatasetService.import_vector_bytes( db, project_id=project.id, filename="grb-buildings.geojson", content=raw, source="grb_wfs", source_name="grb", dataset_role="reference", reference_layer_name="buildings", source_metadata={"license": "Open data"}, provenance_metadata={"adapter": "test"}, temporal_series_key="grb:2026-08", observed_at=datetime(2026, 8, 1, tzinfo=UTC), source_version="2026-08-01", temporal_granularity="snapshot", ) assert repeated.id == result.id assert len(db.rows[Dataset]) == 1 assert len(db.rows[SourceSnapshot]) == 1 assert len(db.rows[VectorFeature]) == 1 def test_governed_lambert_geojson_persists_canonical_consumption_bytes_and_provenance_evidence( monkeypatch, tmp_path: Path ) -> None: """Projected source bytes must never be the file that vector operations consume.""" project = Project(id=uuid4(), name="Canonical GeoJSON storage") db = _Session(project) raw, (longitude, latitude, max_longitude, max_latitude) = _lambert_grb_payload() consumption_path = tmp_path / "consumption" / "grb-buildings.geojson" provenance_path = tmp_path / "provenance" / "grb-buildings.geojson" def _persist_dataset_file(**kwargs): stored = kwargs["content"] consumption_path.parent.mkdir(parents=True, exist_ok=True) consumption_path.write_bytes(stored) return _storage_info(consumption_path.parent, stored) def _persist_file(storage_path, content, original_filename, content_type): del storage_path, original_filename, content_type provenance_path.parent.mkdir(parents=True, exist_ok=True) provenance_path.write_bytes(content) return _storage_info(provenance_path.parent, content) monkeypatch.setattr( "app.services.dataset_service.StorageService.persist_dataset_file", _persist_dataset_file, ) monkeypatch.setattr( "app.services.dataset_service.StorageService.persist_file", _persist_file, ) result = DatasetService.import_vector_bytes( db, project_id=project.id, filename="grb-lambert.geojson", content=raw, source="grb_wfs", source_name="grb", dataset_role="reference", reference_layer_name="buildings", source_metadata={"license": "Open data"}, provenance_metadata={"adapter": "test"}, temporal_series_key="grb:lambert:2026-08", observed_at=datetime(2026, 8, 1, tzinfo=UTC), source_version="2026-08-01-lambert", temporal_granularity="snapshot", ) dataset = next(item for item in db.rows[Dataset] if item.id == result.id) dataset_version = db.rows[DatasetVersion][0] snapshot = db.rows[SourceSnapshot][0] canonical_bytes = Path(str(dataset.storage_path)).read_bytes() canonical_payload = json.loads(canonical_bytes) source_artifact = dataset.provenance_metadata["source_artifact"] assert result.status == "ready" assert dataset.crs == "EPSG:4326" assert canonical_payload["crs"]["properties"]["name"] == "EPSG:4326" assert canonical_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx( [longitude, latitude], abs=0.000001 ) assert sha256(canonical_bytes).hexdigest() == dataset.checksum_sha256 assert dataset_version.checksum_sha256 == dataset.checksum_sha256 assert snapshot.checksum_sha256 == dataset.checksum_sha256 assert source_artifact["retention"] == "provenance_evidence_only" assert source_artifact["checksum_sha256"] == sha256(raw).hexdigest() assert source_artifact["storage_path"] != dataset.storage_path assert Path(source_artifact["storage_path"]).read_bytes() == raw assert dataset.provenance_metadata["canonical_consumption_artifact"] == { "checksum_sha256": dataset.checksum_sha256, "crs": "EPSG:4326", "storage_role": "dataset_consumption", } inspection = VectorOperationsService.inspect(db, dataset.id) assert inspection.crs == "EPSG:4326" assert inspection.bounds_json == { "min_x": pytest.approx(longitude, abs=0.000001), "min_y": pytest.approx(latitude, abs=0.000001), "max_x": pytest.approx(max_longitude, abs=0.000001), "max_y": pytest.approx(max_latitude, abs=0.000001), } response_payload = DatasetService.get_dataset_geojson(db, dataset.id) assert response_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx( [longitude, latitude], abs=0.000001 ) # The storage identity is enforced at the operation boundary too; a # replacement with different canonical bytes is not silently processed. Path(str(dataset.storage_path)).write_bytes(canonical_bytes + b"\n") with pytest.raises(AppError) as exc_info: VectorOperationsService.inspect(db, dataset.id) assert exc_info.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH" def test_metadata_refresh_refuses_mutated_governed_artifact( monkeypatch, tmp_path: Path ) -> None: """A passed snapshot cannot be silently re-described from mutable storage.""" project = Project(id=uuid4(), name="Phase 2 immutable refresh") db = _Session(project) raw = _valid_payload() monkeypatch.setattr( "app.services.dataset_service.StorageService.persist_dataset_file", lambda **_kwargs: _storage_info(tmp_path, raw), ) result = DatasetService.import_vector_bytes( db, project_id=project.id, filename="grb-buildings.geojson", content=raw, source="grb_wfs", source_name="grb", dataset_role="reference", reference_layer_name="buildings", source_metadata={"license": "Open data"}, provenance_metadata={"adapter": "test"}, temporal_series_key="grb:2026-08", observed_at=datetime(2026, 8, 1, tzinfo=UTC), source_version="2026-08-01", temporal_granularity="snapshot", ) dataset = next(item for item in db.rows[Dataset] if item.id == result.id) original_checksum = dataset.checksum_sha256 original_metadata = dict(dataset.metadata_json or {}) original_commit_count = db.commits # Simulate an out-of-band storage replacement at the same path. The # refresh endpoint must not parse it into an already-passed contract row. Path(str(dataset.storage_path)).write_bytes(_grb_payload_without_required_id()) with pytest.raises(AppError) as exc_info: DatasetService.refresh_metadata(db, dataset.id) assert exc_info.value.code == "GOVERNED_DATASET_REINGEST_REQUIRED" assert dataset.status == "ready" assert dataset.validation_status == "passed" assert dataset.checksum_sha256 == original_checksum assert dataset.metadata_json == original_metadata assert db.commits == original_commit_count def test_governed_import_quarantines_bad_artifacts_and_refuses_unknown_source( monkeypatch, tmp_path: Path ) -> None: project = Project(id=uuid4(), name="Phase 2 quarantine") db = _Session(project) raw = b'{"type":"FeatureCollection","features":[]}' monkeypatch.setattr( "app.services.dataset_service.StorageService.persist_dataset_file", lambda **_kwargs: _storage_info(tmp_path, raw), ) quarantined = DatasetService.import_vector_bytes( db, project_id=project.id, filename="empty.geojson", content=raw, source="grb_wfs", source_name="grb", dataset_role="reference", reference_layer_name="buildings", source_metadata={"license": "Open data"}, provenance_metadata={}, temporal_series_key="grb:2026-08-empty", observed_at=datetime(2026, 8, 1, tzinfo=UTC), source_version="2026-08-01-empty", temporal_granularity="snapshot", ) assert quarantined.status == "quarantined" assert quarantined.validation_status == "failed" assert quarantined.quarantine_status == "quarantined" assert len(db.rows[DatasetQuarantine]) == 1 assert db.rows[SourceSnapshot][0].ingest_status == "quarantined" with pytest.raises(AppError) as exc_info: DatasetService.import_vector_bytes( db, project_id=project.id, filename="unregistered.geojson", content=_valid_payload(), source="caller_controlled", source_name="caller_claimed_grb", dataset_role="reference", reference_layer_name="buildings", source_metadata={"license": "Open data"}, provenance_metadata={}, ) assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND" def test_governed_grb_vector_quarantines_missing_server_owned_required_attribute( monkeypatch, tmp_path: Path ) -> None: project = Project(id=uuid4(), name="Phase 2 source schema") db = _Session(project) raw = _grb_payload_without_required_id() monkeypatch.setattr( "app.services.dataset_service.StorageService.persist_dataset_file", lambda **kwargs: _storage_info(tmp_path, kwargs["content"]), ) quarantined = DatasetService.import_vector_bytes( db, project_id=project.id, filename="grb-missing-id.geojson", content=raw, source="grb_wfs", source_name="grb", dataset_role="reference", reference_layer_name="buildings", source_metadata={"license": "Open data"}, provenance_metadata={"adapter": "test"}, temporal_series_key="grb:missing-id", observed_at=datetime(2026, 8, 1, tzinfo=UTC), source_version="2026-08-01-missing-id", temporal_granularity="snapshot", ) dataset = next(item for item in db.rows[Dataset] if item.id == quarantined.id) assert quarantined.status == "quarantined" assert dataset.validation_status == "failed" assert dataset.quarantine_status == "quarantined" issue = dataset.validation_report_json["issues"][0] assert issue["code"] == "SOURCE_SCHEMA_REQUIRED_ATTRIBUTE_MISSING" assert issue["category"] == "source_schema" assert len(db.rows[DatasetQuarantine]) == 1 def test_partitioned_vector_ingest_is_idempotent_and_quarantines_noncanonical_partition_coordinates( monkeypatch, tmp_path: Path, ) -> None: project = Project(id=uuid4(), name="Partitioned governed ingest") area = Area(id=uuid4(), project_id=project.id, name="Partitioned AOI") db = _Session(project) db.rows[Area] = [area] feature = { "type": "Feature", "id": "GBG.1", "properties": {"id": "GBG.1", "source_feature_id": "GBG.1"}, "geometry": { "type": "Polygon", "coordinates": [ [[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]] ], }, } partition_payload = {"type": "FeatureCollection", "features": [feature]} partition_path = tmp_path / "partition-01.geojson" partition_path.write_text(json.dumps(partition_payload), encoding="utf-8") artifact_payload = { "type": "FeatureCollection", "crs": "EPSG:4326", "features": [feature], } artifact_path = tmp_path / "grb-partitioned.geojson" artifact_raw = json.dumps(artifact_payload).encode("utf-8") artifact_path.write_bytes(artifact_raw) monkeypatch.setattr( "app.services.dataset_service.StorageService.persist_dataset_file_from_path", lambda **_kwargs: _storage_info(tmp_path, artifact_raw), ) result = DatasetService.import_partitioned_vector_artifact( db, project_id=project.id, area_id=area.id, artifact_path=artifact_path, partition_paths=[partition_path], original_filename="grb-partitioned.geojson", source="operator_official_import", dataset_role="reference", source_name="grb", reference_layer_name="buildings", metadata_json={ "feature_count": 1, "crs": "EPSG:4326", "bounds_json": { "min_x": 4.69, "min_y": 51.09, "max_x": 4.70, "max_y": 51.10, }, }, source_metadata={"license": "Open data"}, provenance_metadata={ "artifact_sha256": sha256(artifact_raw).hexdigest(), "partition_checksums": { partition_path.name: sha256(partition_path.read_bytes()).hexdigest() }, }, temporal_series_key="grb:partitioned:test", observed_at=datetime(2026, 8, 1, tzinfo=UTC), source_version="2026-08-01", ) dataset = next(item for item in db.rows[Dataset] if item.id == result.id) assert result.status == "ready" assert dataset.validation_status == "passed" assert dataset.provenance_status == "complete" assert dataset.source_name == "grb" assert len(db.rows[SourceSnapshot]) == 1 assert len(db.rows[VectorFeature]) == 1 assert dataset.metadata_json["partitioned_geometry_audit"][ "partition_checksums_sha256" ] == {partition_path.name: sha256(partition_path.read_bytes()).hexdigest()} assert dataset.provenance_metadata["partition_checksum_manifest_sha256"] assert dataset.provenance_metadata["partitioned_artifact_binding_sha256"] repeated = DatasetService.import_partitioned_vector_artifact( db, project_id=project.id, area_id=area.id, artifact_path=artifact_path, partition_paths=[partition_path], original_filename="grb-partitioned.geojson", source="operator_official_import", dataset_role="reference", source_name="grb", reference_layer_name="buildings", metadata_json={ "feature_count": 1, "crs": "EPSG:4326", "bounds_json": { "min_x": 4.69, "min_y": 51.09, "max_x": 4.70, "max_y": 51.10, }, }, source_metadata={"license": "Open data"}, provenance_metadata={ "artifact_sha256": sha256(artifact_raw).hexdigest(), "partition_checksums": { partition_path.name: sha256(partition_path.read_bytes()).hexdigest() }, }, temporal_series_key="grb:partitioned:test", observed_at=datetime(2026, 8, 1, tzinfo=UTC), source_version="2026-08-01", ) assert repeated.id == result.id assert len(db.rows[Dataset]) == 1 assert len(db.rows[VectorFeature]) == 1 lambert_feature = { **feature, "id": "GBG.lambert", "properties": {"id": "GBG.lambert"}, "geometry": { "type": "Polygon", "coordinates": [ [[150000, 170000], [150010, 170000], [150010, 170010], [150000, 170000]] ], }, } lambert_partition = tmp_path / "partition-lambert.geojson" lambert_partition.write_text( json.dumps({"type": "FeatureCollection", "features": [lambert_feature]}), encoding="utf-8", ) lambert_artifact = tmp_path / "grb-lambert.geojson" lambert_raw = json.dumps( {"type": "FeatureCollection", "features": [lambert_feature]} ).encode("utf-8") lambert_artifact.write_bytes(lambert_raw) monkeypatch.setattr( "app.services.dataset_service.StorageService.persist_dataset_file_from_path", lambda **_kwargs: _storage_info(tmp_path, lambert_raw), ) quarantined = DatasetService.import_partitioned_vector_artifact( db, project_id=project.id, area_id=area.id, artifact_path=lambert_artifact, partition_paths=[lambert_partition], original_filename="grb-lambert.geojson", source="operator_official_import", dataset_role="reference", source_name="grb", reference_layer_name="buildings", metadata_json={ "feature_count": 1, "crs": "EPSG:4326", "bounds_json": { "min_x": 150000, "min_y": 170000, "max_x": 150010, "max_y": 170010, }, }, source_metadata={"license": "Open data"}, provenance_metadata={ "artifact_sha256": sha256(lambert_raw).hexdigest(), "partition_checksums": { lambert_partition.name: sha256( lambert_partition.read_bytes() ).hexdigest() }, }, temporal_series_key="grb:partitioned:lambert", observed_at=datetime(2026, 8, 2, tzinfo=UTC), source_version="2026-08-02", ) assert quarantined.status == "quarantined" assert quarantined.validation_status == "failed" assert quarantined.quarantine_status == "quarantined" missing_manifest_partition = tmp_path / "partition-missing-manifest.geojson" missing_manifest_partition.write_text( json.dumps(partition_payload), encoding="utf-8" ) missing_manifest_artifact = tmp_path / "grb-missing-manifest.geojson" missing_manifest_raw = json.dumps( {"type": "FeatureCollection", "features": [feature]} ).encode("utf-8") missing_manifest_artifact.write_bytes(missing_manifest_raw) monkeypatch.setattr( "app.services.dataset_service.StorageService.persist_dataset_file_from_path", lambda **_kwargs: _storage_info(tmp_path, missing_manifest_raw), ) missing_manifest = DatasetService.import_partitioned_vector_artifact( db, project_id=project.id, area_id=area.id, artifact_path=missing_manifest_artifact, partition_paths=[missing_manifest_partition], original_filename="grb-missing-manifest.geojson", source="operator_official_import", dataset_role="reference", source_name="grb", reference_layer_name="buildings", metadata_json={ "feature_count": 1, "crs": "EPSG:4326", "bounds_json": { "min_x": 4.69, "min_y": 51.09, "max_x": 4.70, "max_y": 51.10, }, }, source_metadata={"license": "Open data"}, provenance_metadata={ "artifact_sha256": sha256(missing_manifest_raw).hexdigest() }, temporal_series_key="grb:partitioned:missing-manifest", observed_at=datetime(2026, 8, 3, tzinfo=UTC), source_version="2026-08-03", ) assert missing_manifest.status == "quarantined" assert ( missing_manifest.validation_report_json["issues"][0]["code"] == "PARTITION_CHECKSUM_MANIFEST_REQUIRED" ) def test_partitioned_geometry_audit_handles_more_than_generic_topology_limit_without_materializing_geometries( tmp_path: Path, ) -> None: feature_count = 10_001 partition_path = tmp_path / "large-partition.geojson" partition_path.write_text( json.dumps( { "type": "FeatureCollection", "features": [ { "type": "Feature", "id": f"GBG.{index}", "properties": {"id": f"GBG.{index}"}, "geometry": { "type": "Point", "coordinates": [4.0 + index / 10_000_000, 51.0], }, } for index in range(feature_count) ], } ), encoding="utf-8", ) audit = _PartitionedGeoJsonRecords( [partition_path], expected_feature_count=feature_count, declared_partition_checksums={ partition_path.name: sha256(partition_path.read_bytes()).hexdigest() }, ).audit() assert audit.feature_count == feature_count assert audit.bounds_json["min_x"] == 4.0 assert audit.bounds_json["max_x"] > audit.bounds_json["min_x"] assert audit.representative_record.geometry.geom_type == "MultiPoint"