from __future__ import annotations from datetime import datetime, timezone from hashlib import sha256 import json from pathlib import Path from types import SimpleNamespace from uuid import uuid4 from app.models import Dataset, DatasetVersion, SourceRegistry, SourceSnapshot from app.services.data_contract_validation import ( build_vector_ingest_input, validate_registered_asset, ) from app.services.demo_workflow_service import DemoWorkflowService from app.services.derived_dataset_governance_service import ( DerivedDatasetGovernanceService, ) from app.services.raster_operations_service import RasterOperationsService from app.services.storage_service import StorageService from app.services.vector_feature_service import VectorFeatureService from app.services.vector_operations_service import VectorOperationsService _CHECKSUM = "a" * 64 class _FakeSession: def __init__(self, rows=None) -> None: self.rows = rows or {} self.added = [] def get(self, model, row_id): row = self.rows.get((model, row_id)) if row is not None: return row return next( ( item for item in self.added if isinstance(item, model) and item.id == row_id ), None, ) def add(self, row) -> None: self.added.append(row) def commit(self) -> None: return None def refresh(self, row) -> None: return None class _GovernedSession: """Small ORM-shaped session for the real governance branch. Registry persistence is monkeypatched below; the test exercises the service's orchestration and report decisions without needing PostGIS. """ def __init__(self) -> None: self.added = [] self.flushes = 0 class _EmptyQuery: def filter(self, *_args, **_kwargs): return self @staticmethod def all(): return [] @staticmethod def one_or_none(): return None def query(self, *_args, **_kwargs): # Governance now performs a bounded lineage traversal during quarantine. # This focused harness intentionally has no persisted siblings/edges. return self._EmptyQuery() def add(self, row) -> None: self.added.append(row) def flush(self) -> None: self.flushes += 1 def _governed_parent() -> Dataset: source_id = uuid4() snapshot_id = uuid4() source = SourceRegistry( id=source_id, source_key="grb", display_name="GRB parent fixture", classification="authoritative", authority_name="Digitaal Vlaanderen", authority_scope_json={"zone": "Flanders"}, usage_policy_json={"ground_truth_allowed": True}, ) snapshot = SourceSnapshot( id=snapshot_id, source_registry_id=source_id, snapshot_key="governed-parent", checksum_sha256=_CHECKSUM, freshness_status="current", ingest_status="ingested", ) dataset = Dataset( id=uuid4(), project_id=uuid4(), name="governed.geojson", dataset_type="vector", source="grb", source_name="grb", status="ready", checksum_sha256=_CHECKSUM, data_contract_key="geointel.vector.geojson", data_contract_version="1.0.0", validation_status="passed", provenance_status="complete", lineage_status="not_applicable", quarantine_status="not_quarantined", source_registry_id=source_id, source_snapshot_id=snapshot_id, ) dataset.source_registry = source dataset.source_snapshot = snapshot return dataset def test_lineage_evidence_quarantines_ungoverned_parent_without_inventing_a_checksum() -> ( None ): parent = _governed_parent() valid_lineage = DerivedDatasetGovernanceService._lineage_evidence( parent, "vector.clip", {"area_id": "a"} ) valid_report = validate_registered_asset( build_vector_ingest_input( asset_id="derived-valid", source_crs="EPSG:4326", storage_crs="EPSG:4326", feature_collection={ "type": "FeatureCollection", "features": [ { "type": "Feature", "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, "properties": {}, } ], }, checksum_sha256=_CHECKSUM, computed_checksum_sha256=_CHECKSUM, source_registry_id="derived-source", source_snapshot_id="derived-snapshot", imported_at=datetime.now(timezone.utc), metadata={ "license": "internal derived artifact", "bounds_json": { "min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0, }, }, temporal_unknown_reason="derived input has no precise observation timestamp", source_version_unknown_reason="transform version is recorded separately", lineage=valid_lineage, ) ) assert valid_report.validation_status.value == "passed" assert valid_lineage.upstream_asset_ids == (str(parent.id),) assert valid_lineage.upstream_checksums_sha256 == (_CHECKSUM,) parent.validation_status = "not_validated" rejected_lineage = DerivedDatasetGovernanceService._lineage_evidence( parent, "vector.clip", {} ) rejected_report = validate_registered_asset( build_vector_ingest_input( asset_id="derived-rejected", source_crs="EPSG:4326", storage_crs="EPSG:4326", feature_collection={ "type": "FeatureCollection", "features": [ { "type": "Feature", "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, "properties": {}, } ], }, checksum_sha256=_CHECKSUM, computed_checksum_sha256=_CHECKSUM, source_registry_id="derived-source", source_snapshot_id="derived-snapshot", imported_at=datetime.now(timezone.utc), metadata={ "license": "internal derived artifact", "bounds_json": { "min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0, }, }, temporal_unknown_reason="derived input has no precise observation timestamp", source_version_unknown_reason="transform version is recorded separately", lineage=rejected_lineage, ) ) assert rejected_lineage.upstream_checksums_sha256 == ( "parent_dataset_not_governed", ) assert rejected_report.validation_status.value == "failed" assert rejected_report.quarantine_status.value == "quarantined" assert any( issue.code == "UPSTREAM_CHECKSUM_FORMAT_INVALID" for issue in rejected_report.issues ) def test_govern_vector_binds_snapshot_contract_and_lineage_before_marking_ready( monkeypatch, ) -> None: from app.services.source_registry_service import SourceRegistryService db = _GovernedSession() parent = _governed_parent() parent_version_id = uuid4() dataset = Dataset( id=uuid4(), project_id=parent.project_id, name="derived.geojson", dataset_type="vector", source="operation:clip", source_name="derived", checksum_sha256=_CHECKSUM, imported_at=datetime.now(timezone.utc), crs="EPSG:4326", metadata_json={ "bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0} }, status="validating", ) version = DatasetVersion( id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM ) source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact") snapshot = SimpleNamespace(id=uuid4(), source_registry_id=source.id) edges = [] monkeypatch.setattr( SourceRegistryService, "ensure_server_owned_source", lambda *_args, **_kwargs: source, ) monkeypatch.setattr( SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot ) monkeypatch.setattr( DerivedDatasetGovernanceService, "_latest_parent_version_id", lambda *_args: parent_version_id, ) def _bind(target, **kwargs): target.source_registry_id = kwargs["source"].id target.source_snapshot_id = kwargs["snapshot"].id target.data_contract_key = kwargs["data_contract_key"] target.data_contract_version = kwargs["data_contract_version"] target.validation_status = kwargs["validation_status"] target.provenance_status = kwargs["provenance_status"] target.lineage_status = kwargs["lineage_status"] return target monkeypatch.setattr(SourceRegistryService, "bind_dataset_provenance", _bind) monkeypatch.setattr(SourceRegistryService, "bind_dataset_version_provenance", _bind) monkeypatch.setattr( SourceRegistryService, "record_lineage_edge", lambda *_args, **kwargs: edges.append(kwargs), ) ready = DerivedDatasetGovernanceService.govern_vector( db, dataset=dataset, dataset_version=version, feature_collection={ "type": "FeatureCollection", "features": [ { "type": "Feature", "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, "properties": {}, } ], }, source_key="derived", operation="vector.clip", parent_dataset=parent, operation_parameters={"area_id": "a"}, ) assert ready is True assert dataset.status == "ready" assert dataset.validation_status == "passed" assert dataset.provenance_status == "complete" assert dataset.source_registry_id == source.id assert version.source_snapshot_id == snapshot.id assert db.flushes >= 1 assert edges[0]["parent_dataset_id"] == parent.id assert edges[0]["parent_dataset_version_id"] == parent_version_id assert edges[0]["child_dataset_version_id"] == version.id def test_govern_vector_quarantines_output_when_parent_is_manual_or_experimental( monkeypatch, ) -> None: from app.services.source_registry_service import SourceRegistryService db = _GovernedSession() parent = _governed_parent() parent.source = "manual" parent.source_name = "manual" parent.source_registry.source_key = "manual" parent.source_registry.classification = "experimental" parent.source_registry.usage_policy_json = {"ground_truth_allowed": False} dataset = Dataset( id=uuid4(), project_id=parent.project_id, name="manual-derived.geojson", dataset_type="vector", source="operation:clip", source_name="derived", checksum_sha256=_CHECKSUM, imported_at=datetime.now(timezone.utc), crs="EPSG:4326", metadata_json={ "bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0} }, status="validating", ) version = DatasetVersion( id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM ) source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact") snapshot = SimpleNamespace( id=uuid4(), source_registry_id=source.id, ingest_status="ingested" ) monkeypatch.setattr( SourceRegistryService, "ensure_server_owned_source", lambda *_args, **_kwargs: source, ) monkeypatch.setattr( SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot ) monkeypatch.setattr( DerivedDatasetGovernanceService, "_latest_parent_version_id", lambda *_args: None, ) monkeypatch.setattr( SourceRegistryService, "record_lineage_edge", lambda *_args, **_kwargs: None ) ready = DerivedDatasetGovernanceService.govern_vector( db, dataset=dataset, dataset_version=version, feature_collection={ "type": "FeatureCollection", "features": [ { "type": "Feature", "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, "properties": {}, } ], }, source_key="derived", operation="vector.clip", parent_dataset=parent, ) assert ready is False assert dataset.status == "quarantined" assert dataset.quarantine_status == "quarantined" assert dataset.validation_status == "failed" assert any( issue["code"] == "PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING" for issue in dataset.validation_report_json["issues"] ) assert snapshot.ingest_status == "quarantined" def test_vector_selection_uses_map_selection_registry_and_skips_features_when_quarantined( monkeypatch, tmp_path ) -> None: source = _governed_parent() source.area_id = None source.storage_path = str(tmp_path / "source.geojson") db = _FakeSession({(Dataset, source.id): source}) output_path = tmp_path / "selection.geojson" calls = [] persisted_features = [] monkeypatch.setattr( VectorFeatureService, "select_features_by_bbox", lambda *_args, **_kwargs: { "selection_bbox": { "min_x": 4.9, "min_y": 50.9, "max_x": 5.2, "max_y": 51.2, "crs": "EPSG:4326", }, "feature_count": 1, "limit": 250, "truncated": False, "geojson": { "type": "FeatureCollection", "features": [ { "type": "Feature", "id": "source-feature", "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, "properties": { "vector_feature_id": "source-feature", "dataset_id": str(source.id), }, } ], }, }, ) def _persist_dataset_file(**kwargs): output_path.write_bytes(kwargs["content"]) return { "original_filename": kwargs["original_filename"], "stored_filename": output_path.name, "content_type": kwargs["content_type"], "size_bytes": len(kwargs["content"]), "checksum_sha256": _CHECKSUM, "storage_path": str(output_path), } monkeypatch.setattr(StorageService, "persist_dataset_file", _persist_dataset_file) def _quarantine(db, **kwargs): calls.append(kwargs) kwargs["dataset"].status = "quarantined" return False monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine) monkeypatch.setattr( VectorFeatureService, "persist_geojson_features", lambda **kwargs: persisted_features.append(kwargs), ) response = VectorOperationsService.derive_selection_dataset( db=db, dataset_id=source.id, bbox={ "min_x": 4.9, "min_y": 50.9, "max_x": 5.2, "max_y": 51.2, "crs": "EPSG:4326", }, ) assert response.status == "quarantined" assert calls[0]["source_key"] == "map_selection" assert calls[0]["parent_dataset"] is source assert calls[0]["operation"] == "vector.selection" assert persisted_features == [] def test_vector_buffer_uses_projected_metres_instead_of_wgs84_degrees( monkeypatch, tmp_path ) -> None: source = _governed_parent() source.storage_path = str(tmp_path / "source.geojson") source.crs = "EPSG:4326" Path(source.storage_path).write_text( json.dumps( { "type": "FeatureCollection", "crs": "EPSG:4326", "features": [ { "type": "Feature", "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, "properties": {}, }, ], } ), encoding="utf-8", ) # A governed consumption artifact must carry the checksum of these exact # bytes; vector operations deliberately refuse a stale fixture checksum. source.checksum_sha256 = sha256(Path(source.storage_path).read_bytes()).hexdigest() source.source_snapshot.checksum_sha256 = source.checksum_sha256 db = _FakeSession({(Dataset, source.id): source}) captured = {} def _persist(**kwargs): captured.update(kwargs) return uuid4() monkeypatch.setattr(VectorOperationsService, "_persist_derived_dataset", _persist) VectorOperationsService.buffer( db, source.id, distance_m=100.0, dissolve=False, output_name=None ) coordinates = captured["feature_collection"]["features"][0]["geometry"][ "coordinates" ][0] longitudes = [coordinate[0] for coordinate in coordinates] latitudes = [coordinate[1] for coordinate in coordinates] assert max(longitudes) - min(longitudes) < 0.01 assert max(latitudes) - min(latitudes) < 0.01 def test_raster_operation_uses_derived_registry_before_commit( monkeypatch, tmp_path ) -> None: source = _governed_parent() source.dataset_type = "raster" source.storage_path = str(tmp_path / "source.tif") output_path = tmp_path / "derived.tif" output_path.write_bytes(b"derived-raster") db = _FakeSession() calls = [] def _govern(db, **kwargs): calls.append(kwargs) kwargs["dataset"].status = "quarantined" return False monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_raster", _govern) result = RasterOperationsService._persist_derived_dataset( db, source_dataset=source, source_dataset_id=source.id, operation="ndvi", output_path=str(output_path), output_name="derived.tif", metadata={ "crs": "EPSG:31370", "bounds": [100000.0, 100000.0, 100001.0, 100001.0], "resolution": [1.0, 1.0], "width": 1, "height": 1, "band_count": 1, "dtype": ["float32"], "operation_parameters": {"nir_band": 4, "red_band": 3}, }, ) derived = next(item for item in db.added if isinstance(item, Dataset)) assert result == derived.id assert derived.status == "quarantined" assert derived.source_name == "derived" assert calls[0]["source_key"] == "derived" assert calls[0]["parent_dataset"] is source assert calls[0]["operation"] == "raster.ndvi" def test_demo_fixture_creation_is_governed_and_does_not_persist_features_when_rejected( monkeypatch, tmp_path ) -> None: project_id = uuid4() area_id = uuid4() db = _FakeSession() calls = [] features = [] monkeypatch.setattr( StorageService, "persist_dataset_file", lambda **kwargs: { "storage_path": str(tmp_path / kwargs["original_filename"]), "original_filename": kwargs["original_filename"], "stored_filename": kwargs["original_filename"], "content_type": kwargs["content_type"], "size_bytes": len(kwargs["content"]), "checksum_sha256": _CHECKSUM, }, ) def _quarantine(db, **kwargs): calls.append(kwargs) kwargs["dataset"].status = "quarantined" return False monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine) monkeypatch.setattr( VectorFeatureService, "persist_geojson_features", lambda **kwargs: features.append(kwargs), ) payload = { "type": "FeatureCollection", "features": [ { "type": "Feature", "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, "properties": {}, } ], } dataset = DemoWorkflowService._create_dataset( db, project_id=project_id, area_id=area_id, filename="fixture.geojson", payload=payload, raw=json.dumps(payload).encode("utf-8"), role="source", source_name="fixture", reference_layer_name=None, ) assert dataset.status == "quarantined" assert calls[0]["source_key"] == "fixture" assert calls[0]["operation"] == "demo.fixture_vector" assert features == []