diff --git a/backend/app/services/export_service.py b/backend/app/services/export_service.py index e092d033..e884031c 100644 --- a/backend/app/services/export_service.py +++ b/backend/app/services/export_service.py @@ -3,6 +3,7 @@ from __future__ import annotations import json import re import uuid +from datetime import datetime, timezone from html import escape from pathlib import Path from typing import Any @@ -35,6 +36,82 @@ from app.services.vector_feature_service import VectorFeatureService class ExportService: + # RFC 7946 allows foreign members on a FeatureCollection and requires + # parsers to ignore ones they do not know, so the provenance travels with + # the file without breaking QGIS, ogr2ogr or any other reader. + PROVENANCE_MEMBER = "geointel_provenance" + + @staticmethod + def provenance_member( + *, + source: str, + project_id: uuid.UUID, + dataset_id: uuid.UUID | None = None, + analysis_run_id: uuid.UUID | None = None, + source_name: str | None = None, + source_version: str | None = None, + observed_at: Any = None, + selection_bbox: dict[str, Any] | None = None, + selection_area_id: uuid.UUID | None = None, + feature_count: int | None = None, + total_feature_count: int | None = None, + truncated: bool = False, + warnings: list[str] | None = None, + ) -> dict[str, Any]: + """Describe an exported FeatureCollection inside the file itself. + + A capped export previously recorded ``truncated`` on the export record + only, so the downloaded file looked complete. Completeness is derived + from the counts as well as the flag: a caller that forgets to pass the + flag cannot produce a file that claims to hold everything. + """ + + complete = not truncated + if feature_count is not None and total_feature_count is not None: + complete = complete and feature_count >= total_feature_count + + member: dict[str, Any] = { + "source": source, + "project_id": str(project_id), + "exported_at": datetime.now(timezone.utc).isoformat(), + "complete": complete, + "completeness_note": None, + } + if dataset_id is not None: + member["dataset_id"] = str(dataset_id) + if analysis_run_id is not None: + member["analysis_run_id"] = str(analysis_run_id) + if source_name: + member["source_name"] = source_name + if source_version: + member["source_version"] = source_version + if observed_at is not None: + member["observed_at"] = observed_at.isoformat() if hasattr(observed_at, "isoformat") else str(observed_at) + if selection_bbox is not None: + member["selection_bbox"] = selection_bbox + if selection_area_id is not None: + member["selection_area_id"] = str(selection_area_id) + if feature_count is not None: + member["feature_count"] = feature_count + if total_feature_count is not None: + member["total_feature_count"] = total_feature_count + if warnings: + member["warnings"] = list(warnings) + + if not complete: + written = feature_count if feature_count is not None else "?" + available = total_feature_count if total_feature_count is not None else "?" + member["completeness_note"] = ( + f"Dit bestand bevat {written} van {available} objecten uit de selectie. Het is een " + "begrensde uitsnede, geen volledige export." + ) + return member + + @staticmethod + def attach_provenance(feature_collection: dict[str, Any], member: dict[str, Any]) -> dict[str, Any]: + feature_collection[ExportService.PROVENANCE_MEMBER] = member + return feature_collection + @staticmethod def _detection_export_trust(db: Session, run: AnalysisRun) -> dict[str, Any]: """Classify persisted AI output without turning confidence into truth.""" @@ -447,6 +524,28 @@ class ExportService: preclipped_partition_filter=preclipped_partition_filter, ) selection = VectorFeatureService.select_features_by_bbox(db, **selection_kwargs) + summary = selection.get("summary") if isinstance(selection.get("summary"), dict) else {} + ExportService.attach_provenance( + selection["geojson"], + ExportService.provenance_member( + source="vector_selection", + project_id=dataset.project_id, + dataset_id=dataset.id, + source_name=dataset.source_name, + source_version=dataset.source_version, + observed_at=dataset.observed_at, + selection_bbox=selection["selection_bbox"], + selection_area_id=area_id, + feature_count=selection["feature_count"], + total_feature_count=selection.get("total_feature_count"), + truncated=selection["truncated"], + warnings=[ + warning + for warning in (summary.get("selection_edge_warning"), summary.get("warning")) + if warning + ], + ), + ) filename = ExportService._filename(name, f"{dataset.id}-selection.geojson", ".geojson") export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename) metadata = { @@ -487,6 +586,18 @@ class ExportService: DatasetConsumptionGate.assert_eligible(dataset, purpose="export") feature_collection = DatasetService.get_dataset_geojson(db, dataset_id) + ExportService.attach_provenance( + feature_collection, + ExportService.provenance_member( + source="dataset", + project_id=dataset.project_id, + dataset_id=dataset.id, + source_name=dataset.source_name, + source_version=dataset.source_version, + observed_at=dataset.observed_at, + feature_count=len(feature_collection.get("features", [])), + ), + ) filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson") export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename) metadata = { @@ -530,6 +641,16 @@ class ExportService: ) feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id) feature_collection["geointel_result"] = trust + ExportService.attach_provenance( + feature_collection, + ExportService.provenance_member( + source="detection_run", + project_id=run.project_id, + dataset_id=run.dataset_id, + analysis_run_id=run.id, + feature_count=len(feature_collection.get("features", [])), + ), + ) for feature in feature_collection.get("features", []): properties = feature.get("properties") if isinstance(feature, dict) else None if isinstance(properties, dict): @@ -565,6 +686,16 @@ class ExportService: ExportService._assert_run_source_dataset_exportable(db, run) feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id) + ExportService.attach_provenance( + feature_collection, + ExportService.provenance_member( + source="segmentation_run", + project_id=run.project_id, + dataset_id=run.dataset_id, + analysis_run_id=run.id, + feature_count=len(feature_collection.get("features", [])), + ), + ) filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson") export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename) metadata = { diff --git a/backend/tests/test_export_file_states_completeness.py b/backend/tests/test_export_file_states_completeness.py new file mode 100644 index 00000000..961672fa --- /dev/null +++ b/backend/tests/test_export_file_states_completeness.py @@ -0,0 +1,125 @@ +"""A downloaded export must carry its own provenance and its own limits. + +Counterpart to ``test_export_provenance_member``: this reads the file the +service actually writes, so it proves the member survives serialisation rather +than only that the helper builds it. +""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +from app.models import Dataset, Export +from app.services.export_service import ExportService +from app.services.storage_service import StorageService +from app.services.vector_feature_service import VectorFeatureService + +from tests.test_sprint107_map_selection_export import FakeSession, _govern_fixture_dataset + + +BBOX = {"min_x": 4.9, "min_y": 50.9, "max_x": 5.2, "max_y": 51.2, "crs": "EPSG:4326"} + + +def _dataset(dataset_id): + dataset = _govern_fixture_dataset( + Dataset( + id=dataset_id, + project_id=uuid4(), + name="grb-buildings.geojson", + dataset_type="vector", + source="grb", + source_name="grb", + source_version="2024-06", + observed_at=datetime(2024, 6, 1, tzinfo=timezone.utc), + status="ready", + ) + ) + return dataset + + +def _selection(*, feature_count: int, total: int, truncated: bool, warning: str | None = None): + return { + "selection_bbox": BBOX, + "feature_count": feature_count, + "total_feature_count": total, + "limit": 250, + "truncated": truncated, + "summary": {"selection_edge_warning": warning} if warning else {}, + "geojson": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, + "properties": {"vector_feature_id": f"vf-{index}"}, + } + for index in range(feature_count) + ], + }, + } + + +def _export(monkeypatch, tmp_path: Path, selection) -> dict: + dataset_id = uuid4() + dataset = _dataset(dataset_id) + db = FakeSession({(Dataset, dataset_id): dataset}) + export_path = tmp_path / "selection.geojson" + monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path)) + monkeypatch.setattr(VectorFeatureService, "select_features_by_bbox", lambda *_args, **_kwargs: selection) + + ExportService.export_vector_selection_geojson(db, dataset_id, BBOX, limit=250, name="selection") + + assert [item for item in db.added if isinstance(item, Export)] + return json.loads(export_path.read_text(encoding="utf-8")) + + +def test_a_truncated_export_file_states_that_it_is_partial(monkeypatch, tmp_path: Path) -> None: + written = _export(monkeypatch, tmp_path, _selection(feature_count=2, total=1_400, truncated=True)) + + provenance = written["geointel_provenance"] + assert provenance["complete"] is False + assert "1400" in provenance["completeness_note"].replace(".", "") + # The features are still there; the file simply no longer implies it holds + # everything the selection contains. + assert len(written["features"]) == 2 + + +def test_a_complete_export_file_says_so(monkeypatch, tmp_path: Path) -> None: + written = _export(monkeypatch, tmp_path, _selection(feature_count=2, total=2, truncated=False)) + + provenance = written["geointel_provenance"] + assert provenance["complete"] is True + assert provenance["completeness_note"] is None + + +def test_the_file_identifies_its_source_edition(monkeypatch, tmp_path: Path) -> None: + written = _export(monkeypatch, tmp_path, _selection(feature_count=1, total=1, truncated=False)) + + provenance = written["geointel_provenance"] + assert provenance["source_name"] == "grb" + assert provenance["source_version"] == "2024-06" + assert provenance["observed_at"].startswith("2024-06-01") + assert provenance["selection_bbox"] == BBOX + + +def test_selection_caveats_travel_into_the_file(monkeypatch, tmp_path: Path) -> None: + written = _export( + monkeypatch, + tmp_path, + _selection(feature_count=1, total=1, truncated=False, warning="22 objecten liggen deels buiten de selectie."), + ) + + assert written["geointel_provenance"]["warnings"] == ["22 objecten liggen deels buiten de selectie."] + + +def test_the_file_remains_a_valid_feature_collection(monkeypatch, tmp_path: Path) -> None: + """The provenance is a foreign member, not a change to the GeoJSON shape.""" + + written = _export(monkeypatch, tmp_path, _selection(feature_count=1, total=1, truncated=False)) + + assert written["type"] == "FeatureCollection" + assert isinstance(written["features"], list) + assert written["features"][0]["type"] == "Feature" diff --git a/backend/tests/test_export_provenance_member.py b/backend/tests/test_export_provenance_member.py new file mode 100644 index 00000000..30b4f874 --- /dev/null +++ b/backend/tests/test_export_provenance_member.py @@ -0,0 +1,129 @@ +"""An exported GeoJSON must say what it is and what it leaves out. + +The vector selection export caps its features and records ``truncated`` in the +export *record*. The file itself said nothing: an operator downloads +``mol-selection.geojson``, opens it in QGIS and sees 250 buildings where the +workbench said 1.400, with nothing in the file to indicate the difference. + +For a product whose promise is that an export is a reproducible result, a file +that looks complete and is not is the sharpest possible violation. RFC 7946 +allows foreign members on a FeatureCollection, and the detection export already +uses one; this makes that convention uniform. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from uuid import uuid4 + +from app.services.export_service import ExportService + + +def test_provenance_names_the_source_and_the_moment() -> None: + dataset_id = uuid4() + project_id = uuid4() + + member = ExportService.provenance_member( + source="vector_selection", + project_id=project_id, + dataset_id=dataset_id, + source_name="grb", + source_version="2024-06", + observed_at=datetime(2024, 6, 1, tzinfo=timezone.utc), + ) + + assert member["source"] == "vector_selection" + assert member["dataset_id"] == str(dataset_id) + assert member["project_id"] == str(project_id) + assert member["source_name"] == "grb" + assert member["source_version"] == "2024-06" + assert member["observed_at"].startswith("2024-06-01") + assert member["exported_at"] + + +def test_a_truncated_export_says_so_in_words() -> None: + member = ExportService.provenance_member( + source="vector_selection", + project_id=uuid4(), + dataset_id=uuid4(), + feature_count=250, + total_feature_count=1_400, + truncated=True, + ) + + assert member["complete"] is False + assert member["feature_count"] == 250 + assert member["total_feature_count"] == 1_400 + assert "1400" in member["completeness_note"].replace(".", "").replace(",", "") + assert "250" in member["completeness_note"] + + +def test_a_complete_export_is_stated_as_complete() -> None: + member = ExportService.provenance_member( + source="detection_run", + project_id=uuid4(), + dataset_id=uuid4(), + feature_count=12, + total_feature_count=12, + truncated=False, + ) + + assert member["complete"] is True + assert member["completeness_note"] is None + + +def test_counts_that_disagree_are_treated_as_incomplete() -> None: + """A caller that forgets the flag must not produce a file claiming completeness.""" + + member = ExportService.provenance_member( + source="vector_selection", + project_id=uuid4(), + dataset_id=uuid4(), + feature_count=100, + total_feature_count=140, + truncated=False, + ) + + assert member["complete"] is False + + +def test_selection_context_travels_with_the_file() -> None: + area_id = uuid4() + bbox = {"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3, "crs": "EPSG:4326"} + + member = ExportService.provenance_member( + source="vector_selection", + project_id=uuid4(), + dataset_id=uuid4(), + selection_bbox=bbox, + selection_area_id=area_id, + warnings=["22 van de 100 objecten liggen deels buiten de selectie."], + ) + + assert member["selection_bbox"] == bbox + assert member["selection_area_id"] == str(area_id) + assert member["warnings"] == ["22 van de 100 objecten liggen deels buiten de selectie."] + + +def test_empty_context_is_omitted_rather_than_written_as_null_noise() -> None: + member = ExportService.provenance_member( + source="dataset", + project_id=uuid4(), + dataset_id=uuid4(), + ) + + assert "selection_bbox" not in member + assert "warnings" not in member + assert "source_version" not in member + + +def test_the_member_attaches_under_a_reserved_key() -> None: + collection = {"type": "FeatureCollection", "features": []} + + ExportService.attach_provenance( + collection, + ExportService.provenance_member(source="dataset", project_id=uuid4(), dataset_id=uuid4()), + ) + + assert collection["type"] == "FeatureCollection" + assert collection["geointel_provenance"]["source"] == "dataset"