Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

make an exported file state its own provenance and limits

Browse Source 
The vector selection export caps its features and recorded truncated on the
export record. The file said nothing: an operator downloads
mol-selection.geojson, opens 250 buildings in QGIS where the workbench said
1.400, and has nothing to tell them the difference. For a product whose promise
is that an export is a reproducible result, a file that looks complete and is
not is the sharpest possible violation of it.

Every exported FeatureCollection now carries a geointel_provenance foreign
member — RFC 7946 requires parsers to ignore unknown members, so QGIS and
ogr2ogr are unaffected, and the detection export already used the same
convention for its trust classification. It names the source, the dataset and
its edition, the selection, and whether the file is complete; completeness is
derived from the counts as well as the flag, so a caller that forgets the flag
cannot produce a file claiming to hold everything.

Applied uniformly to the selection, dataset, detection and segmentation
exports, which previously disclosed three different amounts of nothing.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jens
2026-08-22 14:56:37 +02:00
co-authored by Claude Opus 5
parent 12aaf1bb4d
commit 0d8f146fc4
3 changed files with 385 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/export_service.py
+131
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import json
import re
import uuid
from datetime import datetime, timezone
from html import escape
from pathlib import Path
from typing import Any
@@ -35,6 +36,82 @@ from app.services.vector_feature_service import VectorFeatureService
class ExportService:
# RFC 7946 allows foreign members on a FeatureCollection and requires
# parsers to ignore ones they do not know, so the provenance travels with
# the file without breaking QGIS, ogr2ogr or any other reader.
PROVENANCE_MEMBER = "geointel_provenance"
@staticmethod
def provenance_member(
*,
source: str,
project_id: uuid.UUID,
dataset_id: uuid.UUID | None = None,
analysis_run_id: uuid.UUID | None = None,
source_name: str | None = None,
source_version: str | None = None,
observed_at: Any = None,
selection_bbox: dict[str, Any] | None = None,
selection_area_id: uuid.UUID | None = None,
feature_count: int | None = None,
total_feature_count: int | None = None,
truncated: bool = False,
warnings: list[str] | None = None,
) -> dict[str, Any]:
"""Describe an exported FeatureCollection inside the file itself.
A capped export previously recorded ``truncated`` on the export record
only, so the downloaded file looked complete. Completeness is derived
from the counts as well as the flag: a caller that forgets to pass the
flag cannot produce a file that claims to hold everything.
"""
complete = not truncated
if feature_count is not None and total_feature_count is not None:
complete = complete and feature_count >= total_feature_count
member: dict[str, Any] = {
"source": source,
"project_id": str(project_id),
"exported_at": datetime.now(timezone.utc).isoformat(),
"complete": complete,
"completeness_note": None,
}
if dataset_id is not None:
member["dataset_id"] = str(dataset_id)
if analysis_run_id is not None:
member["analysis_run_id"] = str(analysis_run_id)
if source_name:
member["source_name"] = source_name
if source_version:
member["source_version"] = source_version
if observed_at is not None:
member["observed_at"] = observed_at.isoformat() if hasattr(observed_at, "isoformat") else str(observed_at)
if selection_bbox is not None:
member["selection_bbox"] = selection_bbox
if selection_area_id is not None:
member["selection_area_id"] = str(selection_area_id)
if feature_count is not None:
member["feature_count"] = feature_count
if total_feature_count is not None:
member["total_feature_count"] = total_feature_count
if warnings:
member["warnings"] = list(warnings)
if not complete:
written = feature_count if feature_count is not None else "?"
available = total_feature_count if total_feature_count is not None else "?"
member["completeness_note"] = (
f"Dit bestand bevat {written} van {available} objecten uit de selectie. Het is een "
"begrensde uitsnede, geen volledige export."
)
return member
@staticmethod
def attach_provenance(feature_collection: dict[str, Any], member: dict[str, Any]) -> dict[str, Any]:
feature_collection[ExportService.PROVENANCE_MEMBER] = member
return feature_collection
@staticmethod
def _detection_export_trust(db: Session, run: AnalysisRun) -> dict[str, Any]:
"""Classify persisted AI output without turning confidence into truth."""
@@ -447,6 +524,28 @@ class ExportService:
preclipped_partition_filter=preclipped_partition_filter,
)
selection = VectorFeatureService.select_features_by_bbox(db, **selection_kwargs)
summary = selection.get("summary") if isinstance(selection.get("summary"), dict) else {}
ExportService.attach_provenance(
selection["geojson"],
ExportService.provenance_member(
source="vector_selection",
project_id=dataset.project_id,
dataset_id=dataset.id,
source_name=dataset.source_name,
source_version=dataset.source_version,
observed_at=dataset.observed_at,
selection_bbox=selection["selection_bbox"],
selection_area_id=area_id,
feature_count=selection["feature_count"],
total_feature_count=selection.get("total_feature_count"),
truncated=selection["truncated"],
warnings=[
warning
for warning in (summary.get("selection_edge_warning"), summary.get("warning"))
if warning
],
),
)
filename = ExportService._filename(name, f"{dataset.id}-selection.geojson", ".geojson")
export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename)
metadata = {
@@ -487,6 +586,18 @@ class ExportService:
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
feature_collection = DatasetService.get_dataset_geojson(db, dataset_id)
ExportService.attach_provenance(
feature_collection,
ExportService.provenance_member(
source="dataset",
project_id=dataset.project_id,
dataset_id=dataset.id,
source_name=dataset.source_name,
source_version=dataset.source_version,
observed_at=dataset.observed_at,
feature_count=len(feature_collection.get("features", [])),
),
)
filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson")
export_path = StorageService.dataset_export_path(str(dataset.project_id), str(dataset.id), filename)
metadata = {
@@ -530,6 +641,16 @@ class ExportService:
)
feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id)
feature_collection["geointel_result"] = trust
ExportService.attach_provenance(
feature_collection,
ExportService.provenance_member(
source="detection_run",
project_id=run.project_id,
dataset_id=run.dataset_id,
analysis_run_id=run.id,
feature_count=len(feature_collection.get("features", [])),
),
)
for feature in feature_collection.get("features", []):
properties = feature.get("properties") if isinstance(feature, dict) else None
if isinstance(properties, dict):
@@ -565,6 +686,16 @@ class ExportService:
ExportService._assert_run_source_dataset_exportable(db, run)
feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id)
ExportService.attach_provenance(
feature_collection,
ExportService.provenance_member(
source="segmentation_run",
project_id=run.project_id,
dataset_id=run.dataset_id,
analysis_run_id=run.id,
feature_count=len(feature_collection.get("features", [])),
),
)
filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson")
export_path = StorageService.dataset_export_path(str(run.project_id), str(run.dataset_id or run.id), filename)
metadata = {
backend/tests/test_export_file_states_completeness.py
+125
View File
@@ -0,0 +1,125 @@
"""A downloaded export must carry its own provenance and its own limits.
Counterpart to ``test_export_provenance_member``: this reads the file the
service actually writes, so it proves the member survives serialisation rather
than only that the helper builds it.
"""
from __future__ import annotations
import json
from datetime import datetime, timezone
from pathlib import Path
from uuid import uuid4
from app.models import Dataset, Export
from app.services.export_service import ExportService
from app.services.storage_service import StorageService
from app.services.vector_feature_service import VectorFeatureService
from tests.test_sprint107_map_selection_export import FakeSession, _govern_fixture_dataset
BBOX = {"min_x": 4.9, "min_y": 50.9, "max_x": 5.2, "max_y": 51.2, "crs": "EPSG:4326"}
def _dataset(dataset_id):
dataset = _govern_fixture_dataset(
Dataset(
id=dataset_id,
project_id=uuid4(),
name="grb-buildings.geojson",
dataset_type="vector",
source="grb",
source_name="grb",
source_version="2024-06",
observed_at=datetime(2024, 6, 1, tzinfo=timezone.utc),
status="ready",
)
)
return dataset
def _selection(*, feature_count: int, total: int, truncated: bool, warning: str | None = None):
return {
"selection_bbox": BBOX,
"feature_count": feature_count,
"total_feature_count": total,
"limit": 250,
"truncated": truncated,
"summary": {"selection_edge_warning": warning} if warning else {},
"geojson": {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {"vector_feature_id": f"vf-{index}"},
}
for index in range(feature_count)
],
},
}
def _export(monkeypatch, tmp_path: Path, selection) -> dict:
dataset_id = uuid4()
dataset = _dataset(dataset_id)
db = FakeSession({(Dataset, dataset_id): dataset})
export_path = tmp_path / "selection.geojson"
monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path))
monkeypatch.setattr(VectorFeatureService, "select_features_by_bbox", lambda *_args, **_kwargs: selection)
ExportService.export_vector_selection_geojson(db, dataset_id, BBOX, limit=250, name="selection")
assert [item for item in db.added if isinstance(item, Export)]
return json.loads(export_path.read_text(encoding="utf-8"))
def test_a_truncated_export_file_states_that_it_is_partial(monkeypatch, tmp_path: Path) -> None:
written = _export(monkeypatch, tmp_path, _selection(feature_count=2, total=1_400, truncated=True))
provenance = written["geointel_provenance"]
assert provenance["complete"] is False
assert "1400" in provenance["completeness_note"].replace(".", "")
# The features are still there; the file simply no longer implies it holds
# everything the selection contains.
assert len(written["features"]) == 2
def test_a_complete_export_file_says_so(monkeypatch, tmp_path: Path) -> None:
written = _export(monkeypatch, tmp_path, _selection(feature_count=2, total=2, truncated=False))
provenance = written["geointel_provenance"]
assert provenance["complete"] is True
assert provenance["completeness_note"] is None
def test_the_file_identifies_its_source_edition(monkeypatch, tmp_path: Path) -> None:
written = _export(monkeypatch, tmp_path, _selection(feature_count=1, total=1, truncated=False))
provenance = written["geointel_provenance"]
assert provenance["source_name"] == "grb"
assert provenance["source_version"] == "2024-06"
assert provenance["observed_at"].startswith("2024-06-01")
assert provenance["selection_bbox"] == BBOX
def test_selection_caveats_travel_into_the_file(monkeypatch, tmp_path: Path) -> None:
written = _export(
monkeypatch,
tmp_path,
_selection(feature_count=1, total=1, truncated=False, warning="22 objecten liggen deels buiten de selectie."),
)
assert written["geointel_provenance"]["warnings"] == ["22 objecten liggen deels buiten de selectie."]
def test_the_file_remains_a_valid_feature_collection(monkeypatch, tmp_path: Path) -> None:
"""The provenance is a foreign member, not a change to the GeoJSON shape."""
written = _export(monkeypatch, tmp_path, _selection(feature_count=1, total=1, truncated=False))
assert written["type"] == "FeatureCollection"
assert isinstance(written["features"], list)
assert written["features"][0]["type"] == "Feature"
backend/tests/test_export_provenance_member.py
+129
View File
@@ -0,0 +1,129 @@
"""An exported GeoJSON must say what it is and what it leaves out.
The vector selection export caps its features and records ``truncated`` in the
export *record*. The file itself said nothing: an operator downloads
``mol-selection.geojson``, opens it in QGIS and sees 250 buildings where the
workbench said 1.400, with nothing in the file to indicate the difference.
For a product whose promise is that an export is a reproducible result, a file
that looks complete and is not is the sharpest possible violation. RFC 7946
allows foreign members on a FeatureCollection, and the detection export already
uses one; this makes that convention uniform.
"""
from __future__ import annotations
from datetime import datetime, timezone
from uuid import uuid4
from app.services.export_service import ExportService
def test_provenance_names_the_source_and_the_moment() -> None:
dataset_id = uuid4()
project_id = uuid4()
member = ExportService.provenance_member(
source="vector_selection",
project_id=project_id,
dataset_id=dataset_id,
source_name="grb",
source_version="2024-06",
observed_at=datetime(2024, 6, 1, tzinfo=timezone.utc),
)
assert member["source"] == "vector_selection"
assert member["dataset_id"] == str(dataset_id)
assert member["project_id"] == str(project_id)
assert member["source_name"] == "grb"
assert member["source_version"] == "2024-06"
assert member["observed_at"].startswith("2024-06-01")
assert member["exported_at"]
def test_a_truncated_export_says_so_in_words() -> None:
member = ExportService.provenance_member(
source="vector_selection",
project_id=uuid4(),
dataset_id=uuid4(),
feature_count=250,
total_feature_count=1_400,
truncated=True,
)
assert member["complete"] is False
assert member["feature_count"] == 250
assert member["total_feature_count"] == 1_400
assert "1400" in member["completeness_note"].replace(".", "").replace(",", "")
assert "250" in member["completeness_note"]
def test_a_complete_export_is_stated_as_complete() -> None:
member = ExportService.provenance_member(
source="detection_run",
project_id=uuid4(),
dataset_id=uuid4(),
feature_count=12,
total_feature_count=12,
truncated=False,
)
assert member["complete"] is True
assert member["completeness_note"] is None
def test_counts_that_disagree_are_treated_as_incomplete() -> None:
"""A caller that forgets the flag must not produce a file claiming completeness."""
member = ExportService.provenance_member(
source="vector_selection",
project_id=uuid4(),
dataset_id=uuid4(),
feature_count=100,
total_feature_count=140,
truncated=False,
)
assert member["complete"] is False
def test_selection_context_travels_with_the_file() -> None:
area_id = uuid4()
bbox = {"min_x": 5.0, "min_y": 51.1, "max_x": 5.2, "max_y": 51.3, "crs": "EPSG:4326"}
member = ExportService.provenance_member(
source="vector_selection",
project_id=uuid4(),
dataset_id=uuid4(),
selection_bbox=bbox,
selection_area_id=area_id,
warnings=["22 van de 100 objecten liggen deels buiten de selectie."],
)
assert member["selection_bbox"] == bbox
assert member["selection_area_id"] == str(area_id)
assert member["warnings"] == ["22 van de 100 objecten liggen deels buiten de selectie."]
def test_empty_context_is_omitted_rather_than_written_as_null_noise() -> None:
member = ExportService.provenance_member(
source="dataset",
project_id=uuid4(),
dataset_id=uuid4(),
)
assert "selection_bbox" not in member
assert "warnings" not in member
assert "source_version" not in member
def test_the_member_attaches_under_a_reserved_key() -> None:
collection = {"type": "FeatureCollection", "features": []}
ExportService.attach_provenance(
collection,
ExportService.provenance_member(source="dataset", project_id=uuid4(), dataset_id=uuid4()),
)
assert collection["type"] == "FeatureCollection"
assert collection["geointel_provenance"]["source"] == "dataset"