feat(provenance): govern source snapshots and data inputs

This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
+40 -7
View File
@@ -17,7 +17,7 @@ from app.core.config import Settings
from app.core.errors import AppError
from app.db.session import get_db
from app.main import app
from app.models import Area, Dataset, DatasetVersion, Job, Project
from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot
from app.schemas.dhmv import DhmvAcquireRequest, TerrainPartitionSelectionRequest, TerrainSelectionRequest
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.terrain_analysis_service import TerrainAnalysisService
@@ -27,8 +27,8 @@ ROOT = Path(__file__).resolve().parents[2]
class FakeQuery:
def __init__(self, result=None):
self.result = result
def __init__(self, results=None):
self.results = list(results or [])
def filter(self, *_args):
return self
@@ -37,10 +37,13 @@ class FakeQuery:
return self
def first(self):
return self.result
return self.results[0] if self.results else None
def one_or_none(self):
return self.first()
def all(self):
return self.result if isinstance(self.result, list) else []
return list(self.results)
class FakeSession:
@@ -58,6 +61,14 @@ class FakeSession:
def add(self, row):
self.added.append(row)
def flush(self):
# Exercise the governed source/snapshot import path with database-like
# primary-key assignment instead of silently falling back to legacy
# fixture behavior.
for row in self.added:
if getattr(row, "id", None) is None:
row.id = uuid4()
def commit(self):
return None
@@ -67,8 +78,18 @@ class FakeSession:
def refresh(self, row):
return row
def query(self, _model):
return FakeQuery(self.query_result)
def query(self, model):
rows = [
row
for (row_model, _row_id), row in self.rows.items()
if row_model is model and isinstance(row, model)
]
rows.extend(row for row in self.added if isinstance(row, model))
if isinstance(self.query_result, model):
rows.append(self.query_result)
elif isinstance(self.query_result, list):
rows.extend(row for row in self.query_result if isinstance(row, model))
return FakeQuery(rows)
class FakeResponse:
@@ -318,6 +339,8 @@ def test_dhmv_acquisition_clips_validates_and_persists_via_dataset_service(tmp_p
dataset = next(item for item in db.added if isinstance(item, Dataset))
version = next(item for item in db.added if isinstance(item, DatasetVersion))
source = next(item for item in db.added if isinstance(item, SourceRegistry))
snapshot = next(item for item in db.added if isinstance(item, SourceSnapshot))
assert result["output_dataset_id"] == str(dataset.id)
assert dataset.source_name == "digitaal_vlaanderen_dhmv"
assert dataset.area_id == area_id
@@ -331,6 +354,16 @@ def test_dhmv_acquisition_clips_validates_and_persists_via_dataset_service(tmp_p
assert dataset.provenance_metadata["water_depth_available"] is False
assert dataset.provenance_metadata["water_volume_available"] is False
assert len(dataset.provenance_metadata["response_sha256"]) == 64
assert dataset.source_registry_id == source.id
assert dataset.source_snapshot_id == snapshot.id
assert dataset.validation_status == "passed"
assert dataset.provenance_status == "complete"
assert dataset.lineage_status == "not_applicable"
assert dataset.quarantine_status == "not_quarantined"
assert snapshot.source_registry_id == source.id
assert snapshot.checksum_sha256 == dataset.checksum_sha256
assert snapshot.ingest_status == "ingested"
assert snapshot.freshness_status == "current"
with rasterio.open(dataset.storage_path) as stored:
assert stored.crs.to_epsg() == 31370
assert stored.count == 1