feat(provenance): govern source snapshots and data inputs

This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
+41 -21
View File
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
from app.schemas.demo import DemoWorkflowResponse
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
from app.services.geojson_service import parse_geojson_payload
from app.services.qa_service import QaService
from app.services.quality_service import QualityService
@@ -30,17 +31,17 @@ class DemoWorkflowService:
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
@staticmethod
def _add_initial_version(db: Session, dataset: Dataset) -> None:
db.add(
DatasetVersion(
dataset_id=dataset.id,
version=1,
storage_path=dataset.storage_path,
checksum_sha256=dataset.checksum_sha256,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
)
def _add_initial_version(db: Session, dataset: Dataset) -> DatasetVersion:
version = DatasetVersion(
dataset_id=dataset.id,
version=1,
storage_path=dataset.storage_path,
checksum_sha256=dataset.checksum_sha256,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
)
db.add(version)
return version
@staticmethod
def _repo_root() -> Path:
@@ -233,24 +234,34 @@ class DemoWorkflowService:
crs=metadata.get("crs"),
bounds_json=metadata.get("bounds_json"),
metadata_json=metadata,
status="ready",
status="validating",
)
db.add(dataset)
DemoWorkflowService._add_initial_version(db, dataset)
version = DemoWorkflowService._add_initial_version(db, dataset)
is_ready = DerivedDatasetGovernanceService.govern_vector(
db,
dataset=dataset,
dataset_version=version,
feature_collection=payload,
source_key="fixture",
operation="demo.fixture_vector",
operation_parameters={"fixture_name": filename, "role": role},
)
if is_ready:
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=dataset.id,
payload=payload,
feature_class=reference_layer_name or "building",
commit=False,
)
db.commit()
db.refresh(dataset)
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=dataset.id,
payload=payload,
feature_class=reference_layer_name or "building",
)
return dataset
@staticmethod
def _create_demo_raster_bytes() -> bytes:
numpy = importlib.import_module("numpy")
rasterio = importlib.import_module("rasterio")
rasterio_io = importlib.import_module("rasterio.io")
rasterio_transform = importlib.import_module("rasterio.transform")
@@ -318,10 +329,19 @@ class DemoWorkflowService:
crs=metadata.get("crs"),
bounds_json=bounds_json,
metadata_json=metadata,
status="ready",
status="validating",
)
db.add(dataset)
DemoWorkflowService._add_initial_version(db, dataset)
version = DemoWorkflowService._add_initial_version(db, dataset)
DerivedDatasetGovernanceService.govern_raster(
db,
dataset=dataset,
dataset_version=version,
raster_metadata=metadata,
source_key="fixture",
operation="demo.fixture_raster",
operation_parameters={"fixture_name": DemoWorkflowService.RASTER_FILENAME},
)
db.commit()
db.refresh(dataset)
return dataset