feat(provenance): govern source snapshots and data inputs
This commit is contained in:
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
|
||||
from app.schemas.demo import DemoWorkflowResponse
|
||||
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
|
||||
from app.services.geojson_service import parse_geojson_payload
|
||||
from app.services.qa_service import QaService
|
||||
from app.services.quality_service import QualityService
|
||||
@@ -30,17 +31,17 @@ class DemoWorkflowService:
|
||||
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
|
||||
|
||||
@staticmethod
|
||||
def _add_initial_version(db: Session, dataset: Dataset) -> None:
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
def _add_initial_version(db: Session, dataset: Dataset) -> DatasetVersion:
|
||||
version = DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
db.add(version)
|
||||
return version
|
||||
|
||||
@staticmethod
|
||||
def _repo_root() -> Path:
|
||||
@@ -233,24 +234,34 @@ class DemoWorkflowService:
|
||||
crs=metadata.get("crs"),
|
||||
bounds_json=metadata.get("bounds_json"),
|
||||
metadata_json=metadata,
|
||||
status="ready",
|
||||
status="validating",
|
||||
)
|
||||
db.add(dataset)
|
||||
DemoWorkflowService._add_initial_version(db, dataset)
|
||||
version = DemoWorkflowService._add_initial_version(db, dataset)
|
||||
is_ready = DerivedDatasetGovernanceService.govern_vector(
|
||||
db,
|
||||
dataset=dataset,
|
||||
dataset_version=version,
|
||||
feature_collection=payload,
|
||||
source_key="fixture",
|
||||
operation="demo.fixture_vector",
|
||||
operation_parameters={"fixture_name": filename, "role": role},
|
||||
)
|
||||
if is_ready:
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
db=db,
|
||||
dataset_id=dataset.id,
|
||||
payload=payload,
|
||||
feature_class=reference_layer_name or "building",
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
db=db,
|
||||
dataset_id=dataset.id,
|
||||
payload=payload,
|
||||
feature_class=reference_layer_name or "building",
|
||||
)
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def _create_demo_raster_bytes() -> bytes:
|
||||
numpy = importlib.import_module("numpy")
|
||||
rasterio = importlib.import_module("rasterio")
|
||||
rasterio_io = importlib.import_module("rasterio.io")
|
||||
rasterio_transform = importlib.import_module("rasterio.transform")
|
||||
|
||||
@@ -318,10 +329,19 @@ class DemoWorkflowService:
|
||||
crs=metadata.get("crs"),
|
||||
bounds_json=bounds_json,
|
||||
metadata_json=metadata,
|
||||
status="ready",
|
||||
status="validating",
|
||||
)
|
||||
db.add(dataset)
|
||||
DemoWorkflowService._add_initial_version(db, dataset)
|
||||
version = DemoWorkflowService._add_initial_version(db, dataset)
|
||||
DerivedDatasetGovernanceService.govern_raster(
|
||||
db,
|
||||
dataset=dataset,
|
||||
dataset_version=version,
|
||||
raster_metadata=metadata,
|
||||
source_key="fixture",
|
||||
operation="demo.fixture_raster",
|
||||
operation_parameters={"fixture_name": DemoWorkflowService.RASTER_FILENAME},
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
return dataset
|
||||
|
||||
Reference in New Issue
Block a user