Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(provenance): govern source snapshots and data inputs

Browse Source
This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/vector_operations_service.py
+128 -25
View File
@@ -3,13 +3,17 @@ from __future__ import annotations
import json
import uuid
from datetime import datetime, timezone
from hashlib import sha256
from pathlib import Path
import re
from typing import Any
from geoalchemy2.shape import to_shape
from pyproj import CRS, Transformer
from shapely.geometry import GeometryCollection, MultiPolygon, shape
from shapely.geometry.base import BaseGeometry
from shapely.geometry import mapping
from shapely.ops import transform as shapely_transform
from shapely.ops import unary_union
from shapely.validation import make_valid
from sqlalchemy.orm import Session
@@ -18,12 +22,16 @@ from app.core.errors import AppError
from app.models import Area, Dataset, DatasetVersion
from app.schemas.dataset import DatasetCreateResponse
from app.schemas.operations import VectorOperationResult
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
from app.services.geojson_service import parse_geojson_payload
from app.services.storage_service import StorageService
from app.services.vector_feature_service import VectorFeatureService
class VectorOperationsService:
CANONICAL_VECTOR_CRS = "EPSG:4326"
_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
@staticmethod
def _require_vector_dataset(dataset: Dataset) -> None:
if dataset.dataset_type not in {"vector", "geojson"}:
@@ -37,7 +45,8 @@ class VectorOperationsService:
if not path.exists():
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
stored_bytes = path.read_bytes()
payload = json.loads(stored_bytes.decode("utf-8"))
except Exception as exc:
raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=400) from exc
@@ -47,6 +56,55 @@ class VectorOperationsService:
features = payload.get("features")
if not isinstance(features, list):
raise AppError(code="INVALID_GEOJSON", message="Dataset payload is missing features", status_code=400)
# The normal Dataset storage path is a consumption artifact, not a
# provenance source archive. Refuse projected/original source bytes
# here rather than letting a spatial operation interpret them as
# canonical map coordinates.
raw_crs = payload.get("crs")
if isinstance(raw_crs, dict):
crs_properties = raw_crs.get("properties")
raw_crs = crs_properties.get("name") if isinstance(crs_properties, dict) else None
stored_crs = str(raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS).strip().upper()
dataset_crs = str(dataset.crs or "").strip().upper()
if stored_crs != VectorOperationsService.CANONICAL_VECTOR_CRS or (
dataset_crs and dataset_crs != VectorOperationsService.CANONICAL_VECTOR_CRS
):
raise AppError(
code="DATASET_STORAGE_CRS_MISMATCH",
message="Vector operations require canonical EPSG:4326 dataset storage.",
details={
"stored_crs": raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS,
"dataset_crs": dataset.crs,
"expected_crs": VectorOperationsService.CANONICAL_VECTOR_CRS,
},
status_code=409,
)
expected_checksum = str(dataset.checksum_sha256 or "").strip().lower()
actual_checksum = sha256(stored_bytes).hexdigest()
governed_artifact = bool(
getattr(dataset, "data_contract_key", None)
or (
isinstance(getattr(dataset, "metadata_json", None), dict)
and dataset.metadata_json.get("canonical_storage_crs")
)
)
if expected_checksum and VectorOperationsService._CHECKSUM_SHA256.fullmatch(expected_checksum):
if expected_checksum != actual_checksum:
raise AppError(
code="DATASET_STORAGE_CHECKSUM_MISMATCH",
message="Vector dataset storage no longer matches its validated checksum.",
details={"expected_checksum_sha256": expected_checksum, "actual_checksum_sha256": actual_checksum},
status_code=409,
)
elif governed_artifact:
raise AppError(
code="DATASET_STORAGE_CHECKSUM_UNVERIFIABLE",
message="Governed vector storage requires a valid SHA-256 checksum before use.",
details={"checksum_sha256": dataset.checksum_sha256},
status_code=409,
)
return payload, [feature for feature in features if isinstance(feature, dict)]
@staticmethod
@@ -73,6 +131,25 @@ class VectorOperationsService:
raise AppError(code="EMPTY_VECTOR_DATASET", message="Dataset has no parseable geometries", status_code=422)
return geometries
@staticmethod
def _buffer_in_metres(geometry: BaseGeometry, distance_m: float, source_crs: str) -> BaseGeometry:
"""Buffer in a Belgian projected CRS, never in angular degrees."""
try:
input_crs = CRS.from_user_input(source_crs)
metric_crs = CRS.from_epsg(31370)
if input_crs == metric_crs:
return geometry.buffer(distance_m)
forward = Transformer.from_crs(input_crs, metric_crs, always_xy=True)
backward = Transformer.from_crs(metric_crs, input_crs, always_xy=True)
return shapely_transform(backward.transform, shapely_transform(forward.transform, geometry).buffer(distance_m))
except Exception as exc:
raise AppError(
code="INVALID_CRS",
message="A valid explicit CRS is required for metre-based vector buffering.",
status_code=400,
) from exc
@staticmethod
def inspect(db: Session, dataset_id: uuid.UUID) -> VectorOperationResult:
dataset = db.get(Dataset, dataset_id)
@@ -94,7 +171,7 @@ class VectorOperationsService:
feature_count=len(geometries),
geometry_type_summary=geometry_type_summary,
bounds_json={"min_x": float(bounds[0]), "min_y": float(bounds[1]), "max_x": float(bounds[2]), "max_y": float(bounds[3])},
crs=payload.get("crs") if isinstance(payload.get("crs"), str) else dataset.crs,
crs=VectorOperationsService.CANONICAL_VECTOR_CRS,
)
@staticmethod
@@ -184,9 +261,13 @@ class VectorOperationsService:
if distance_m <= 0:
raise AppError(code="INVALID_PARAMETERS", message="distance_m must be greater than 0", status_code=400)
_, features = VectorOperationsService._load_dataset_payload(source_dataset)
payload, features = VectorOperationsService._load_dataset_payload(source_dataset)
source_crs = VectorOperationsService.CANONICAL_VECTOR_CRS
geometries = VectorOperationsService._extract_geometries(features)
buffered_features = [(feature, geometry.buffer(distance_m)) for feature, geometry in geometries]
buffered_features = [
(feature, VectorOperationsService._buffer_in_metres(geometry, distance_m, source_crs))
for feature, geometry in geometries
]
output_features: list[dict[str, Any]] = []
for feature, geometry in buffered_features:
@@ -431,7 +512,14 @@ class VectorOperationsService:
if not output_name_value.strip():
output_name_value = f"{default_name}.geojson"
stored = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
# All current governed vector storage is EPSG:4326. A legacy source
# with another CRS is not relabelled here: the derived contract will
# quarantine the result instead of placing non-WGS84 coordinates on
# the map as if they were WGS84.
output_crs = VectorOperationsService.CANONICAL_VECTOR_CRS
output_feature_collection = dict(feature_collection)
output_feature_collection["crs"] = output_crs
stored = json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
storage_info = StorageService.persist_dataset_file(
project_id=str(source_dataset.project_id),
dataset_id=str(derived_id),
@@ -441,7 +529,7 @@ class VectorOperationsService:
content_type="application/geo+json",
)
metadata = parse_geojson_payload(json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")))
metadata = parse_geojson_payload(json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":")))
if metadata_extra:
metadata.update(metadata_extra)
derived_dataset = Dataset(
@@ -452,7 +540,7 @@ class VectorOperationsService:
dataset_type="vector",
source=f"operation:{operation}",
dataset_role=dataset_role,
source_name=source_name,
source_name=source_name or "derived",
source_metadata=source_metadata,
provenance_metadata=provenance_metadata,
imported_at=datetime.now(timezone.utc),
@@ -478,29 +566,44 @@ class VectorOperationsService:
resolution_json=metadata.get("resolution_json"),
bands_json=metadata.get("bands_json"),
metadata_json=metadata,
status="ready",
status="validating",
)
db.add(derived_dataset)
db.add(
DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
dataset_version = DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
db.commit()
db.refresh(derived_dataset)
if persist_vector_features:
db.add(dataset_version)
derived_source_key = "map_selection" if source_name == "map_selection" else "derived"
is_ready = DerivedDatasetGovernanceService.govern_vector(
db,
dataset=derived_dataset,
dataset_version=dataset_version,
feature_collection=output_feature_collection,
source_key=derived_source_key,
operation=f"vector.{operation}",
parent_dataset=source_dataset,
operation_parameters={
"operation": operation,
"output_name": output_name_value,
**(metadata_extra or {}),
},
)
if persist_vector_features and is_ready:
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=derived_dataset.id,
payload=feature_collection,
payload=output_feature_collection,
commit=False,
)
db.commit()
db.refresh(derived_dataset)
return derived_id