645 lines
21 KiB
Python
645 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from hashlib import sha256
|
|
import json
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from uuid import uuid4
|
|
|
|
from app.models import Dataset, DatasetVersion, SourceRegistry, SourceSnapshot
|
|
from app.services.data_contract_validation import (
|
|
build_vector_ingest_input,
|
|
validate_registered_asset,
|
|
)
|
|
from app.services.demo_workflow_service import DemoWorkflowService
|
|
from app.services.derived_dataset_governance_service import (
|
|
DerivedDatasetGovernanceService,
|
|
)
|
|
from app.services.raster_operations_service import RasterOperationsService
|
|
from app.services.storage_service import StorageService
|
|
from app.services.vector_feature_service import VectorFeatureService
|
|
from app.services.vector_operations_service import VectorOperationsService
|
|
|
|
|
|
_CHECKSUM = "a" * 64
|
|
|
|
|
|
class _FakeSession:
|
|
def __init__(self, rows=None) -> None:
|
|
self.rows = rows or {}
|
|
self.added = []
|
|
|
|
def get(self, model, row_id):
|
|
row = self.rows.get((model, row_id))
|
|
if row is not None:
|
|
return row
|
|
return next(
|
|
(
|
|
item
|
|
for item in self.added
|
|
if isinstance(item, model) and item.id == row_id
|
|
),
|
|
None,
|
|
)
|
|
|
|
def add(self, row) -> None:
|
|
self.added.append(row)
|
|
|
|
def commit(self) -> None:
|
|
return None
|
|
|
|
def refresh(self, row) -> None:
|
|
return None
|
|
|
|
|
|
class _GovernedSession:
|
|
"""Small ORM-shaped session for the real governance branch.
|
|
|
|
Registry persistence is monkeypatched below; the test exercises the
|
|
service's orchestration and report decisions without needing PostGIS.
|
|
"""
|
|
|
|
def __init__(self) -> None:
|
|
self.added = []
|
|
self.flushes = 0
|
|
|
|
class _EmptyQuery:
|
|
def filter(self, *_args, **_kwargs):
|
|
return self
|
|
|
|
@staticmethod
|
|
def all():
|
|
return []
|
|
|
|
@staticmethod
|
|
def one_or_none():
|
|
return None
|
|
|
|
def query(self, *_args, **_kwargs):
|
|
# Governance now performs a bounded lineage traversal during quarantine.
|
|
# This focused harness intentionally has no persisted siblings/edges.
|
|
return self._EmptyQuery()
|
|
|
|
def add(self, row) -> None:
|
|
self.added.append(row)
|
|
|
|
def flush(self) -> None:
|
|
self.flushes += 1
|
|
|
|
|
|
def _governed_parent() -> Dataset:
|
|
source_id = uuid4()
|
|
snapshot_id = uuid4()
|
|
source = SourceRegistry(
|
|
id=source_id,
|
|
source_key="grb",
|
|
display_name="GRB parent fixture",
|
|
classification="authoritative",
|
|
authority_name="Digitaal Vlaanderen",
|
|
authority_scope_json={"zone": "Flanders"},
|
|
usage_policy_json={"ground_truth_allowed": True},
|
|
)
|
|
snapshot = SourceSnapshot(
|
|
id=snapshot_id,
|
|
source_registry_id=source_id,
|
|
snapshot_key="governed-parent",
|
|
checksum_sha256=_CHECKSUM,
|
|
freshness_status="current",
|
|
ingest_status="ingested",
|
|
)
|
|
dataset = Dataset(
|
|
id=uuid4(),
|
|
project_id=uuid4(),
|
|
name="governed.geojson",
|
|
dataset_type="vector",
|
|
source="grb",
|
|
source_name="grb",
|
|
status="ready",
|
|
checksum_sha256=_CHECKSUM,
|
|
data_contract_key="geointel.vector.geojson",
|
|
data_contract_version="1.0.0",
|
|
validation_status="passed",
|
|
provenance_status="complete",
|
|
lineage_status="not_applicable",
|
|
quarantine_status="not_quarantined",
|
|
source_registry_id=source_id,
|
|
source_snapshot_id=snapshot_id,
|
|
)
|
|
dataset.source_registry = source
|
|
dataset.source_snapshot = snapshot
|
|
return dataset
|
|
|
|
|
|
def test_lineage_evidence_quarantines_ungoverned_parent_without_inventing_a_checksum() -> (
|
|
None
|
|
):
|
|
parent = _governed_parent()
|
|
valid_lineage = DerivedDatasetGovernanceService._lineage_evidence(
|
|
parent, "vector.clip", {"area_id": "a"}
|
|
)
|
|
valid_report = validate_registered_asset(
|
|
build_vector_ingest_input(
|
|
asset_id="derived-valid",
|
|
source_crs="EPSG:4326",
|
|
storage_crs="EPSG:4326",
|
|
feature_collection={
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
|
"properties": {},
|
|
}
|
|
],
|
|
},
|
|
checksum_sha256=_CHECKSUM,
|
|
computed_checksum_sha256=_CHECKSUM,
|
|
source_registry_id="derived-source",
|
|
source_snapshot_id="derived-snapshot",
|
|
imported_at=datetime.now(timezone.utc),
|
|
metadata={
|
|
"license": "internal derived artifact",
|
|
"bounds_json": {
|
|
"min_x": 5.0,
|
|
"min_y": 51.0,
|
|
"max_x": 5.0,
|
|
"max_y": 51.0,
|
|
},
|
|
},
|
|
temporal_unknown_reason="derived input has no precise observation timestamp",
|
|
source_version_unknown_reason="transform version is recorded separately",
|
|
lineage=valid_lineage,
|
|
)
|
|
)
|
|
|
|
assert valid_report.validation_status.value == "passed"
|
|
assert valid_lineage.upstream_asset_ids == (str(parent.id),)
|
|
assert valid_lineage.upstream_checksums_sha256 == (_CHECKSUM,)
|
|
|
|
parent.validation_status = "not_validated"
|
|
rejected_lineage = DerivedDatasetGovernanceService._lineage_evidence(
|
|
parent, "vector.clip", {}
|
|
)
|
|
rejected_report = validate_registered_asset(
|
|
build_vector_ingest_input(
|
|
asset_id="derived-rejected",
|
|
source_crs="EPSG:4326",
|
|
storage_crs="EPSG:4326",
|
|
feature_collection={
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
|
"properties": {},
|
|
}
|
|
],
|
|
},
|
|
checksum_sha256=_CHECKSUM,
|
|
computed_checksum_sha256=_CHECKSUM,
|
|
source_registry_id="derived-source",
|
|
source_snapshot_id="derived-snapshot",
|
|
imported_at=datetime.now(timezone.utc),
|
|
metadata={
|
|
"license": "internal derived artifact",
|
|
"bounds_json": {
|
|
"min_x": 5.0,
|
|
"min_y": 51.0,
|
|
"max_x": 5.0,
|
|
"max_y": 51.0,
|
|
},
|
|
},
|
|
temporal_unknown_reason="derived input has no precise observation timestamp",
|
|
source_version_unknown_reason="transform version is recorded separately",
|
|
lineage=rejected_lineage,
|
|
)
|
|
)
|
|
|
|
assert rejected_lineage.upstream_checksums_sha256 == (
|
|
"parent_dataset_not_governed",
|
|
)
|
|
assert rejected_report.validation_status.value == "failed"
|
|
assert rejected_report.quarantine_status.value == "quarantined"
|
|
assert any(
|
|
issue.code == "UPSTREAM_CHECKSUM_FORMAT_INVALID"
|
|
for issue in rejected_report.issues
|
|
)
|
|
|
|
|
|
def test_govern_vector_binds_snapshot_contract_and_lineage_before_marking_ready(
|
|
monkeypatch,
|
|
) -> None:
|
|
from app.services.source_registry_service import SourceRegistryService
|
|
|
|
db = _GovernedSession()
|
|
parent = _governed_parent()
|
|
parent_version_id = uuid4()
|
|
dataset = Dataset(
|
|
id=uuid4(),
|
|
project_id=parent.project_id,
|
|
name="derived.geojson",
|
|
dataset_type="vector",
|
|
source="operation:clip",
|
|
source_name="derived",
|
|
checksum_sha256=_CHECKSUM,
|
|
imported_at=datetime.now(timezone.utc),
|
|
crs="EPSG:4326",
|
|
metadata_json={
|
|
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
|
|
},
|
|
status="validating",
|
|
)
|
|
version = DatasetVersion(
|
|
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
|
|
)
|
|
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
|
|
snapshot = SimpleNamespace(id=uuid4(), source_registry_id=source.id)
|
|
edges = []
|
|
|
|
monkeypatch.setattr(
|
|
SourceRegistryService,
|
|
"ensure_server_owned_source",
|
|
lambda *_args, **_kwargs: source,
|
|
)
|
|
monkeypatch.setattr(
|
|
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
|
|
)
|
|
monkeypatch.setattr(
|
|
DerivedDatasetGovernanceService,
|
|
"_latest_parent_version_id",
|
|
lambda *_args: parent_version_id,
|
|
)
|
|
|
|
def _bind(target, **kwargs):
|
|
target.source_registry_id = kwargs["source"].id
|
|
target.source_snapshot_id = kwargs["snapshot"].id
|
|
target.data_contract_key = kwargs["data_contract_key"]
|
|
target.data_contract_version = kwargs["data_contract_version"]
|
|
target.validation_status = kwargs["validation_status"]
|
|
target.provenance_status = kwargs["provenance_status"]
|
|
target.lineage_status = kwargs["lineage_status"]
|
|
return target
|
|
|
|
monkeypatch.setattr(SourceRegistryService, "bind_dataset_provenance", _bind)
|
|
monkeypatch.setattr(SourceRegistryService, "bind_dataset_version_provenance", _bind)
|
|
monkeypatch.setattr(
|
|
SourceRegistryService,
|
|
"record_lineage_edge",
|
|
lambda *_args, **kwargs: edges.append(kwargs),
|
|
)
|
|
|
|
ready = DerivedDatasetGovernanceService.govern_vector(
|
|
db,
|
|
dataset=dataset,
|
|
dataset_version=version,
|
|
feature_collection={
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
|
"properties": {},
|
|
}
|
|
],
|
|
},
|
|
source_key="derived",
|
|
operation="vector.clip",
|
|
parent_dataset=parent,
|
|
operation_parameters={"area_id": "a"},
|
|
)
|
|
|
|
assert ready is True
|
|
assert dataset.status == "ready"
|
|
assert dataset.validation_status == "passed"
|
|
assert dataset.provenance_status == "complete"
|
|
assert dataset.source_registry_id == source.id
|
|
assert version.source_snapshot_id == snapshot.id
|
|
assert db.flushes >= 1
|
|
assert edges[0]["parent_dataset_id"] == parent.id
|
|
assert edges[0]["parent_dataset_version_id"] == parent_version_id
|
|
assert edges[0]["child_dataset_version_id"] == version.id
|
|
|
|
|
|
def test_govern_vector_quarantines_output_when_parent_is_manual_or_experimental(
|
|
monkeypatch,
|
|
) -> None:
|
|
from app.services.source_registry_service import SourceRegistryService
|
|
|
|
db = _GovernedSession()
|
|
parent = _governed_parent()
|
|
parent.source = "manual"
|
|
parent.source_name = "manual"
|
|
parent.source_registry.source_key = "manual"
|
|
parent.source_registry.classification = "experimental"
|
|
parent.source_registry.usage_policy_json = {"ground_truth_allowed": False}
|
|
dataset = Dataset(
|
|
id=uuid4(),
|
|
project_id=parent.project_id,
|
|
name="manual-derived.geojson",
|
|
dataset_type="vector",
|
|
source="operation:clip",
|
|
source_name="derived",
|
|
checksum_sha256=_CHECKSUM,
|
|
imported_at=datetime.now(timezone.utc),
|
|
crs="EPSG:4326",
|
|
metadata_json={
|
|
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
|
|
},
|
|
status="validating",
|
|
)
|
|
version = DatasetVersion(
|
|
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
|
|
)
|
|
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
|
|
snapshot = SimpleNamespace(
|
|
id=uuid4(), source_registry_id=source.id, ingest_status="ingested"
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
SourceRegistryService,
|
|
"ensure_server_owned_source",
|
|
lambda *_args, **_kwargs: source,
|
|
)
|
|
monkeypatch.setattr(
|
|
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
|
|
)
|
|
monkeypatch.setattr(
|
|
DerivedDatasetGovernanceService,
|
|
"_latest_parent_version_id",
|
|
lambda *_args: None,
|
|
)
|
|
monkeypatch.setattr(
|
|
SourceRegistryService, "record_lineage_edge", lambda *_args, **_kwargs: None
|
|
)
|
|
|
|
ready = DerivedDatasetGovernanceService.govern_vector(
|
|
db,
|
|
dataset=dataset,
|
|
dataset_version=version,
|
|
feature_collection={
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
|
"properties": {},
|
|
}
|
|
],
|
|
},
|
|
source_key="derived",
|
|
operation="vector.clip",
|
|
parent_dataset=parent,
|
|
)
|
|
|
|
assert ready is False
|
|
assert dataset.status == "quarantined"
|
|
assert dataset.quarantine_status == "quarantined"
|
|
assert dataset.validation_status == "failed"
|
|
assert any(
|
|
issue["code"] == "PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING"
|
|
for issue in dataset.validation_report_json["issues"]
|
|
)
|
|
assert snapshot.ingest_status == "quarantined"
|
|
|
|
|
|
def test_vector_selection_uses_map_selection_registry_and_skips_features_when_quarantined(
|
|
monkeypatch, tmp_path
|
|
) -> None:
|
|
source = _governed_parent()
|
|
source.area_id = None
|
|
source.storage_path = str(tmp_path / "source.geojson")
|
|
db = _FakeSession({(Dataset, source.id): source})
|
|
output_path = tmp_path / "selection.geojson"
|
|
calls = []
|
|
persisted_features = []
|
|
|
|
monkeypatch.setattr(
|
|
VectorFeatureService,
|
|
"select_features_by_bbox",
|
|
lambda *_args, **_kwargs: {
|
|
"selection_bbox": {
|
|
"min_x": 4.9,
|
|
"min_y": 50.9,
|
|
"max_x": 5.2,
|
|
"max_y": 51.2,
|
|
"crs": "EPSG:4326",
|
|
},
|
|
"feature_count": 1,
|
|
"limit": 250,
|
|
"truncated": False,
|
|
"geojson": {
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"id": "source-feature",
|
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
|
"properties": {
|
|
"vector_feature_id": "source-feature",
|
|
"dataset_id": str(source.id),
|
|
},
|
|
}
|
|
],
|
|
},
|
|
},
|
|
)
|
|
|
|
def _persist_dataset_file(**kwargs):
|
|
output_path.write_bytes(kwargs["content"])
|
|
return {
|
|
"original_filename": kwargs["original_filename"],
|
|
"stored_filename": output_path.name,
|
|
"content_type": kwargs["content_type"],
|
|
"size_bytes": len(kwargs["content"]),
|
|
"checksum_sha256": _CHECKSUM,
|
|
"storage_path": str(output_path),
|
|
}
|
|
|
|
monkeypatch.setattr(StorageService, "persist_dataset_file", _persist_dataset_file)
|
|
|
|
def _quarantine(db, **kwargs):
|
|
calls.append(kwargs)
|
|
kwargs["dataset"].status = "quarantined"
|
|
return False
|
|
|
|
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
|
|
monkeypatch.setattr(
|
|
VectorFeatureService,
|
|
"persist_geojson_features",
|
|
lambda **kwargs: persisted_features.append(kwargs),
|
|
)
|
|
|
|
response = VectorOperationsService.derive_selection_dataset(
|
|
db=db,
|
|
dataset_id=source.id,
|
|
bbox={
|
|
"min_x": 4.9,
|
|
"min_y": 50.9,
|
|
"max_x": 5.2,
|
|
"max_y": 51.2,
|
|
"crs": "EPSG:4326",
|
|
},
|
|
)
|
|
|
|
assert response.status == "quarantined"
|
|
assert calls[0]["source_key"] == "map_selection"
|
|
assert calls[0]["parent_dataset"] is source
|
|
assert calls[0]["operation"] == "vector.selection"
|
|
assert persisted_features == []
|
|
|
|
|
|
def test_vector_buffer_uses_projected_metres_instead_of_wgs84_degrees(
|
|
monkeypatch, tmp_path
|
|
) -> None:
|
|
source = _governed_parent()
|
|
source.storage_path = str(tmp_path / "source.geojson")
|
|
source.crs = "EPSG:4326"
|
|
Path(source.storage_path).write_text(
|
|
json.dumps(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"crs": "EPSG:4326",
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
|
"properties": {},
|
|
},
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
# A governed consumption artifact must carry the checksum of these exact
|
|
# bytes; vector operations deliberately refuse a stale fixture checksum.
|
|
source.checksum_sha256 = sha256(Path(source.storage_path).read_bytes()).hexdigest()
|
|
source.source_snapshot.checksum_sha256 = source.checksum_sha256
|
|
db = _FakeSession({(Dataset, source.id): source})
|
|
captured = {}
|
|
|
|
def _persist(**kwargs):
|
|
captured.update(kwargs)
|
|
return uuid4()
|
|
|
|
monkeypatch.setattr(VectorOperationsService, "_persist_derived_dataset", _persist)
|
|
VectorOperationsService.buffer(
|
|
db, source.id, distance_m=100.0, dissolve=False, output_name=None
|
|
)
|
|
|
|
coordinates = captured["feature_collection"]["features"][0]["geometry"][
|
|
"coordinates"
|
|
][0]
|
|
longitudes = [coordinate[0] for coordinate in coordinates]
|
|
latitudes = [coordinate[1] for coordinate in coordinates]
|
|
assert max(longitudes) - min(longitudes) < 0.01
|
|
assert max(latitudes) - min(latitudes) < 0.01
|
|
|
|
|
|
def test_raster_operation_uses_derived_registry_before_commit(
|
|
monkeypatch, tmp_path
|
|
) -> None:
|
|
source = _governed_parent()
|
|
source.dataset_type = "raster"
|
|
source.storage_path = str(tmp_path / "source.tif")
|
|
output_path = tmp_path / "derived.tif"
|
|
output_path.write_bytes(b"derived-raster")
|
|
db = _FakeSession()
|
|
calls = []
|
|
|
|
def _govern(db, **kwargs):
|
|
calls.append(kwargs)
|
|
kwargs["dataset"].status = "quarantined"
|
|
return False
|
|
|
|
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_raster", _govern)
|
|
|
|
result = RasterOperationsService._persist_derived_dataset(
|
|
db,
|
|
source_dataset=source,
|
|
source_dataset_id=source.id,
|
|
operation="ndvi",
|
|
output_path=str(output_path),
|
|
output_name="derived.tif",
|
|
metadata={
|
|
"crs": "EPSG:31370",
|
|
"bounds": [100000.0, 100000.0, 100001.0, 100001.0],
|
|
"resolution": [1.0, 1.0],
|
|
"width": 1,
|
|
"height": 1,
|
|
"band_count": 1,
|
|
"dtype": ["float32"],
|
|
"operation_parameters": {"nir_band": 4, "red_band": 3},
|
|
},
|
|
)
|
|
|
|
derived = next(item for item in db.added if isinstance(item, Dataset))
|
|
assert result == derived.id
|
|
assert derived.status == "quarantined"
|
|
assert derived.source_name == "derived"
|
|
assert calls[0]["source_key"] == "derived"
|
|
assert calls[0]["parent_dataset"] is source
|
|
assert calls[0]["operation"] == "raster.ndvi"
|
|
|
|
|
|
def test_demo_fixture_creation_is_governed_and_does_not_persist_features_when_rejected(
|
|
monkeypatch, tmp_path
|
|
) -> None:
|
|
project_id = uuid4()
|
|
area_id = uuid4()
|
|
db = _FakeSession()
|
|
calls = []
|
|
features = []
|
|
|
|
monkeypatch.setattr(
|
|
StorageService,
|
|
"persist_dataset_file",
|
|
lambda **kwargs: {
|
|
"storage_path": str(tmp_path / kwargs["original_filename"]),
|
|
"original_filename": kwargs["original_filename"],
|
|
"stored_filename": kwargs["original_filename"],
|
|
"content_type": kwargs["content_type"],
|
|
"size_bytes": len(kwargs["content"]),
|
|
"checksum_sha256": _CHECKSUM,
|
|
},
|
|
)
|
|
|
|
def _quarantine(db, **kwargs):
|
|
calls.append(kwargs)
|
|
kwargs["dataset"].status = "quarantined"
|
|
return False
|
|
|
|
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
|
|
monkeypatch.setattr(
|
|
VectorFeatureService,
|
|
"persist_geojson_features",
|
|
lambda **kwargs: features.append(kwargs),
|
|
)
|
|
payload = {
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
|
"properties": {},
|
|
}
|
|
],
|
|
}
|
|
|
|
dataset = DemoWorkflowService._create_dataset(
|
|
db,
|
|
project_id=project_id,
|
|
area_id=area_id,
|
|
filename="fixture.geojson",
|
|
payload=payload,
|
|
raw=json.dumps(payload).encode("utf-8"),
|
|
role="source",
|
|
source_name="fixture",
|
|
reference_layer_name=None,
|
|
)
|
|
|
|
assert dataset.status == "quarantined"
|
|
assert calls[0]["source_key"] == "fixture"
|
|
assert calls[0]["operation"] == "demo.fixture_vector"
|
|
assert features == []
|