feat(provenance): govern source snapshots and data inputs
This commit is contained in:
Jens
@@ -0,0 +1,765 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
from pyproj import Transformer
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import (
|
||||
Area,
|
||||
Dataset,
|
||||
DatasetQuarantine,
|
||||
DatasetVersion,
|
||||
Project,
|
||||
SourceRegistry,
|
||||
SourceSnapshot,
|
||||
VectorFeature,
|
||||
)
|
||||
from app.services.dataset_service import DatasetService, _PartitionedGeoJsonRecords
|
||||
from app.services.vector_operations_service import VectorOperationsService
|
||||
|
||||
|
||||
class _Query:
|
||||
def __init__(self, session: "_Session", model: type) -> None:
|
||||
self.session = session
|
||||
self.model = model
|
||||
self.predicates = []
|
||||
|
||||
def filter(self, *predicates):
|
||||
self.predicates.extend(predicates)
|
||||
return self
|
||||
|
||||
def one_or_none(self):
|
||||
matches = self._matches()
|
||||
if len(matches) > 1:
|
||||
raise AssertionError(
|
||||
f"expected one {self.model.__name__}, found {len(matches)}"
|
||||
)
|
||||
return matches[0] if matches else None
|
||||
|
||||
def all(self):
|
||||
return self._matches()
|
||||
|
||||
def _matches(self):
|
||||
matches = list(self.session.rows.get(self.model, []))
|
||||
for predicate in self.predicates:
|
||||
field_name = predicate.left.key
|
||||
expected = predicate.right.value
|
||||
operator_name = getattr(predicate.operator, "__name__", "")
|
||||
if operator_name == "in_op":
|
||||
matches = [
|
||||
item for item in matches if getattr(item, field_name) in expected
|
||||
]
|
||||
else:
|
||||
matches = [
|
||||
item for item in matches if getattr(item, field_name) == expected
|
||||
]
|
||||
return matches
|
||||
|
||||
|
||||
class _Session:
|
||||
"""Small ORM-shaped harness that exercises the real governed path."""
|
||||
|
||||
def __init__(self, project: Project) -> None:
|
||||
self.rows: dict[type, list[object]] = {Project: [project]}
|
||||
self.commits = 0
|
||||
self.rollbacks = 0
|
||||
self.flushes = 0
|
||||
|
||||
def get(self, model: type, item_id: UUID):
|
||||
return next(
|
||||
(
|
||||
item
|
||||
for item in self.rows.get(model, [])
|
||||
if getattr(item, "id", None) == item_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
def query(self, model: type) -> _Query:
|
||||
return _Query(self, model)
|
||||
|
||||
def add(self, item: object) -> None:
|
||||
if getattr(item, "id", None) is None:
|
||||
setattr(item, "id", uuid4())
|
||||
self.rows.setdefault(type(item), []).append(item)
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushes += 1
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def rollback(self) -> None:
|
||||
self.rollbacks += 1
|
||||
|
||||
def refresh(self, _item: object) -> None:
|
||||
return None
|
||||
|
||||
def expunge(self, _item: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _storage_info(tmp_path: Path, content: bytes) -> dict[str, object]:
|
||||
path = tmp_path / "grb-buildings.geojson"
|
||||
path.write_bytes(content)
|
||||
return {
|
||||
"storage_path": str(path),
|
||||
"original_filename": path.name,
|
||||
"stored_filename": path.name,
|
||||
"content_type": "application/geo+json",
|
||||
"size_bytes": len(content),
|
||||
"checksum_sha256": sha256(content).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def _valid_payload() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "gbg-1",
|
||||
"properties": {"id": "gbg-1"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _grb_payload_without_required_id() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"unrelated": "not a GRB identity"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _lambert_grb_payload() -> tuple[bytes, tuple[float, float, float, float]]:
|
||||
"""Create a valid GRB-shaped source artifact in its declared native CRS."""
|
||||
|
||||
longitude, latitude = 4.70, 51.10
|
||||
max_longitude, max_latitude = 4.7001, 51.1001
|
||||
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
lambert_ring = [
|
||||
to_lambert.transform(longitude, latitude),
|
||||
to_lambert.transform(max_longitude, latitude),
|
||||
to_lambert.transform(max_longitude, max_latitude),
|
||||
to_lambert.transform(longitude, latitude),
|
||||
]
|
||||
return (
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:31370"}},
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "GBG.lambert.1",
|
||||
"properties": {"id": "GBG.lambert.1"},
|
||||
"geometry": {"type": "Polygon", "coordinates": [lambert_ring]},
|
||||
}
|
||||
],
|
||||
}
|
||||
).encode("utf-8"),
|
||||
(longitude, latitude, max_longitude, max_latitude),
|
||||
)
|
||||
|
||||
|
||||
def test_governed_vector_import_persists_snapshot_contract_and_queryable_features(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Phase 2 governed ingest")
|
||||
db = _Session(project)
|
||||
raw = _valid_payload()
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||
)
|
||||
|
||||
result = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-buildings.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
"license": "Open data",
|
||||
"source_url": "https://example.invalid/grb",
|
||||
},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
snapshot = db.rows[SourceSnapshot][0]
|
||||
source = db.rows[SourceRegistry][0]
|
||||
assert result.status == "ready"
|
||||
assert dataset.source_name == "grb"
|
||||
assert dataset.source_registry_id == source.id
|
||||
assert dataset.source_snapshot_id == snapshot.id
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.provenance_status == "complete"
|
||||
assert dataset.lineage_status == "complete"
|
||||
assert dataset.quarantine_status == "not_quarantined"
|
||||
assert dataset.crs == "EPSG:4326"
|
||||
assert snapshot.checksum_sha256 == sha256(raw).hexdigest()
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
assert db.commits == 1
|
||||
|
||||
# A retry with identical governed evidence is idempotent and does not
|
||||
# create a second source snapshot, dataset or vector feature.
|
||||
repeated = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-buildings.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
assert repeated.id == result.id
|
||||
assert len(db.rows[Dataset]) == 1
|
||||
assert len(db.rows[SourceSnapshot]) == 1
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
|
||||
|
||||
def test_governed_lambert_geojson_persists_canonical_consumption_bytes_and_provenance_evidence(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Projected source bytes must never be the file that vector operations consume."""
|
||||
|
||||
project = Project(id=uuid4(), name="Canonical GeoJSON storage")
|
||||
db = _Session(project)
|
||||
raw, (longitude, latitude, max_longitude, max_latitude) = _lambert_grb_payload()
|
||||
consumption_path = tmp_path / "consumption" / "grb-buildings.geojson"
|
||||
provenance_path = tmp_path / "provenance" / "grb-buildings.geojson"
|
||||
|
||||
def _persist_dataset_file(**kwargs):
|
||||
stored = kwargs["content"]
|
||||
consumption_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
consumption_path.write_bytes(stored)
|
||||
return _storage_info(consumption_path.parent, stored)
|
||||
|
||||
def _persist_file(storage_path, content, original_filename, content_type):
|
||||
del storage_path, original_filename, content_type
|
||||
provenance_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
provenance_path.write_bytes(content)
|
||||
return _storage_info(provenance_path.parent, content)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
_persist_dataset_file,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_file",
|
||||
_persist_file,
|
||||
)
|
||||
|
||||
result = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-lambert.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:lambert:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01-lambert",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
dataset_version = db.rows[DatasetVersion][0]
|
||||
snapshot = db.rows[SourceSnapshot][0]
|
||||
canonical_bytes = Path(str(dataset.storage_path)).read_bytes()
|
||||
canonical_payload = json.loads(canonical_bytes)
|
||||
source_artifact = dataset.provenance_metadata["source_artifact"]
|
||||
|
||||
assert result.status == "ready"
|
||||
assert dataset.crs == "EPSG:4326"
|
||||
assert canonical_payload["crs"]["properties"]["name"] == "EPSG:4326"
|
||||
assert canonical_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
|
||||
[longitude, latitude], abs=0.000001
|
||||
)
|
||||
assert sha256(canonical_bytes).hexdigest() == dataset.checksum_sha256
|
||||
assert dataset_version.checksum_sha256 == dataset.checksum_sha256
|
||||
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||
assert source_artifact["retention"] == "provenance_evidence_only"
|
||||
assert source_artifact["checksum_sha256"] == sha256(raw).hexdigest()
|
||||
assert source_artifact["storage_path"] != dataset.storage_path
|
||||
assert Path(source_artifact["storage_path"]).read_bytes() == raw
|
||||
assert dataset.provenance_metadata["canonical_consumption_artifact"] == {
|
||||
"checksum_sha256": dataset.checksum_sha256,
|
||||
"crs": "EPSG:4326",
|
||||
"storage_role": "dataset_consumption",
|
||||
}
|
||||
|
||||
inspection = VectorOperationsService.inspect(db, dataset.id)
|
||||
assert inspection.crs == "EPSG:4326"
|
||||
assert inspection.bounds_json == {
|
||||
"min_x": pytest.approx(longitude, abs=0.000001),
|
||||
"min_y": pytest.approx(latitude, abs=0.000001),
|
||||
"max_x": pytest.approx(max_longitude, abs=0.000001),
|
||||
"max_y": pytest.approx(max_latitude, abs=0.000001),
|
||||
}
|
||||
response_payload = DatasetService.get_dataset_geojson(db, dataset.id)
|
||||
assert response_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
|
||||
[longitude, latitude], abs=0.000001
|
||||
)
|
||||
|
||||
# The storage identity is enforced at the operation boundary too; a
|
||||
# replacement with different canonical bytes is not silently processed.
|
||||
Path(str(dataset.storage_path)).write_bytes(canonical_bytes + b"\n")
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
VectorOperationsService.inspect(db, dataset.id)
|
||||
assert exc_info.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH"
|
||||
|
||||
|
||||
def test_metadata_refresh_refuses_mutated_governed_artifact(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""A passed snapshot cannot be silently re-described from mutable storage."""
|
||||
|
||||
project = Project(id=uuid4(), name="Phase 2 immutable refresh")
|
||||
db = _Session(project)
|
||||
raw = _valid_payload()
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||
)
|
||||
result = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-buildings.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
original_checksum = dataset.checksum_sha256
|
||||
original_metadata = dict(dataset.metadata_json or {})
|
||||
original_commit_count = db.commits
|
||||
|
||||
# Simulate an out-of-band storage replacement at the same path. The
|
||||
# refresh endpoint must not parse it into an already-passed contract row.
|
||||
Path(str(dataset.storage_path)).write_bytes(_grb_payload_without_required_id())
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetService.refresh_metadata(db, dataset.id)
|
||||
|
||||
assert exc_info.value.code == "GOVERNED_DATASET_REINGEST_REQUIRED"
|
||||
assert dataset.status == "ready"
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.checksum_sha256 == original_checksum
|
||||
assert dataset.metadata_json == original_metadata
|
||||
assert db.commits == original_commit_count
|
||||
|
||||
|
||||
def test_governed_import_quarantines_bad_artifacts_and_refuses_unknown_source(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Phase 2 quarantine")
|
||||
db = _Session(project)
|
||||
raw = b'{"type":"FeatureCollection","features":[]}'
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||
)
|
||||
|
||||
quarantined = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="empty.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={},
|
||||
temporal_series_key="grb:2026-08-empty",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01-empty",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
assert quarantined.status == "quarantined"
|
||||
assert quarantined.validation_status == "failed"
|
||||
assert quarantined.quarantine_status == "quarantined"
|
||||
assert len(db.rows[DatasetQuarantine]) == 1
|
||||
assert db.rows[SourceSnapshot][0].ingest_status == "quarantined"
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="unregistered.geojson",
|
||||
content=_valid_payload(),
|
||||
source="caller_controlled",
|
||||
source_name="caller_claimed_grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={},
|
||||
)
|
||||
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
|
||||
|
||||
|
||||
def test_governed_grb_vector_quarantines_missing_server_owned_required_attribute(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Phase 2 source schema")
|
||||
db = _Session(project)
|
||||
raw = _grb_payload_without_required_id()
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **kwargs: _storage_info(tmp_path, kwargs["content"]),
|
||||
)
|
||||
|
||||
quarantined = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-missing-id.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:missing-id",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01-missing-id",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == quarantined.id)
|
||||
assert quarantined.status == "quarantined"
|
||||
assert dataset.validation_status == "failed"
|
||||
assert dataset.quarantine_status == "quarantined"
|
||||
issue = dataset.validation_report_json["issues"][0]
|
||||
assert issue["code"] == "SOURCE_SCHEMA_REQUIRED_ATTRIBUTE_MISSING"
|
||||
assert issue["category"] == "source_schema"
|
||||
assert len(db.rows[DatasetQuarantine]) == 1
|
||||
|
||||
|
||||
def test_partitioned_vector_ingest_is_idempotent_and_quarantines_noncanonical_partition_coordinates(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Partitioned governed ingest")
|
||||
area = Area(id=uuid4(), project_id=project.id, name="Partitioned AOI")
|
||||
db = _Session(project)
|
||||
db.rows[Area] = [area]
|
||||
|
||||
feature = {
|
||||
"type": "Feature",
|
||||
"id": "GBG.1",
|
||||
"properties": {"id": "GBG.1", "source_feature_id": "GBG.1"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||
],
|
||||
},
|
||||
}
|
||||
partition_payload = {"type": "FeatureCollection", "features": [feature]}
|
||||
partition_path = tmp_path / "partition-01.geojson"
|
||||
partition_path.write_text(json.dumps(partition_payload), encoding="utf-8")
|
||||
artifact_payload = {
|
||||
"type": "FeatureCollection",
|
||||
"crs": "EPSG:4326",
|
||||
"features": [feature],
|
||||
}
|
||||
artifact_path = tmp_path / "grb-partitioned.geojson"
|
||||
artifact_raw = json.dumps(artifact_payload).encode("utf-8")
|
||||
artifact_path.write_bytes(artifact_raw)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||
lambda **_kwargs: _storage_info(tmp_path, artifact_raw),
|
||||
)
|
||||
|
||||
result = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=artifact_path,
|
||||
partition_paths=[partition_path],
|
||||
original_filename="grb-partitioned.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 4.69,
|
||||
"min_y": 51.09,
|
||||
"max_x": 4.70,
|
||||
"max_y": 51.10,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(artifact_raw).hexdigest(),
|
||||
"partition_checksums": {
|
||||
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||
},
|
||||
},
|
||||
temporal_series_key="grb:partitioned:test",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
assert result.status == "ready"
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.provenance_status == "complete"
|
||||
assert dataset.source_name == "grb"
|
||||
assert len(db.rows[SourceSnapshot]) == 1
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
assert dataset.metadata_json["partitioned_geometry_audit"][
|
||||
"partition_checksums_sha256"
|
||||
] == {partition_path.name: sha256(partition_path.read_bytes()).hexdigest()}
|
||||
assert dataset.provenance_metadata["partition_checksum_manifest_sha256"]
|
||||
assert dataset.provenance_metadata["partitioned_artifact_binding_sha256"]
|
||||
|
||||
repeated = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=artifact_path,
|
||||
partition_paths=[partition_path],
|
||||
original_filename="grb-partitioned.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 4.69,
|
||||
"min_y": 51.09,
|
||||
"max_x": 4.70,
|
||||
"max_y": 51.10,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(artifact_raw).hexdigest(),
|
||||
"partition_checksums": {
|
||||
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||
},
|
||||
},
|
||||
temporal_series_key="grb:partitioned:test",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
assert repeated.id == result.id
|
||||
assert len(db.rows[Dataset]) == 1
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
|
||||
lambert_feature = {
|
||||
**feature,
|
||||
"id": "GBG.lambert",
|
||||
"properties": {"id": "GBG.lambert"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[150000, 170000], [150010, 170000], [150010, 170010], [150000, 170000]]
|
||||
],
|
||||
},
|
||||
}
|
||||
lambert_partition = tmp_path / "partition-lambert.geojson"
|
||||
lambert_partition.write_text(
|
||||
json.dumps({"type": "FeatureCollection", "features": [lambert_feature]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
lambert_artifact = tmp_path / "grb-lambert.geojson"
|
||||
lambert_raw = json.dumps(
|
||||
{"type": "FeatureCollection", "features": [lambert_feature]}
|
||||
).encode("utf-8")
|
||||
lambert_artifact.write_bytes(lambert_raw)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||
lambda **_kwargs: _storage_info(tmp_path, lambert_raw),
|
||||
)
|
||||
|
||||
quarantined = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=lambert_artifact,
|
||||
partition_paths=[lambert_partition],
|
||||
original_filename="grb-lambert.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 150000,
|
||||
"min_y": 170000,
|
||||
"max_x": 150010,
|
||||
"max_y": 170010,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(lambert_raw).hexdigest(),
|
||||
"partition_checksums": {
|
||||
lambert_partition.name: sha256(
|
||||
lambert_partition.read_bytes()
|
||||
).hexdigest()
|
||||
},
|
||||
},
|
||||
temporal_series_key="grb:partitioned:lambert",
|
||||
observed_at=datetime(2026, 8, 2, tzinfo=UTC),
|
||||
source_version="2026-08-02",
|
||||
)
|
||||
assert quarantined.status == "quarantined"
|
||||
assert quarantined.validation_status == "failed"
|
||||
assert quarantined.quarantine_status == "quarantined"
|
||||
|
||||
missing_manifest_partition = tmp_path / "partition-missing-manifest.geojson"
|
||||
missing_manifest_partition.write_text(
|
||||
json.dumps(partition_payload), encoding="utf-8"
|
||||
)
|
||||
missing_manifest_artifact = tmp_path / "grb-missing-manifest.geojson"
|
||||
missing_manifest_raw = json.dumps(
|
||||
{"type": "FeatureCollection", "features": [feature]}
|
||||
).encode("utf-8")
|
||||
missing_manifest_artifact.write_bytes(missing_manifest_raw)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||
lambda **_kwargs: _storage_info(tmp_path, missing_manifest_raw),
|
||||
)
|
||||
|
||||
missing_manifest = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=missing_manifest_artifact,
|
||||
partition_paths=[missing_manifest_partition],
|
||||
original_filename="grb-missing-manifest.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 4.69,
|
||||
"min_y": 51.09,
|
||||
"max_x": 4.70,
|
||||
"max_y": 51.10,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(missing_manifest_raw).hexdigest()
|
||||
},
|
||||
temporal_series_key="grb:partitioned:missing-manifest",
|
||||
observed_at=datetime(2026, 8, 3, tzinfo=UTC),
|
||||
source_version="2026-08-03",
|
||||
)
|
||||
assert missing_manifest.status == "quarantined"
|
||||
assert (
|
||||
missing_manifest.validation_report_json["issues"][0]["code"]
|
||||
== "PARTITION_CHECKSUM_MANIFEST_REQUIRED"
|
||||
)
|
||||
|
||||
|
||||
def test_partitioned_geometry_audit_handles_more_than_generic_topology_limit_without_materializing_geometries(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
feature_count = 10_001
|
||||
partition_path = tmp_path / "large-partition.geojson"
|
||||
partition_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": f"GBG.{index}",
|
||||
"properties": {"id": f"GBG.{index}"},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [4.0 + index / 10_000_000, 51.0],
|
||||
},
|
||||
}
|
||||
for index in range(feature_count)
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
audit = _PartitionedGeoJsonRecords(
|
||||
[partition_path],
|
||||
expected_feature_count=feature_count,
|
||||
declared_partition_checksums={
|
||||
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||
},
|
||||
).audit()
|
||||
|
||||
assert audit.feature_count == feature_count
|
||||
assert audit.bounds_json["min_x"] == 4.0
|
||||
assert audit.bounds_json["max_x"] > audit.bounds_json["min_x"]
|
||||
assert audit.representative_record.geometry.geom_type == "MultiPoint"
|
||||
Reference in New Issue
Block a user