Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
766 lines
27 KiB
Python
766 lines
27 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from hashlib import sha256
|
|
import json
|
|
from pathlib import Path
|
|
from uuid import UUID, uuid4
|
|
|
|
import pytest
|
|
from pyproj import Transformer
|
|
|
|
from app.core.errors import AppError
|
|
from app.models import (
|
|
Area,
|
|
Dataset,
|
|
DatasetQuarantine,
|
|
DatasetVersion,
|
|
Project,
|
|
SourceRegistry,
|
|
SourceSnapshot,
|
|
VectorFeature,
|
|
)
|
|
from app.services.dataset_service import DatasetService, _PartitionedGeoJsonRecords
|
|
from app.services.vector_operations_service import VectorOperationsService
|
|
|
|
|
|
class _Query:
|
|
def __init__(self, session: "_Session", model: type) -> None:
|
|
self.session = session
|
|
self.model = model
|
|
self.predicates = []
|
|
|
|
def filter(self, *predicates):
|
|
self.predicates.extend(predicates)
|
|
return self
|
|
|
|
def one_or_none(self):
|
|
matches = self._matches()
|
|
if len(matches) > 1:
|
|
raise AssertionError(
|
|
f"expected one {self.model.__name__}, found {len(matches)}"
|
|
)
|
|
return matches[0] if matches else None
|
|
|
|
def all(self):
|
|
return self._matches()
|
|
|
|
def _matches(self):
|
|
matches = list(self.session.rows.get(self.model, []))
|
|
for predicate in self.predicates:
|
|
field_name = predicate.left.key
|
|
expected = predicate.right.value
|
|
operator_name = getattr(predicate.operator, "__name__", "")
|
|
if operator_name == "in_op":
|
|
matches = [
|
|
item for item in matches if getattr(item, field_name) in expected
|
|
]
|
|
else:
|
|
matches = [
|
|
item for item in matches if getattr(item, field_name) == expected
|
|
]
|
|
return matches
|
|
|
|
|
|
class _Session:
|
|
"""Small ORM-shaped harness that exercises the real governed path."""
|
|
|
|
def __init__(self, project: Project) -> None:
|
|
self.rows: dict[type, list[object]] = {Project: [project]}
|
|
self.commits = 0
|
|
self.rollbacks = 0
|
|
self.flushes = 0
|
|
|
|
def get(self, model: type, item_id: UUID):
|
|
return next(
|
|
(
|
|
item
|
|
for item in self.rows.get(model, [])
|
|
if getattr(item, "id", None) == item_id
|
|
),
|
|
None,
|
|
)
|
|
|
|
def query(self, model: type) -> _Query:
|
|
return _Query(self, model)
|
|
|
|
def add(self, item: object) -> None:
|
|
if getattr(item, "id", None) is None:
|
|
setattr(item, "id", uuid4())
|
|
self.rows.setdefault(type(item), []).append(item)
|
|
|
|
def flush(self) -> None:
|
|
self.flushes += 1
|
|
|
|
def commit(self) -> None:
|
|
self.commits += 1
|
|
|
|
def rollback(self) -> None:
|
|
self.rollbacks += 1
|
|
|
|
def refresh(self, _item: object) -> None:
|
|
return None
|
|
|
|
def expunge(self, _item: object) -> None:
|
|
return None
|
|
|
|
|
|
def _storage_info(tmp_path: Path, content: bytes) -> dict[str, object]:
|
|
path = tmp_path / "grb-buildings.geojson"
|
|
path.write_bytes(content)
|
|
return {
|
|
"storage_path": str(path),
|
|
"original_filename": path.name,
|
|
"stored_filename": path.name,
|
|
"content_type": "application/geo+json",
|
|
"size_bytes": len(content),
|
|
"checksum_sha256": sha256(content).hexdigest(),
|
|
}
|
|
|
|
|
|
def _valid_payload() -> bytes:
|
|
return json.dumps(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"id": "gbg-1",
|
|
"properties": {"id": "gbg-1"},
|
|
"geometry": {
|
|
"type": "Polygon",
|
|
"coordinates": [
|
|
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
|
],
|
|
},
|
|
}
|
|
],
|
|
}
|
|
).encode("utf-8")
|
|
|
|
|
|
def _grb_payload_without_required_id() -> bytes:
|
|
return json.dumps(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"properties": {"unrelated": "not a GRB identity"},
|
|
"geometry": {
|
|
"type": "Polygon",
|
|
"coordinates": [
|
|
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
|
],
|
|
},
|
|
}
|
|
],
|
|
}
|
|
).encode("utf-8")
|
|
|
|
|
|
def _lambert_grb_payload() -> tuple[bytes, tuple[float, float, float, float]]:
|
|
"""Create a valid GRB-shaped source artifact in its declared native CRS."""
|
|
|
|
longitude, latitude = 4.70, 51.10
|
|
max_longitude, max_latitude = 4.7001, 51.1001
|
|
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
|
lambert_ring = [
|
|
to_lambert.transform(longitude, latitude),
|
|
to_lambert.transform(max_longitude, latitude),
|
|
to_lambert.transform(max_longitude, max_latitude),
|
|
to_lambert.transform(longitude, latitude),
|
|
]
|
|
return (
|
|
json.dumps(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"crs": {"type": "name", "properties": {"name": "EPSG:31370"}},
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"id": "GBG.lambert.1",
|
|
"properties": {"id": "GBG.lambert.1"},
|
|
"geometry": {"type": "Polygon", "coordinates": [lambert_ring]},
|
|
}
|
|
],
|
|
}
|
|
).encode("utf-8"),
|
|
(longitude, latitude, max_longitude, max_latitude),
|
|
)
|
|
|
|
|
|
def test_governed_vector_import_persists_snapshot_contract_and_queryable_features(
|
|
monkeypatch, tmp_path: Path
|
|
) -> None:
|
|
project = Project(id=uuid4(), name="Phase 2 governed ingest")
|
|
db = _Session(project)
|
|
raw = _valid_payload()
|
|
monkeypatch.setattr(
|
|
"app.services.dataset_service.StorageService.persist_dataset_file",
|
|
lambda **_kwargs: _storage_info(tmp_path, raw),
|
|
)
|
|
|
|
result = DatasetService.import_vector_bytes(
|
|
db,
|
|
project_id=project.id,
|
|
filename="grb-buildings.geojson",
|
|
content=raw,
|
|
source="grb_wfs",
|
|
source_name="grb",
|
|
dataset_role="reference",
|
|
reference_layer_name="buildings",
|
|
source_metadata={
|
|
"license": "Open data",
|
|
"source_url": "https://example.invalid/grb",
|
|
},
|
|
provenance_metadata={"adapter": "test"},
|
|
temporal_series_key="grb:2026-08",
|
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
|
source_version="2026-08-01",
|
|
temporal_granularity="snapshot",
|
|
)
|
|
|
|
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
|
snapshot = db.rows[SourceSnapshot][0]
|
|
source = db.rows[SourceRegistry][0]
|
|
assert result.status == "ready"
|
|
assert dataset.source_name == "grb"
|
|
assert dataset.source_registry_id == source.id
|
|
assert dataset.source_snapshot_id == snapshot.id
|
|
assert dataset.validation_status == "passed"
|
|
assert dataset.provenance_status == "complete"
|
|
assert dataset.lineage_status == "complete"
|
|
assert dataset.quarantine_status == "not_quarantined"
|
|
assert dataset.crs == "EPSG:4326"
|
|
assert snapshot.checksum_sha256 == sha256(raw).hexdigest()
|
|
assert len(db.rows[VectorFeature]) == 1
|
|
assert db.commits == 1
|
|
|
|
# A retry with identical governed evidence is idempotent and does not
|
|
# create a second source snapshot, dataset or vector feature.
|
|
repeated = DatasetService.import_vector_bytes(
|
|
db,
|
|
project_id=project.id,
|
|
filename="grb-buildings.geojson",
|
|
content=raw,
|
|
source="grb_wfs",
|
|
source_name="grb",
|
|
dataset_role="reference",
|
|
reference_layer_name="buildings",
|
|
source_metadata={"license": "Open data"},
|
|
provenance_metadata={"adapter": "test"},
|
|
temporal_series_key="grb:2026-08",
|
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
|
source_version="2026-08-01",
|
|
temporal_granularity="snapshot",
|
|
)
|
|
assert repeated.id == result.id
|
|
assert len(db.rows[Dataset]) == 1
|
|
assert len(db.rows[SourceSnapshot]) == 1
|
|
assert len(db.rows[VectorFeature]) == 1
|
|
|
|
|
|
def test_governed_lambert_geojson_persists_canonical_consumption_bytes_and_provenance_evidence(
|
|
monkeypatch, tmp_path: Path
|
|
) -> None:
|
|
"""Projected source bytes must never be the file that vector operations consume."""
|
|
|
|
project = Project(id=uuid4(), name="Canonical GeoJSON storage")
|
|
db = _Session(project)
|
|
raw, (longitude, latitude, max_longitude, max_latitude) = _lambert_grb_payload()
|
|
consumption_path = tmp_path / "consumption" / "grb-buildings.geojson"
|
|
provenance_path = tmp_path / "provenance" / "grb-buildings.geojson"
|
|
|
|
def _persist_dataset_file(**kwargs):
|
|
stored = kwargs["content"]
|
|
consumption_path.parent.mkdir(parents=True, exist_ok=True)
|
|
consumption_path.write_bytes(stored)
|
|
return _storage_info(consumption_path.parent, stored)
|
|
|
|
def _persist_file(storage_path, content, original_filename, content_type):
|
|
del storage_path, original_filename, content_type
|
|
provenance_path.parent.mkdir(parents=True, exist_ok=True)
|
|
provenance_path.write_bytes(content)
|
|
return _storage_info(provenance_path.parent, content)
|
|
|
|
monkeypatch.setattr(
|
|
"app.services.dataset_service.StorageService.persist_dataset_file",
|
|
_persist_dataset_file,
|
|
)
|
|
monkeypatch.setattr(
|
|
"app.services.dataset_service.StorageService.persist_file",
|
|
_persist_file,
|
|
)
|
|
|
|
result = DatasetService.import_vector_bytes(
|
|
db,
|
|
project_id=project.id,
|
|
filename="grb-lambert.geojson",
|
|
content=raw,
|
|
source="grb_wfs",
|
|
source_name="grb",
|
|
dataset_role="reference",
|
|
reference_layer_name="buildings",
|
|
source_metadata={"license": "Open data"},
|
|
provenance_metadata={"adapter": "test"},
|
|
temporal_series_key="grb:lambert:2026-08",
|
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
|
source_version="2026-08-01-lambert",
|
|
temporal_granularity="snapshot",
|
|
)
|
|
|
|
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
|
dataset_version = db.rows[DatasetVersion][0]
|
|
snapshot = db.rows[SourceSnapshot][0]
|
|
canonical_bytes = Path(str(dataset.storage_path)).read_bytes()
|
|
canonical_payload = json.loads(canonical_bytes)
|
|
source_artifact = dataset.provenance_metadata["source_artifact"]
|
|
|
|
assert result.status == "ready"
|
|
assert dataset.crs == "EPSG:4326"
|
|
assert canonical_payload["crs"]["properties"]["name"] == "EPSG:4326"
|
|
assert canonical_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
|
|
[longitude, latitude], abs=0.000001
|
|
)
|
|
assert sha256(canonical_bytes).hexdigest() == dataset.checksum_sha256
|
|
assert dataset_version.checksum_sha256 == dataset.checksum_sha256
|
|
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
|
assert source_artifact["retention"] == "provenance_evidence_only"
|
|
assert source_artifact["checksum_sha256"] == sha256(raw).hexdigest()
|
|
assert source_artifact["storage_path"] != dataset.storage_path
|
|
assert Path(source_artifact["storage_path"]).read_bytes() == raw
|
|
assert dataset.provenance_metadata["canonical_consumption_artifact"] == {
|
|
"checksum_sha256": dataset.checksum_sha256,
|
|
"crs": "EPSG:4326",
|
|
"storage_role": "dataset_consumption",
|
|
}
|
|
|
|
inspection = VectorOperationsService.inspect(db, dataset.id)
|
|
assert inspection.crs == "EPSG:4326"
|
|
assert inspection.bounds_json == {
|
|
"min_x": pytest.approx(longitude, abs=0.000001),
|
|
"min_y": pytest.approx(latitude, abs=0.000001),
|
|
"max_x": pytest.approx(max_longitude, abs=0.000001),
|
|
"max_y": pytest.approx(max_latitude, abs=0.000001),
|
|
}
|
|
response_payload = DatasetService.get_dataset_geojson(db, dataset.id)
|
|
assert response_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
|
|
[longitude, latitude], abs=0.000001
|
|
)
|
|
|
|
# The storage identity is enforced at the operation boundary too; a
|
|
# replacement with different canonical bytes is not silently processed.
|
|
Path(str(dataset.storage_path)).write_bytes(canonical_bytes + b"\n")
|
|
with pytest.raises(AppError) as exc_info:
|
|
VectorOperationsService.inspect(db, dataset.id)
|
|
assert exc_info.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH"
|
|
|
|
|
|
def test_metadata_refresh_refuses_mutated_governed_artifact(
|
|
monkeypatch, tmp_path: Path
|
|
) -> None:
|
|
"""A passed snapshot cannot be silently re-described from mutable storage."""
|
|
|
|
project = Project(id=uuid4(), name="Phase 2 immutable refresh")
|
|
db = _Session(project)
|
|
raw = _valid_payload()
|
|
monkeypatch.setattr(
|
|
"app.services.dataset_service.StorageService.persist_dataset_file",
|
|
lambda **_kwargs: _storage_info(tmp_path, raw),
|
|
)
|
|
result = DatasetService.import_vector_bytes(
|
|
db,
|
|
project_id=project.id,
|
|
filename="grb-buildings.geojson",
|
|
content=raw,
|
|
source="grb_wfs",
|
|
source_name="grb",
|
|
dataset_role="reference",
|
|
reference_layer_name="buildings",
|
|
source_metadata={"license": "Open data"},
|
|
provenance_metadata={"adapter": "test"},
|
|
temporal_series_key="grb:2026-08",
|
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
|
source_version="2026-08-01",
|
|
temporal_granularity="snapshot",
|
|
)
|
|
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
|
original_checksum = dataset.checksum_sha256
|
|
original_metadata = dict(dataset.metadata_json or {})
|
|
original_commit_count = db.commits
|
|
|
|
# Simulate an out-of-band storage replacement at the same path. The
|
|
# refresh endpoint must not parse it into an already-passed contract row.
|
|
Path(str(dataset.storage_path)).write_bytes(_grb_payload_without_required_id())
|
|
with pytest.raises(AppError) as exc_info:
|
|
DatasetService.refresh_metadata(db, dataset.id)
|
|
|
|
assert exc_info.value.code == "GOVERNED_DATASET_REINGEST_REQUIRED"
|
|
assert dataset.status == "ready"
|
|
assert dataset.validation_status == "passed"
|
|
assert dataset.checksum_sha256 == original_checksum
|
|
assert dataset.metadata_json == original_metadata
|
|
assert db.commits == original_commit_count
|
|
|
|
|
|
def test_governed_import_quarantines_bad_artifacts_and_refuses_unknown_source(
|
|
monkeypatch, tmp_path: Path
|
|
) -> None:
|
|
project = Project(id=uuid4(), name="Phase 2 quarantine")
|
|
db = _Session(project)
|
|
raw = b'{"type":"FeatureCollection","features":[]}'
|
|
monkeypatch.setattr(
|
|
"app.services.dataset_service.StorageService.persist_dataset_file",
|
|
lambda **_kwargs: _storage_info(tmp_path, raw),
|
|
)
|
|
|
|
quarantined = DatasetService.import_vector_bytes(
|
|
db,
|
|
project_id=project.id,
|
|
filename="empty.geojson",
|
|
content=raw,
|
|
source="grb_wfs",
|
|
source_name="grb",
|
|
dataset_role="reference",
|
|
reference_layer_name="buildings",
|
|
source_metadata={"license": "Open data"},
|
|
provenance_metadata={},
|
|
temporal_series_key="grb:2026-08-empty",
|
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
|
source_version="2026-08-01-empty",
|
|
temporal_granularity="snapshot",
|
|
)
|
|
assert quarantined.status == "quarantined"
|
|
assert quarantined.validation_status == "failed"
|
|
assert quarantined.quarantine_status == "quarantined"
|
|
assert len(db.rows[DatasetQuarantine]) == 1
|
|
assert db.rows[SourceSnapshot][0].ingest_status == "quarantined"
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
DatasetService.import_vector_bytes(
|
|
db,
|
|
project_id=project.id,
|
|
filename="unregistered.geojson",
|
|
content=_valid_payload(),
|
|
source="caller_controlled",
|
|
source_name="caller_claimed_grb",
|
|
dataset_role="reference",
|
|
reference_layer_name="buildings",
|
|
source_metadata={"license": "Open data"},
|
|
provenance_metadata={},
|
|
)
|
|
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
|
|
|
|
|
|
def test_governed_grb_vector_quarantines_missing_server_owned_required_attribute(
|
|
monkeypatch, tmp_path: Path
|
|
) -> None:
|
|
project = Project(id=uuid4(), name="Phase 2 source schema")
|
|
db = _Session(project)
|
|
raw = _grb_payload_without_required_id()
|
|
monkeypatch.setattr(
|
|
"app.services.dataset_service.StorageService.persist_dataset_file",
|
|
lambda **kwargs: _storage_info(tmp_path, kwargs["content"]),
|
|
)
|
|
|
|
quarantined = DatasetService.import_vector_bytes(
|
|
db,
|
|
project_id=project.id,
|
|
filename="grb-missing-id.geojson",
|
|
content=raw,
|
|
source="grb_wfs",
|
|
source_name="grb",
|
|
dataset_role="reference",
|
|
reference_layer_name="buildings",
|
|
source_metadata={"license": "Open data"},
|
|
provenance_metadata={"adapter": "test"},
|
|
temporal_series_key="grb:missing-id",
|
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
|
source_version="2026-08-01-missing-id",
|
|
temporal_granularity="snapshot",
|
|
)
|
|
|
|
dataset = next(item for item in db.rows[Dataset] if item.id == quarantined.id)
|
|
assert quarantined.status == "quarantined"
|
|
assert dataset.validation_status == "failed"
|
|
assert dataset.quarantine_status == "quarantined"
|
|
issue = dataset.validation_report_json["issues"][0]
|
|
assert issue["code"] == "SOURCE_SCHEMA_REQUIRED_ATTRIBUTE_MISSING"
|
|
assert issue["category"] == "source_schema"
|
|
assert len(db.rows[DatasetQuarantine]) == 1
|
|
|
|
|
|
def test_partitioned_vector_ingest_is_idempotent_and_quarantines_noncanonical_partition_coordinates(
|
|
monkeypatch,
|
|
tmp_path: Path,
|
|
) -> None:
|
|
project = Project(id=uuid4(), name="Partitioned governed ingest")
|
|
area = Area(id=uuid4(), project_id=project.id, name="Partitioned AOI")
|
|
db = _Session(project)
|
|
db.rows[Area] = [area]
|
|
|
|
feature = {
|
|
"type": "Feature",
|
|
"id": "GBG.1",
|
|
"properties": {"id": "GBG.1", "source_feature_id": "GBG.1"},
|
|
"geometry": {
|
|
"type": "Polygon",
|
|
"coordinates": [
|
|
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
|
],
|
|
},
|
|
}
|
|
partition_payload = {"type": "FeatureCollection", "features": [feature]}
|
|
partition_path = tmp_path / "partition-01.geojson"
|
|
partition_path.write_text(json.dumps(partition_payload), encoding="utf-8")
|
|
artifact_payload = {
|
|
"type": "FeatureCollection",
|
|
"crs": "EPSG:4326",
|
|
"features": [feature],
|
|
}
|
|
artifact_path = tmp_path / "grb-partitioned.geojson"
|
|
artifact_raw = json.dumps(artifact_payload).encode("utf-8")
|
|
artifact_path.write_bytes(artifact_raw)
|
|
monkeypatch.setattr(
|
|
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
|
lambda **_kwargs: _storage_info(tmp_path, artifact_raw),
|
|
)
|
|
|
|
result = DatasetService.import_partitioned_vector_artifact(
|
|
db,
|
|
project_id=project.id,
|
|
area_id=area.id,
|
|
artifact_path=artifact_path,
|
|
partition_paths=[partition_path],
|
|
original_filename="grb-partitioned.geojson",
|
|
source="operator_official_import",
|
|
dataset_role="reference",
|
|
source_name="grb",
|
|
reference_layer_name="buildings",
|
|
metadata_json={
|
|
"feature_count": 1,
|
|
"crs": "EPSG:4326",
|
|
"bounds_json": {
|
|
"min_x": 4.69,
|
|
"min_y": 51.09,
|
|
"max_x": 4.70,
|
|
"max_y": 51.10,
|
|
},
|
|
},
|
|
source_metadata={"license": "Open data"},
|
|
provenance_metadata={
|
|
"artifact_sha256": sha256(artifact_raw).hexdigest(),
|
|
"partition_checksums": {
|
|
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
|
},
|
|
},
|
|
temporal_series_key="grb:partitioned:test",
|
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
|
source_version="2026-08-01",
|
|
)
|
|
|
|
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
|
assert result.status == "ready"
|
|
assert dataset.validation_status == "passed"
|
|
assert dataset.provenance_status == "complete"
|
|
assert dataset.source_name == "grb"
|
|
assert len(db.rows[SourceSnapshot]) == 1
|
|
assert len(db.rows[VectorFeature]) == 1
|
|
assert dataset.metadata_json["partitioned_geometry_audit"][
|
|
"partition_checksums_sha256"
|
|
] == {partition_path.name: sha256(partition_path.read_bytes()).hexdigest()}
|
|
assert dataset.provenance_metadata["partition_checksum_manifest_sha256"]
|
|
assert dataset.provenance_metadata["partitioned_artifact_binding_sha256"]
|
|
|
|
repeated = DatasetService.import_partitioned_vector_artifact(
|
|
db,
|
|
project_id=project.id,
|
|
area_id=area.id,
|
|
artifact_path=artifact_path,
|
|
partition_paths=[partition_path],
|
|
original_filename="grb-partitioned.geojson",
|
|
source="operator_official_import",
|
|
dataset_role="reference",
|
|
source_name="grb",
|
|
reference_layer_name="buildings",
|
|
metadata_json={
|
|
"feature_count": 1,
|
|
"crs": "EPSG:4326",
|
|
"bounds_json": {
|
|
"min_x": 4.69,
|
|
"min_y": 51.09,
|
|
"max_x": 4.70,
|
|
"max_y": 51.10,
|
|
},
|
|
},
|
|
source_metadata={"license": "Open data"},
|
|
provenance_metadata={
|
|
"artifact_sha256": sha256(artifact_raw).hexdigest(),
|
|
"partition_checksums": {
|
|
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
|
},
|
|
},
|
|
temporal_series_key="grb:partitioned:test",
|
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
|
source_version="2026-08-01",
|
|
)
|
|
assert repeated.id == result.id
|
|
assert len(db.rows[Dataset]) == 1
|
|
assert len(db.rows[VectorFeature]) == 1
|
|
|
|
lambert_feature = {
|
|
**feature,
|
|
"id": "GBG.lambert",
|
|
"properties": {"id": "GBG.lambert"},
|
|
"geometry": {
|
|
"type": "Polygon",
|
|
"coordinates": [
|
|
[[150000, 170000], [150010, 170000], [150010, 170010], [150000, 170000]]
|
|
],
|
|
},
|
|
}
|
|
lambert_partition = tmp_path / "partition-lambert.geojson"
|
|
lambert_partition.write_text(
|
|
json.dumps({"type": "FeatureCollection", "features": [lambert_feature]}),
|
|
encoding="utf-8",
|
|
)
|
|
lambert_artifact = tmp_path / "grb-lambert.geojson"
|
|
lambert_raw = json.dumps(
|
|
{"type": "FeatureCollection", "features": [lambert_feature]}
|
|
).encode("utf-8")
|
|
lambert_artifact.write_bytes(lambert_raw)
|
|
monkeypatch.setattr(
|
|
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
|
lambda **_kwargs: _storage_info(tmp_path, lambert_raw),
|
|
)
|
|
|
|
quarantined = DatasetService.import_partitioned_vector_artifact(
|
|
db,
|
|
project_id=project.id,
|
|
area_id=area.id,
|
|
artifact_path=lambert_artifact,
|
|
partition_paths=[lambert_partition],
|
|
original_filename="grb-lambert.geojson",
|
|
source="operator_official_import",
|
|
dataset_role="reference",
|
|
source_name="grb",
|
|
reference_layer_name="buildings",
|
|
metadata_json={
|
|
"feature_count": 1,
|
|
"crs": "EPSG:4326",
|
|
"bounds_json": {
|
|
"min_x": 150000,
|
|
"min_y": 170000,
|
|
"max_x": 150010,
|
|
"max_y": 170010,
|
|
},
|
|
},
|
|
source_metadata={"license": "Open data"},
|
|
provenance_metadata={
|
|
"artifact_sha256": sha256(lambert_raw).hexdigest(),
|
|
"partition_checksums": {
|
|
lambert_partition.name: sha256(
|
|
lambert_partition.read_bytes()
|
|
).hexdigest()
|
|
},
|
|
},
|
|
temporal_series_key="grb:partitioned:lambert",
|
|
observed_at=datetime(2026, 8, 2, tzinfo=UTC),
|
|
source_version="2026-08-02",
|
|
)
|
|
assert quarantined.status == "quarantined"
|
|
assert quarantined.validation_status == "failed"
|
|
assert quarantined.quarantine_status == "quarantined"
|
|
|
|
missing_manifest_partition = tmp_path / "partition-missing-manifest.geojson"
|
|
missing_manifest_partition.write_text(
|
|
json.dumps(partition_payload), encoding="utf-8"
|
|
)
|
|
missing_manifest_artifact = tmp_path / "grb-missing-manifest.geojson"
|
|
missing_manifest_raw = json.dumps(
|
|
{"type": "FeatureCollection", "features": [feature]}
|
|
).encode("utf-8")
|
|
missing_manifest_artifact.write_bytes(missing_manifest_raw)
|
|
monkeypatch.setattr(
|
|
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
|
lambda **_kwargs: _storage_info(tmp_path, missing_manifest_raw),
|
|
)
|
|
|
|
missing_manifest = DatasetService.import_partitioned_vector_artifact(
|
|
db,
|
|
project_id=project.id,
|
|
area_id=area.id,
|
|
artifact_path=missing_manifest_artifact,
|
|
partition_paths=[missing_manifest_partition],
|
|
original_filename="grb-missing-manifest.geojson",
|
|
source="operator_official_import",
|
|
dataset_role="reference",
|
|
source_name="grb",
|
|
reference_layer_name="buildings",
|
|
metadata_json={
|
|
"feature_count": 1,
|
|
"crs": "EPSG:4326",
|
|
"bounds_json": {
|
|
"min_x": 4.69,
|
|
"min_y": 51.09,
|
|
"max_x": 4.70,
|
|
"max_y": 51.10,
|
|
},
|
|
},
|
|
source_metadata={"license": "Open data"},
|
|
provenance_metadata={
|
|
"artifact_sha256": sha256(missing_manifest_raw).hexdigest()
|
|
},
|
|
temporal_series_key="grb:partitioned:missing-manifest",
|
|
observed_at=datetime(2026, 8, 3, tzinfo=UTC),
|
|
source_version="2026-08-03",
|
|
)
|
|
assert missing_manifest.status == "quarantined"
|
|
assert (
|
|
missing_manifest.validation_report_json["issues"][0]["code"]
|
|
== "PARTITION_CHECKSUM_MANIFEST_REQUIRED"
|
|
)
|
|
|
|
|
|
def test_partitioned_geometry_audit_handles_more_than_generic_topology_limit_without_materializing_geometries(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
feature_count = 10_001
|
|
partition_path = tmp_path / "large-partition.geojson"
|
|
partition_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"features": [
|
|
{
|
|
"type": "Feature",
|
|
"id": f"GBG.{index}",
|
|
"properties": {"id": f"GBG.{index}"},
|
|
"geometry": {
|
|
"type": "Point",
|
|
"coordinates": [4.0 + index / 10_000_000, 51.0],
|
|
},
|
|
}
|
|
for index in range(feature_count)
|
|
],
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
audit = _PartitionedGeoJsonRecords(
|
|
[partition_path],
|
|
expected_feature_count=feature_count,
|
|
declared_partition_checksums={
|
|
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
|
},
|
|
).audit()
|
|
|
|
assert audit.feature_count == feature_count
|
|
assert audit.bounds_json["min_x"] == 4.0
|
|
assert audit.bounds_json["max_x"] > audit.bounds_json["min_x"]
|
|
assert audit.representative_record.geometry.geom_type == "MultiPoint"
|