Enable governed GRB evolution
This commit is contained in:
@@ -27,6 +27,10 @@ from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
class TemporalAnalysisService:
|
||||
IDENTITY_COMPARISON_LIMIT = 5_000
|
||||
GOVERNED_GRB_IDENTITY_OPERATORS = {
|
||||
"provision_regional_grb_buildings.py",
|
||||
"provision_regional_grb_context.py",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def list_series(db: Session, project_id: UUID) -> list[TemporalSeriesRead]:
|
||||
@@ -345,7 +349,9 @@ class TemporalAnalysisService:
|
||||
) -> tuple[TemporalObjectChanges, dict[str, Any], list[str]]:
|
||||
earlier_config = earlier.source_metadata if isinstance(earlier.source_metadata, dict) else {}
|
||||
later_config = later.source_metadata if isinstance(later.source_metadata, dict) else {}
|
||||
if not earlier_config.get("identity_stable") or not later_config.get("identity_stable"):
|
||||
earlier_identity = TemporalAnalysisService._identity_contract(earlier)
|
||||
later_identity = TemporalAnalysisService._identity_contract(later)
|
||||
if earlier_identity is None or later_identity is None or earlier_identity != later_identity:
|
||||
return (
|
||||
TemporalObjectChanges(available=False),
|
||||
{"type": "FeatureCollection", "features": []},
|
||||
@@ -368,8 +374,7 @@ class TemporalAnalysisService:
|
||||
if not full_dataset_area:
|
||||
query = query.filter(ST_Intersects(VectorFeature.geometry, selection_shape))
|
||||
return (
|
||||
query.filter(VectorFeature.source_feature_id.isnot(None))
|
||||
.order_by(VectorFeature.source_feature_id.asc())
|
||||
query.order_by(VectorFeature.source_feature_id.asc())
|
||||
.limit(TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT + 1)
|
||||
.all()
|
||||
)
|
||||
@@ -386,6 +391,16 @@ class TemporalAnalysisService:
|
||||
["Object-level preview was skipped because the selection exceeds the 5,000 feature safety limit."],
|
||||
)
|
||||
|
||||
identity_prefixes = earlier_identity[1]
|
||||
if not TemporalAnalysisService._rows_match_identity_contract(earlier_rows, identity_prefixes) or not (
|
||||
TemporalAnalysisService._rows_match_identity_contract(later_rows, identity_prefixes)
|
||||
):
|
||||
return (
|
||||
TemporalObjectChanges(available=False),
|
||||
{"type": "FeatureCollection", "features": []},
|
||||
["De geselecteerde objecten bevatten geen volledig verifieerbare stabiele bronidentiteit."],
|
||||
)
|
||||
|
||||
earlier_by_id = {str(row.source_feature_id): row for row in earlier_rows if row.source_feature_id}
|
||||
later_by_id = {str(row.source_feature_id): row for row in later_rows if row.source_feature_id}
|
||||
earlier_ids = set(earlier_by_id)
|
||||
@@ -459,3 +474,50 @@ class TemporalAnalysisService:
|
||||
{"type": "FeatureCollection", "features": features},
|
||||
warnings,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _identity_contract(dataset: Dataset) -> tuple[str, tuple[str, ...]] | None:
|
||||
source_metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
|
||||
declared_stable = source_metadata.get("identity_stable")
|
||||
if declared_stable is False:
|
||||
return None
|
||||
|
||||
configured_prefixes = source_metadata.get("identity_prefixes")
|
||||
prefixes = tuple(
|
||||
sorted(
|
||||
{
|
||||
str(value).strip()
|
||||
for value in configured_prefixes
|
||||
if str(value).strip()
|
||||
}
|
||||
)
|
||||
) if isinstance(configured_prefixes, list) else ()
|
||||
if declared_stable is True:
|
||||
return str(source_metadata.get("identity_scheme") or "declared_source_feature_id"), prefixes
|
||||
|
||||
provenance = dataset.provenance_metadata if isinstance(dataset.provenance_metadata, dict) else {}
|
||||
if (
|
||||
dataset.source_name != "grb"
|
||||
or not str(dataset.temporal_series_key or "").startswith("grb:")
|
||||
or source_metadata.get("authority_level") != "authoritative"
|
||||
or source_metadata.get("geometry_clipped_to_area") is not True
|
||||
or provenance.get("operator_tool") not in TemporalAnalysisService.GOVERNED_GRB_IDENTITY_OPERATORS
|
||||
or provenance.get("reference_truncated") is not False
|
||||
):
|
||||
return None
|
||||
|
||||
if dataset.reference_layer_name == "buildings" and source_metadata.get("collection") == "GRB/GBG":
|
||||
prefixes = ("GBG.",)
|
||||
else:
|
||||
collections = source_metadata.get("collections")
|
||||
if not isinstance(collections, list) or not collections:
|
||||
return None
|
||||
prefixes = tuple(sorted(f"{str(collection)}:{str(collection)}." for collection in collections))
|
||||
return "grb_ogc_feature_id", prefixes
|
||||
|
||||
@staticmethod
|
||||
def _rows_match_identity_contract(rows: list[VectorFeature], prefixes: tuple[str, ...]) -> bool:
|
||||
identities = [str(row.source_feature_id or "").strip() for row in rows]
|
||||
if any(not identity for identity in identities) or len(set(identities)) != len(identities):
|
||||
return False
|
||||
return not prefixes or all(identity.startswith(prefixes) for identity in identities)
|
||||
|
||||
@@ -6,6 +6,8 @@ from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from geoalchemy2.shape import from_shape
|
||||
from shapely.geometry import Polygon
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, DatasetVersion
|
||||
@@ -46,6 +48,33 @@ class SequenceScalarSession:
|
||||
return ScalarQuery(next(self.values))
|
||||
|
||||
|
||||
class FeatureRowsQuery:
|
||||
def __init__(self, rows: list[object]):
|
||||
self.rows = rows
|
||||
self.row_limit: int | None = None
|
||||
|
||||
def filter(self, *args): # noqa: ANN002, ARG002
|
||||
return self
|
||||
|
||||
def order_by(self, *args): # noqa: ANN002, ARG002
|
||||
return self
|
||||
|
||||
def limit(self, value: int):
|
||||
self.row_limit = value
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return self.rows[: self.row_limit]
|
||||
|
||||
|
||||
class SequentialFeatureSession:
|
||||
def __init__(self, row_sets: list[list[object]]):
|
||||
self.row_sets = iter(row_sets)
|
||||
|
||||
def query(self, _model):
|
||||
return FeatureRowsQuery(next(self.row_sets))
|
||||
|
||||
|
||||
class VersionQuery:
|
||||
def __init__(self, latest: DatasetVersion | None):
|
||||
self.latest = latest
|
||||
@@ -104,6 +133,40 @@ def temporal_dataset(*, project_id, observed_year: int, metric_method: str = "fe
|
||||
)
|
||||
|
||||
|
||||
def governed_grb_dataset(*, project_id, observed_day: int) -> Dataset:
|
||||
dataset = temporal_dataset(project_id=project_id, observed_year=2026, metric_method="intersection_area")
|
||||
dataset.observed_at = datetime(2026, 7, observed_day, tzinfo=timezone.utc)
|
||||
dataset.source_version = f"2026-07-{observed_day:02d}"
|
||||
dataset.source_name = "grb"
|
||||
dataset.reference_layer_name = "buildings"
|
||||
dataset.temporal_series_key = "grb:buildings:kempen-transport-region"
|
||||
dataset.source_metadata = {
|
||||
"authority_level": "authoritative",
|
||||
"geometry_clipped_to_area": True,
|
||||
"collection": "GRB/GBG",
|
||||
"selection_aggregation": {
|
||||
"method": "intersection_area",
|
||||
"label": "Bebouwde grondoppervlakte",
|
||||
"unit": "ha",
|
||||
},
|
||||
}
|
||||
dataset.provenance_metadata = {
|
||||
"operator_tool": "provision_regional_grb_buildings.py",
|
||||
"reference_truncated": False,
|
||||
}
|
||||
return dataset
|
||||
|
||||
|
||||
def persisted_feature(dataset_id, source_feature_id: str | None, polygon: Polygon):
|
||||
return SimpleNamespace(
|
||||
id=uuid4(),
|
||||
dataset_id=dataset_id,
|
||||
source_feature_id=source_feature_id,
|
||||
properties_json={},
|
||||
geometry=from_shape(polygon, srid=4326),
|
||||
)
|
||||
|
||||
|
||||
def test_temporal_migration_and_models_align() -> None:
|
||||
migration = (ROOT / "backend/alembic/versions/202607140001_temporal_dataset_foundation.py").read_text(encoding="utf-8")
|
||||
for field in (
|
||||
@@ -294,6 +357,63 @@ def test_unstable_temporal_identity_returns_clear_end_user_warning() -> None:
|
||||
assert warnings == ["Wijzigingen van individuele objecten kunnen voor deze bron niet betrouwbaar worden gevolgd."]
|
||||
|
||||
|
||||
def test_governed_legacy_grb_snapshots_use_verified_official_feature_identity() -> None:
|
||||
project_id = uuid4()
|
||||
earlier = governed_grb_dataset(project_id=project_id, observed_day=14)
|
||||
later = governed_grb_dataset(project_id=project_id, observed_day=15)
|
||||
original = Polygon([(5.1, 51.1), (5.101, 51.1), (5.101, 51.101), (5.1, 51.101)])
|
||||
changed = Polygon([(5.1, 51.1), (5.102, 51.1), (5.102, 51.101), (5.1, 51.101)])
|
||||
added = Polygon([(5.11, 51.11), (5.111, 51.11), (5.111, 51.111), (5.11, 51.111)])
|
||||
session = SequentialFeatureSession(
|
||||
[
|
||||
[persisted_feature(earlier.id, "GBG.1", original)],
|
||||
[persisted_feature(later.id, "GBG.1", changed), persisted_feature(later.id, "GBG.2", added)],
|
||||
]
|
||||
)
|
||||
|
||||
changes, geojson, warnings = TemporalAnalysisService._compare_identity_features(
|
||||
session,
|
||||
earlier=earlier,
|
||||
later=later,
|
||||
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.2, "max_y": 51.2},
|
||||
preview_limit=100,
|
||||
)
|
||||
|
||||
assert changes.available is True
|
||||
assert changes.added_count == 1
|
||||
assert changes.removed_count == 0
|
||||
assert changes.modified_count == 1
|
||||
assert changes.unchanged_count == 0
|
||||
assert {feature["properties"]["change_type"] for feature in geojson["features"]} == {"added", "modified"}
|
||||
assert warnings == []
|
||||
|
||||
|
||||
def test_governed_grb_object_history_fails_closed_for_unverified_identity() -> None:
|
||||
project_id = uuid4()
|
||||
earlier = governed_grb_dataset(project_id=project_id, observed_day=14)
|
||||
later = governed_grb_dataset(project_id=project_id, observed_day=15)
|
||||
polygon = Polygon([(5.1, 51.1), (5.101, 51.1), (5.101, 51.101), (5.1, 51.101)])
|
||||
fallback_hash = "a" * 64
|
||||
session = SequentialFeatureSession(
|
||||
[
|
||||
[persisted_feature(earlier.id, fallback_hash, polygon)],
|
||||
[persisted_feature(later.id, fallback_hash, polygon)],
|
||||
]
|
||||
)
|
||||
|
||||
changes, geojson, warnings = TemporalAnalysisService._compare_identity_features(
|
||||
session,
|
||||
earlier=earlier,
|
||||
later=later,
|
||||
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.2, "max_y": 51.2},
|
||||
preview_limit=100,
|
||||
)
|
||||
|
||||
assert changes.available is False
|
||||
assert geojson["features"] == []
|
||||
assert warnings == ["De geselecteerde objecten bevatten geen volledig verifieerbare stabiele bronidentiteit."]
|
||||
|
||||
|
||||
def test_temporal_frontend_and_official_operator_contracts_exist() -> None:
|
||||
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||
temporal_api = (ROOT / "frontend/src/services/api/temporal.ts").read_text(encoding="utf-8")
|
||||
@@ -304,6 +424,8 @@ def test_temporal_frontend_and_official_operator_contracts_exist() -> None:
|
||||
assert "Laatste toestand" in workspace
|
||||
assert "Evolutie" in workspace
|
||||
assert "Vergelijk periode" in workspace
|
||||
assert "temporalRangeLabel" in workspace
|
||||
assert "Dagelijkse GRB-edities tonen wijzigingen in de officiële registratie" in workspace
|
||||
assert "/temporal/compare" in temporal_api
|
||||
assert "Statbel" in population and "area_weighted_sum" in population
|
||||
assert '"identity_stable": False' in population
|
||||
|
||||
@@ -119,6 +119,36 @@ def test_interior_buildings_skip_regional_owner_scan(monkeypatch) -> None:
|
||||
assert [item["id"] for item in features] == ["GBG.inside"]
|
||||
|
||||
|
||||
def test_building_partition_rejects_missing_official_source_identity() -> None:
|
||||
operator = load_operator()
|
||||
scopes = importlib.import_module("geographic_scopes")
|
||||
member = scopes.ScopeMember("Alpha", "10001")
|
||||
boundary = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)])
|
||||
scope = scopes.GeographicScope(
|
||||
key="single",
|
||||
display_name="Single",
|
||||
project_name="Single",
|
||||
project_region="Single",
|
||||
area_name="Single boundary",
|
||||
authority_name="Test",
|
||||
authority_url="https://example.test",
|
||||
scope_type="municipality",
|
||||
limitation_message="Test.",
|
||||
members=(member,),
|
||||
)
|
||||
missing_identity = feature("", Polygon([(0.1, 0.1), (0.2, 0.1), (0.2, 0.2), (0.1, 0.2)]))
|
||||
|
||||
with pytest.raises(RuntimeError, match="missing an official source identity"):
|
||||
operator.build_partition_features(
|
||||
[({"type": "FeatureCollection", "features": [missing_identity]}, "https://example.test/grb")],
|
||||
member=member,
|
||||
members=[(member, boundary)],
|
||||
regional_boundary=boundary,
|
||||
scope=scope,
|
||||
max_features=10,
|
||||
)
|
||||
|
||||
|
||||
def test_combined_artifact_streams_partitions_and_rejects_duplicate_source_ids(tmp_path: Path) -> None:
|
||||
operator = load_operator()
|
||||
scope = importlib.import_module("geographic_scopes").KEMPEN_TRANSPORT_REGION_SCOPE
|
||||
@@ -225,6 +255,8 @@ def test_regional_operator_is_packaged_documented_and_uses_service_boundaries()
|
||||
assert "provision_regional_grb_buildings.py" in dockerfile
|
||||
assert "py_compile scripts/provision_regional_grb_buildings.py" in readiness
|
||||
assert "DatasetService.import_partitioned_vector_artifact" in operator
|
||||
assert '"identity_stable": True' in operator
|
||||
assert '"identity_scheme": "grb_ogc_feature_id"' in operator
|
||||
assert "VectorFeatureService.persist_geojson_partitions" in dataset_service
|
||||
assert "insert into vector_features" not in operator.lower()
|
||||
assert "db.add(VectorFeature" not in operator
|
||||
|
||||
@@ -147,6 +147,38 @@ def test_mixed_water_partition_preserves_dimensions_and_source_identity() -> Non
|
||||
assert summary["reference_truncated"] is False
|
||||
|
||||
|
||||
def test_context_partition_rejects_missing_official_source_identity() -> None:
|
||||
operator = load_operator()
|
||||
scopes = __import__("geographic_scopes")
|
||||
member = scopes.ScopeMember("Alpha", "10001")
|
||||
scope = scopes.GeographicScope(
|
||||
key="test-region",
|
||||
display_name="Test region",
|
||||
project_name="Test project",
|
||||
project_region="Test",
|
||||
area_name="Test boundary",
|
||||
authority_name="Test",
|
||||
authority_url="https://example.test/scope",
|
||||
scope_type="policy_region",
|
||||
limitation_message="Test only.",
|
||||
members=(member,),
|
||||
)
|
||||
boundary = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)])
|
||||
definition = operator.LAYER_BY_KEY["roads"]
|
||||
missing_identity = source_feature("", LineString([(0.1, 0.1), (0.2, 0.2)]))
|
||||
|
||||
with pytest.raises(RuntimeError, match="missing an official source identity"):
|
||||
operator.build_partition_features(
|
||||
[(definition.collections[0], {"type": "FeatureCollection", "features": [missing_identity]}, "https://example.test/roads")],
|
||||
definition=definition,
|
||||
member=member,
|
||||
members=[(member, boundary)],
|
||||
regional_boundary=boundary,
|
||||
scope=scope,
|
||||
max_features=10,
|
||||
)
|
||||
|
||||
|
||||
def test_polygon_partition_assignment_does_not_duplicate_cross_boundary_parcel() -> None:
|
||||
operator = load_operator()
|
||||
scopes = __import__("geographic_scopes")
|
||||
@@ -224,5 +256,7 @@ def test_context_operator_is_packaged_and_uses_existing_service_boundary() -> No
|
||||
assert "provision_regional_grb_context.py" in dockerfile
|
||||
assert "py_compile scripts/provision_regional_grb_context.py" in readiness
|
||||
assert "DatasetService.import_partitioned_vector_artifact" in operator
|
||||
assert '"identity_stable": True' in operator
|
||||
assert '"identity_scheme": "grb_ogc_feature_id"' in operator
|
||||
assert "insert into vector_features" not in operator.lower()
|
||||
assert "db.add(VectorFeature" not in operator
|
||||
|
||||
Reference in New Issue
Block a user