Enable governed GRB evolution
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-16 19:30:46 +02:00
parent 1d515e4901
commit a8b18942f5
12 changed files with 448 additions and 59 deletions
+15
View File
@@ -7,6 +7,21 @@
# Changelog # Changelog
## Sprint 224 Governed GRB evolution (2026-07-16)
- Enabled fail-closed object evolution between the retained regional GRB
snapshots by validating official OGC feature identities, approved operator
provenance, completeness and collection-specific prefixes.
- Removed geometry-hash identity fallbacks from future regional GRB imports;
a provider feature without an official identity now aborts acquisition.
- Added explicit identity contracts to future buildings, roads, water and
parcel snapshots while retaining compatibility with the two already
persisted governed editions.
- Improved daily temporal labels and added a clear distinction between an
official registration change and the unknown date of a physical change.
- Documented a source-wide refresh-readiness matrix without enabling browser
fetches, schedulers, migrations or automatic Dataset replacement.
## Sprint 223 Governed regional GRB refresh (2026-07-16) ## Sprint 223 Governed regional GRB refresh (2026-07-16)
- Added a canonical, read-only regional GRB refresh plan for buildings, roads, - Added a canonical, read-only regional GRB refresh plan for buildings, roads,
@@ -27,6 +27,10 @@ from app.services.vector_feature_service import VectorFeatureService
class TemporalAnalysisService: class TemporalAnalysisService:
IDENTITY_COMPARISON_LIMIT = 5_000 IDENTITY_COMPARISON_LIMIT = 5_000
GOVERNED_GRB_IDENTITY_OPERATORS = {
"provision_regional_grb_buildings.py",
"provision_regional_grb_context.py",
}
@staticmethod @staticmethod
def list_series(db: Session, project_id: UUID) -> list[TemporalSeriesRead]: def list_series(db: Session, project_id: UUID) -> list[TemporalSeriesRead]:
@@ -345,7 +349,9 @@ class TemporalAnalysisService:
) -> tuple[TemporalObjectChanges, dict[str, Any], list[str]]: ) -> tuple[TemporalObjectChanges, dict[str, Any], list[str]]:
earlier_config = earlier.source_metadata if isinstance(earlier.source_metadata, dict) else {} earlier_config = earlier.source_metadata if isinstance(earlier.source_metadata, dict) else {}
later_config = later.source_metadata if isinstance(later.source_metadata, dict) else {} later_config = later.source_metadata if isinstance(later.source_metadata, dict) else {}
if not earlier_config.get("identity_stable") or not later_config.get("identity_stable"): earlier_identity = TemporalAnalysisService._identity_contract(earlier)
later_identity = TemporalAnalysisService._identity_contract(later)
if earlier_identity is None or later_identity is None or earlier_identity != later_identity:
return ( return (
TemporalObjectChanges(available=False), TemporalObjectChanges(available=False),
{"type": "FeatureCollection", "features": []}, {"type": "FeatureCollection", "features": []},
@@ -368,8 +374,7 @@ class TemporalAnalysisService:
if not full_dataset_area: if not full_dataset_area:
query = query.filter(ST_Intersects(VectorFeature.geometry, selection_shape)) query = query.filter(ST_Intersects(VectorFeature.geometry, selection_shape))
return ( return (
query.filter(VectorFeature.source_feature_id.isnot(None)) query.order_by(VectorFeature.source_feature_id.asc())
.order_by(VectorFeature.source_feature_id.asc())
.limit(TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT + 1) .limit(TemporalAnalysisService.IDENTITY_COMPARISON_LIMIT + 1)
.all() .all()
) )
@@ -386,6 +391,16 @@ class TemporalAnalysisService:
["Object-level preview was skipped because the selection exceeds the 5,000 feature safety limit."], ["Object-level preview was skipped because the selection exceeds the 5,000 feature safety limit."],
) )
identity_prefixes = earlier_identity[1]
if not TemporalAnalysisService._rows_match_identity_contract(earlier_rows, identity_prefixes) or not (
TemporalAnalysisService._rows_match_identity_contract(later_rows, identity_prefixes)
):
return (
TemporalObjectChanges(available=False),
{"type": "FeatureCollection", "features": []},
["De geselecteerde objecten bevatten geen volledig verifieerbare stabiele bronidentiteit."],
)
earlier_by_id = {str(row.source_feature_id): row for row in earlier_rows if row.source_feature_id} earlier_by_id = {str(row.source_feature_id): row for row in earlier_rows if row.source_feature_id}
later_by_id = {str(row.source_feature_id): row for row in later_rows if row.source_feature_id} later_by_id = {str(row.source_feature_id): row for row in later_rows if row.source_feature_id}
earlier_ids = set(earlier_by_id) earlier_ids = set(earlier_by_id)
@@ -459,3 +474,50 @@ class TemporalAnalysisService:
{"type": "FeatureCollection", "features": features}, {"type": "FeatureCollection", "features": features},
warnings, warnings,
) )
@staticmethod
def _identity_contract(dataset: Dataset) -> tuple[str, tuple[str, ...]] | None:
source_metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {}
declared_stable = source_metadata.get("identity_stable")
if declared_stable is False:
return None
configured_prefixes = source_metadata.get("identity_prefixes")
prefixes = tuple(
sorted(
{
str(value).strip()
for value in configured_prefixes
if str(value).strip()
}
)
) if isinstance(configured_prefixes, list) else ()
if declared_stable is True:
return str(source_metadata.get("identity_scheme") or "declared_source_feature_id"), prefixes
provenance = dataset.provenance_metadata if isinstance(dataset.provenance_metadata, dict) else {}
if (
dataset.source_name != "grb"
or not str(dataset.temporal_series_key or "").startswith("grb:")
or source_metadata.get("authority_level") != "authoritative"
or source_metadata.get("geometry_clipped_to_area") is not True
or provenance.get("operator_tool") not in TemporalAnalysisService.GOVERNED_GRB_IDENTITY_OPERATORS
or provenance.get("reference_truncated") is not False
):
return None
if dataset.reference_layer_name == "buildings" and source_metadata.get("collection") == "GRB/GBG":
prefixes = ("GBG.",)
else:
collections = source_metadata.get("collections")
if not isinstance(collections, list) or not collections:
return None
prefixes = tuple(sorted(f"{str(collection)}:{str(collection)}." for collection in collections))
return "grb_ogc_feature_id", prefixes
@staticmethod
def _rows_match_identity_contract(rows: list[VectorFeature], prefixes: tuple[str, ...]) -> bool:
identities = [str(row.source_feature_id or "").strip() for row in rows]
if any(not identity for identity in identities) or len(set(identities)) != len(identities):
return False
return not prefixes or all(identity.startswith(prefixes) for identity in identities)
@@ -6,6 +6,8 @@ from types import SimpleNamespace
from uuid import uuid4 from uuid import uuid4
import pytest import pytest
from geoalchemy2.shape import from_shape
from shapely.geometry import Polygon
from app.core.errors import AppError from app.core.errors import AppError
from app.models import Dataset, DatasetVersion from app.models import Dataset, DatasetVersion
@@ -46,6 +48,33 @@ class SequenceScalarSession:
return ScalarQuery(next(self.values)) return ScalarQuery(next(self.values))
class FeatureRowsQuery:
def __init__(self, rows: list[object]):
self.rows = rows
self.row_limit: int | None = None
def filter(self, *args): # noqa: ANN002, ARG002
return self
def order_by(self, *args): # noqa: ANN002, ARG002
return self
def limit(self, value: int):
self.row_limit = value
return self
def all(self):
return self.rows[: self.row_limit]
class SequentialFeatureSession:
def __init__(self, row_sets: list[list[object]]):
self.row_sets = iter(row_sets)
def query(self, _model):
return FeatureRowsQuery(next(self.row_sets))
class VersionQuery: class VersionQuery:
def __init__(self, latest: DatasetVersion | None): def __init__(self, latest: DatasetVersion | None):
self.latest = latest self.latest = latest
@@ -104,6 +133,40 @@ def temporal_dataset(*, project_id, observed_year: int, metric_method: str = "fe
) )
def governed_grb_dataset(*, project_id, observed_day: int) -> Dataset:
dataset = temporal_dataset(project_id=project_id, observed_year=2026, metric_method="intersection_area")
dataset.observed_at = datetime(2026, 7, observed_day, tzinfo=timezone.utc)
dataset.source_version = f"2026-07-{observed_day:02d}"
dataset.source_name = "grb"
dataset.reference_layer_name = "buildings"
dataset.temporal_series_key = "grb:buildings:kempen-transport-region"
dataset.source_metadata = {
"authority_level": "authoritative",
"geometry_clipped_to_area": True,
"collection": "GRB/GBG",
"selection_aggregation": {
"method": "intersection_area",
"label": "Bebouwde grondoppervlakte",
"unit": "ha",
},
}
dataset.provenance_metadata = {
"operator_tool": "provision_regional_grb_buildings.py",
"reference_truncated": False,
}
return dataset
def persisted_feature(dataset_id, source_feature_id: str | None, polygon: Polygon):
return SimpleNamespace(
id=uuid4(),
dataset_id=dataset_id,
source_feature_id=source_feature_id,
properties_json={},
geometry=from_shape(polygon, srid=4326),
)
def test_temporal_migration_and_models_align() -> None: def test_temporal_migration_and_models_align() -> None:
migration = (ROOT / "backend/alembic/versions/202607140001_temporal_dataset_foundation.py").read_text(encoding="utf-8") migration = (ROOT / "backend/alembic/versions/202607140001_temporal_dataset_foundation.py").read_text(encoding="utf-8")
for field in ( for field in (
@@ -294,6 +357,63 @@ def test_unstable_temporal_identity_returns_clear_end_user_warning() -> None:
assert warnings == ["Wijzigingen van individuele objecten kunnen voor deze bron niet betrouwbaar worden gevolgd."] assert warnings == ["Wijzigingen van individuele objecten kunnen voor deze bron niet betrouwbaar worden gevolgd."]
def test_governed_legacy_grb_snapshots_use_verified_official_feature_identity() -> None:
project_id = uuid4()
earlier = governed_grb_dataset(project_id=project_id, observed_day=14)
later = governed_grb_dataset(project_id=project_id, observed_day=15)
original = Polygon([(5.1, 51.1), (5.101, 51.1), (5.101, 51.101), (5.1, 51.101)])
changed = Polygon([(5.1, 51.1), (5.102, 51.1), (5.102, 51.101), (5.1, 51.101)])
added = Polygon([(5.11, 51.11), (5.111, 51.11), (5.111, 51.111), (5.11, 51.111)])
session = SequentialFeatureSession(
[
[persisted_feature(earlier.id, "GBG.1", original)],
[persisted_feature(later.id, "GBG.1", changed), persisted_feature(later.id, "GBG.2", added)],
]
)
changes, geojson, warnings = TemporalAnalysisService._compare_identity_features(
session,
earlier=earlier,
later=later,
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.2, "max_y": 51.2},
preview_limit=100,
)
assert changes.available is True
assert changes.added_count == 1
assert changes.removed_count == 0
assert changes.modified_count == 1
assert changes.unchanged_count == 0
assert {feature["properties"]["change_type"] for feature in geojson["features"]} == {"added", "modified"}
assert warnings == []
def test_governed_grb_object_history_fails_closed_for_unverified_identity() -> None:
project_id = uuid4()
earlier = governed_grb_dataset(project_id=project_id, observed_day=14)
later = governed_grb_dataset(project_id=project_id, observed_day=15)
polygon = Polygon([(5.1, 51.1), (5.101, 51.1), (5.101, 51.101), (5.1, 51.101)])
fallback_hash = "a" * 64
session = SequentialFeatureSession(
[
[persisted_feature(earlier.id, fallback_hash, polygon)],
[persisted_feature(later.id, fallback_hash, polygon)],
]
)
changes, geojson, warnings = TemporalAnalysisService._compare_identity_features(
session,
earlier=earlier,
later=later,
bbox={"min_x": 5.0, "min_y": 51.0, "max_x": 5.2, "max_y": 51.2},
preview_limit=100,
)
assert changes.available is False
assert geojson["features"] == []
assert warnings == ["De geselecteerde objecten bevatten geen volledig verifieerbare stabiele bronidentiteit."]
def test_temporal_frontend_and_official_operator_contracts_exist() -> None: def test_temporal_frontend_and_official_operator_contracts_exist() -> None:
workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8") workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8")
temporal_api = (ROOT / "frontend/src/services/api/temporal.ts").read_text(encoding="utf-8") temporal_api = (ROOT / "frontend/src/services/api/temporal.ts").read_text(encoding="utf-8")
@@ -304,6 +424,8 @@ def test_temporal_frontend_and_official_operator_contracts_exist() -> None:
assert "Laatste toestand" in workspace assert "Laatste toestand" in workspace
assert "Evolutie" in workspace assert "Evolutie" in workspace
assert "Vergelijk periode" in workspace assert "Vergelijk periode" in workspace
assert "temporalRangeLabel" in workspace
assert "Dagelijkse GRB-edities tonen wijzigingen in de officiële registratie" in workspace
assert "/temporal/compare" in temporal_api assert "/temporal/compare" in temporal_api
assert "Statbel" in population and "area_weighted_sum" in population assert "Statbel" in population and "area_weighted_sum" in population
assert '"identity_stable": False' in population assert '"identity_stable": False' in population
@@ -119,6 +119,36 @@ def test_interior_buildings_skip_regional_owner_scan(monkeypatch) -> None:
assert [item["id"] for item in features] == ["GBG.inside"] assert [item["id"] for item in features] == ["GBG.inside"]
def test_building_partition_rejects_missing_official_source_identity() -> None:
operator = load_operator()
scopes = importlib.import_module("geographic_scopes")
member = scopes.ScopeMember("Alpha", "10001")
boundary = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)])
scope = scopes.GeographicScope(
key="single",
display_name="Single",
project_name="Single",
project_region="Single",
area_name="Single boundary",
authority_name="Test",
authority_url="https://example.test",
scope_type="municipality",
limitation_message="Test.",
members=(member,),
)
missing_identity = feature("", Polygon([(0.1, 0.1), (0.2, 0.1), (0.2, 0.2), (0.1, 0.2)]))
with pytest.raises(RuntimeError, match="missing an official source identity"):
operator.build_partition_features(
[({"type": "FeatureCollection", "features": [missing_identity]}, "https://example.test/grb")],
member=member,
members=[(member, boundary)],
regional_boundary=boundary,
scope=scope,
max_features=10,
)
def test_combined_artifact_streams_partitions_and_rejects_duplicate_source_ids(tmp_path: Path) -> None: def test_combined_artifact_streams_partitions_and_rejects_duplicate_source_ids(tmp_path: Path) -> None:
operator = load_operator() operator = load_operator()
scope = importlib.import_module("geographic_scopes").KEMPEN_TRANSPORT_REGION_SCOPE scope = importlib.import_module("geographic_scopes").KEMPEN_TRANSPORT_REGION_SCOPE
@@ -225,6 +255,8 @@ def test_regional_operator_is_packaged_documented_and_uses_service_boundaries()
assert "provision_regional_grb_buildings.py" in dockerfile assert "provision_regional_grb_buildings.py" in dockerfile
assert "py_compile scripts/provision_regional_grb_buildings.py" in readiness assert "py_compile scripts/provision_regional_grb_buildings.py" in readiness
assert "DatasetService.import_partitioned_vector_artifact" in operator assert "DatasetService.import_partitioned_vector_artifact" in operator
assert '"identity_stable": True' in operator
assert '"identity_scheme": "grb_ogc_feature_id"' in operator
assert "VectorFeatureService.persist_geojson_partitions" in dataset_service assert "VectorFeatureService.persist_geojson_partitions" in dataset_service
assert "insert into vector_features" not in operator.lower() assert "insert into vector_features" not in operator.lower()
assert "db.add(VectorFeature" not in operator assert "db.add(VectorFeature" not in operator
@@ -147,6 +147,38 @@ def test_mixed_water_partition_preserves_dimensions_and_source_identity() -> Non
assert summary["reference_truncated"] is False assert summary["reference_truncated"] is False
def test_context_partition_rejects_missing_official_source_identity() -> None:
operator = load_operator()
scopes = __import__("geographic_scopes")
member = scopes.ScopeMember("Alpha", "10001")
scope = scopes.GeographicScope(
key="test-region",
display_name="Test region",
project_name="Test project",
project_region="Test",
area_name="Test boundary",
authority_name="Test",
authority_url="https://example.test/scope",
scope_type="policy_region",
limitation_message="Test only.",
members=(member,),
)
boundary = Polygon([(0, 0), (1, 0), (1, 1), (0, 1)])
definition = operator.LAYER_BY_KEY["roads"]
missing_identity = source_feature("", LineString([(0.1, 0.1), (0.2, 0.2)]))
with pytest.raises(RuntimeError, match="missing an official source identity"):
operator.build_partition_features(
[(definition.collections[0], {"type": "FeatureCollection", "features": [missing_identity]}, "https://example.test/roads")],
definition=definition,
member=member,
members=[(member, boundary)],
regional_boundary=boundary,
scope=scope,
max_features=10,
)
def test_polygon_partition_assignment_does_not_duplicate_cross_boundary_parcel() -> None: def test_polygon_partition_assignment_does_not_duplicate_cross_boundary_parcel() -> None:
operator = load_operator() operator = load_operator()
scopes = __import__("geographic_scopes") scopes = __import__("geographic_scopes")
@@ -224,5 +256,7 @@ def test_context_operator_is_packaged_and_uses_existing_service_boundary() -> No
assert "provision_regional_grb_context.py" in dockerfile assert "provision_regional_grb_context.py" in dockerfile
assert "py_compile scripts/provision_regional_grb_context.py" in readiness assert "py_compile scripts/provision_regional_grb_context.py" in readiness
assert "DatasetService.import_partitioned_vector_artifact" in operator assert "DatasetService.import_partitioned_vector_artifact" in operator
assert '"identity_stable": True' in operator
assert '"identity_scheme": "grb_ogc_feature_id"' in operator
assert "insert into vector_features" not in operator.lower() assert "insert into vector_features" not in operator.lower()
assert "db.add(VectorFeature" not in operator assert "db.add(VectorFeature" not in operator
+11
View File
@@ -1850,6 +1850,17 @@ Added/removed/modified object changes are calculated only when source
provenance declares stable feature identities; otherwise provenance declares stable feature identities; otherwise
`object_changes.available=false` and no object history is inferred. `object_changes.available=false` and no object history is inferred.
Governed regional GRB snapshots use the official OGC feature identifier as
their object identity. New imports declare the identity scheme and accepted
collection prefixes explicitly. Existing operator-managed GRB snapshots are
accepted only when their authoritative, complete and area-clipped provenance
matches an approved regional operator and every selected identifier is
present, unique and uses the expected prefix. Missing, duplicate or unexpected
identifiers fail closed to metric-only comparison. The existing 5,000-feature
selection limit also remains in force. Differences between daily GRB editions
describe changes in the official registration; they do not prove that a
physical change happened on the exact publication date.
For `reference_layer_name=agriculture`, the primary map metric is exact For `reference_layer_name=agriculture`, the primary map metric is exact
intersected declared-use area in hectares. Supplemental metrics use intersected declared-use area in hectares. Supplemental metrics use
server-owned filters on the normalized official main-crop group. Annual ALZ server-owned filters on the normalized official main-crop group. Annual ALZ
+32
View File
@@ -9387,3 +9387,35 @@ Validation so far:
Next: Next:
- Run the full release gate, deploy to Tower, execute a live read-only plan and - Run the full release gate, deploy to Tower, execute a live read-only plan and
then stage/apply only the exact confirmed official edition. then stage/apply only the exact confirmed official edition.
## Sprint 224 - Governed GRB evolution (2026-07-16)
Implemented:
- Audited the two retained regional editions for buildings, roads, water and
parcels. All 2,108,425 persisted VectorFeatures have an official
`source_feature_id`; none uses the former 64-character geometry-hash
fallback.
- Added a fail-closed GRB identity contract to temporal object comparison.
Legacy snapshots require authoritative, area-clipped, non-truncated
provenance from an approved regional operator and every selected identifier
must be present, unique and match its collection prefix.
- Made future regional GRB operators abort when an official OGC feature ID is
missing and persist their identity scheme/prefixes explicitly.
- Replaced ambiguous same-year labels such as `2026-2026` with daily edition
ranges and explained that registration changes do not date physical change.
- Documented an honest refresh-readiness matrix for all major source families;
no additional provider fetch or automatic refresh was enabled.
Boundaries:
- No migration, new persistence model, scheduler, browser-triggered import or
parallel temporal engine was added.
- Object comparison remains bounded to 5,000 selected features and falls back
to metric-only output whenever source identity cannot be proven.
Validation so far:
- Focused GRB operator and temporal tests pass: 39 tests.
- Frontend typecheck and production build pass.
Next:
- Run the full release gate, deploy, verify a real bounded GRB edition delta
through the canonical API and inspect the daily Evolution UI on Tower.
+32
View File
@@ -151,6 +151,38 @@ Collectiecatalogus en contract:
- https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections?f=text%2Fhtml - https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections?f=text%2Fhtml
- https://geo.api.vlaanderen.be/GRB/ogc/features/v1/openapi?f=text%2Fhtml - https://geo.api.vlaanderen.be/GRB/ogc/features/v1/openapi?f=text%2Fhtml
### Beheerde GRB-edities en objectevolutie
Elke regionale GRB-import blijft een onveranderlijke Dataset in dezelfde
`temporal_series_key`. De operators bewaren uitsluitend de officiële OGC
feature-ID als `source_feature_id`; een ontbrekende ID wordt niet vervangen
door een geometriehash. Nieuwe snapshots declareren
`identity_scheme=grb_ogc_feature_id` en de toegelaten collectieprefixen.
De tijdelijke vergelijkingsservice toont toegevoegde, verwijderde en gewijzigde
objecten alleen wanneer beide snapshots hetzelfde gecontroleerde
identiteitscontract hebben en alle geselecteerde ID's aanwezig en uniek zijn.
Oudere, reeds beheerde snapshots worden alleen onder hun volledige
authoritatieve operatorprovenance aanvaard. Dagelijkse editieverschillen zijn
wijzigingen in de officiële registratie; de fysieke wijziging kan eerder zijn
gebeurd.
### Refreshgereedheid van officiële bronnen
| bronfamilie | huidige behandeling | volgende veilige stap |
| --- | --- | --- |
| GRB gebouwen/wegen/water/percelen | operationele, expliciete plan-stage-apply refresh met onveranderlijke snapshots | alleen een nieuw officieel gedateerd cataloguseditie na operatorbevestiging ophalen |
| Statbel bevolking | jaarlijkse, expliciete edities in één tijdreeks | een nieuwe publicatie alleen na schema-, sectorgeometrie- en totalencontrole toevoegen |
| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; metricvergelijking zonder objectlineage | eerst een stabiele machineleesbare editiebron verifiëren, daarna dezelfde begrensde operatorflow toepassen |
| orthofoto | vaste lokale opname per expliciete analysezone; catalogusprobe is alleen een signaal | vluchtjaar, productvariant en dekking vergelijken voordat nieuwe pixels worden opgehaald |
| landgebruik, thematische rasters, DHMV en VMM-scenario's | vaste product-/scenario-edities, geen rolling snapshot | alleen een nieuwe gedocumenteerde producteditie als afzonderlijke Dataset verwerven |
| bodemkaart en historische kaarten | historische referentie-editie | niet als verouderde actuele bron labelen; alleen vervangen bij een officiële inhoudelijke heruitgave |
| BWK/Natura 2000 en gebouwen-/adressenregister | expliciete actuele snapshot met eigen methodologische betekenis | eerst een stabiele officiële editieprobe en bron-specifieke reconciliatiecontrole toevoegen |
Geen van deze regels activeert een browserfetch, scheduler of automatische
vervanging. Een bron wordt pas `refreshable` wanneer versie, dekking, schema,
provenance en een begrensde acquisitieroute afzonderlijk verifieerbaar zijn.
### Mol population history ### Mol population history
`scripts/provision_mol_population_history.py` imports official Statbel `scripts/provision_mol_population_history.py` imports official Statbel
+17
View File
@@ -670,3 +670,20 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Replace the water-first recommendation with a cross-domain implementation wave. - [x] Replace the water-first recommendation with a cross-domain implementation wave.
- [ ] Implement the governed Flemish thematic-raster registry for space occupation, open space, population density, node value and service level. - [ ] Implement the governed Flemish thematic-raster registry for space occupation, open space, population density, node value and service level.
- [x] Add the digital soil map through DatasetService and VectorFeatureService. - [x] Add the digital soil map through DatasetService and VectorFeatureService.
# Sprint 224 - Governed GRB evolution
- [x] Audit all eight retained regional GRB snapshots for complete official
source identities and geometry-hash fallbacks.
- [x] Validate old governed snapshots through authoritative, complete and
approved operator provenance before enabling object-level comparison.
- [x] Make future regional GRB imports reject missing official OGC identities.
- [x] Persist explicit identity schemes and collection prefixes on new GRB
snapshots.
- [x] Show precise daily edition ranges and explain registration-date nuance in
the Evolution workspace.
- [x] Document refresh readiness across the complete official-source portfolio.
- [ ] Add the next governed catalog probe only after an official stable version
contract is verified; ALZ annual releases are the first candidate.
- [ ] Keep orthophoto refresh manual until product variant, flight year and
complete selected-area coverage can be compared deterministically.
+74 -47
View File
@@ -322,10 +322,8 @@ function temporalSeriesLabel(items: DatasetCreateResponse[]): string {
} }
const first = items[0] const first = items[0]
const source = first ? getDatasetDisplayName(first) : 'Tijdreeks' const source = first ? getDatasetDisplayName(first) : 'Tijdreeks'
const firstYear = first?.observed_at ? new Date(first.observed_at).getUTCFullYear() : null const range = temporalRangeLabel(items)
const last = items[items.length - 1] return range ? `${source} (${range})` : source
const lastYear = last?.observed_at ? new Date(last.observed_at).getUTCFullYear() : null
return firstYear && lastYear ? `${source} (${firstYear}-${lastYear})` : source
} }
function listThemeTemporalSeries(datasets: DatasetCreateResponse[], theme: DataTheme): TemporalSeriesGroup[] { function listThemeTemporalSeries(datasets: DatasetCreateResponse[], theme: DataTheme): TemporalSeriesGroup[] {
@@ -362,6 +360,27 @@ function formatObservationDate(value: string | null | undefined): string {
return new Intl.DateTimeFormat('nl-BE', { year: 'numeric', month: 'short', day: 'numeric' }).format(new Date(value)) return new Intl.DateTimeFormat('nl-BE', { year: 'numeric', month: 'short', day: 'numeric' }).format(new Date(value))
} }
function temporalRangeLabel(items: DatasetCreateResponse[]): string | null {
const firstValue = items[0]?.observed_at
const lastValue = items[items.length - 1]?.observed_at
if (!firstValue || !lastValue) {
return null
}
const first = new Date(firstValue)
const last = new Date(lastValue)
if (first.getTime() === last.getTime()) {
return formatObservationDate(firstValue)
}
if (first.getUTCFullYear() !== last.getUTCFullYear()) {
return `${first.getUTCFullYear()}-${last.getUTCFullYear()}`
}
if (first.getUTCMonth() === last.getUTCMonth()) {
const monthAndYear = new Intl.DateTimeFormat('nl-BE', { year: 'numeric', month: 'short' }).format(last)
return `${first.getUTCDate()}-${last.getUTCDate()} ${monthAndYear}`
}
return `${formatObservationDate(firstValue)} - ${formatObservationDate(lastValue)}`
}
function formatDatasetObservation(dataset: DatasetCreateResponse): string { function formatDatasetObservation(dataset: DatasetCreateResponse): string {
if (dataset.source_name === 'vmm_flood_hazard') { if (dataset.source_name === 'vmm_flood_hazard') {
return `scenario ${String(dataset.source_metadata?.['climate_context'] ?? '')} · ${String(dataset.source_metadata?.['probability_class'] ?? '')}` return `scenario ${String(dataset.source_metadata?.['climate_context'] ?? '')} · ${String(dataset.source_metadata?.['probability_class'] ?? '')}`
@@ -983,6 +1002,10 @@ export function MapWorkspace({
const activeTemporalSeriesGroup = activeTemporalSeriesGroups.find((group) => group.key === selectedTemporalSeriesKey) const activeTemporalSeriesGroup = activeTemporalSeriesGroups.find((group) => group.key === selectedTemporalSeriesKey)
?? activeTemporalSeriesGroups[0] ?? activeTemporalSeriesGroups[0]
const activeTemporalSeries = activeTemporalSeriesGroup?.items ?? EMPTY_TEMPORAL_SERIES const activeTemporalSeries = activeTemporalSeriesGroup?.items ?? EMPTY_TEMPORAL_SERIES
const activeSeriesIsDailyGrb = activeTemporalSeries.length >= 2
&& activeTemporalSeries.every((dataset) => dataset.source_name === 'grb')
&& new Date(activeTemporalSeries[activeTemporalSeries.length - 1].observed_at ?? 0).getTime()
- new Date(activeTemporalSeries[0].observed_at ?? 0).getTime() <= 7 * 24 * 60 * 60 * 1000
useEffect(() => { useEffect(() => {
onSetContextSourceLabel( onSetContextSourceLabel(
@@ -1416,10 +1439,7 @@ export function MapWorkspace({
const evolutionAvailable = temporalGroups.some((group) => group.items.length >= 2) const evolutionAvailable = temporalGroups.some((group) => group.items.length >= 2)
const available = Boolean(dataset) && (analysisMode === 'current' || evolutionAvailable) const available = Boolean(dataset) && (analysisMode === 'current' || evolutionAvailable)
const active = activeThemeId === theme.id const active = activeThemeId === theme.id
const firstObservation = temporalGroup?.items[0]?.observed_at const temporalRange = temporalGroup ? temporalRangeLabel(temporalGroup.items) : null
const lastObservation = temporalGroup?.items[temporalGroup.items.length - 1]?.observed_at
const firstYear = firstObservation ? new Date(firstObservation).getUTCFullYear() : null
const lastYear = lastObservation ? new Date(lastObservation).getUTCFullYear() : null
return ( return (
<button <button
className={active ? 'geo-theme-option geo-theme-option-active' : 'geo-theme-option'} className={active ? 'geo-theme-option geo-theme-option-active' : 'geo-theme-option'}
@@ -1435,7 +1455,7 @@ export function MapWorkspace({
<small> <small>
{analysisMode === 'evolution' {analysisMode === 'evolution'
? evolutionAvailable ? evolutionAvailable
? `${temporalGroup.items.length} meetmomenten${firstYear && lastYear ? ` · ${firstYear}-${lastYear}` : ''}` ? `${temporalGroup.items.length} meetmomenten${temporalRange ? ` · ${temporalRange}` : ''}`
: dataset : dataset
? 'Alleen huidige toestand' ? 'Alleen huidige toestand'
: 'Bron nog niet ingeladen' : 'Bron nog niet ingeladen'
@@ -1500,48 +1520,55 @@ export function MapWorkspace({
) : null} ) : null}
{analysisMode === 'evolution' ? ( {analysisMode === 'evolution' ? (
<div className="geo-time-controls" aria-label="Meetmomenten vergelijken"> <>
{activeTemporalSeriesGroups.length > 1 ? ( <div className="geo-time-controls" aria-label="Meetmomenten vergelijken">
<label className="geo-series-control"> {activeTemporalSeriesGroups.length > 1 ? (
Reeks <label className="geo-series-control">
<select Reeks
value={activeTemporalSeriesGroup?.key ?? ''} <select
onChange={(event) => { value={activeTemporalSeriesGroup?.key ?? ''}
setSelectedTemporalSeriesKey(event.target.value) onChange={(event) => {
clearTemporalComparison() setSelectedTemporalSeriesKey(event.target.value)
}} clearTemporalComparison()
> }}
{activeTemporalSeriesGroups.map((group) => ( >
<option key={group.key} value={group.key}>{group.label}</option> {activeTemporalSeriesGroups.map((group) => (
<option key={group.key} value={group.key}>{group.label}</option>
))}
</select>
</label>
) : null}
<label>
Van
<select value={earlierDatasetId} onChange={(event) => { setEarlierDatasetId(event.target.value); clearTemporalComparison() }} disabled={activeTemporalSeries.length < 2}>
{activeTemporalSeries.map((dataset) => (
<option key={dataset.id} value={dataset.id}>{formatObservationDate(dataset.observed_at)}</option>
))} ))}
</select> </select>
</label> </label>
<label>
Naar
<select value={laterDatasetId} onChange={(event) => { setLaterDatasetId(event.target.value); clearTemporalComparison() }} disabled={activeTemporalSeries.length < 2}>
{activeTemporalSeries.map((dataset) => (
<option key={dataset.id} value={dataset.id}>{formatObservationDate(dataset.observed_at)}</option>
))}
</select>
</label>
<button
className="primary-action"
type="button"
disabled={!mapSelectionBbox || !earlierDatasetId || !laterDatasetId || temporalComparisonLoading}
onClick={runTemporalComparison}
>
{temporalComparisonLoading ? 'Vergelijken…' : 'Vergelijk periode'}
</button>
</div>
{activeSeriesIsDailyGrb ? (
<p className="geo-data-notice">
Dagelijkse GRB-edities tonen wijzigingen in de officiële registratie. Ze bewijzen niet dat een fysieke verandering exact tussen deze twee kalenderdagen plaatsvond.
</p>
) : null} ) : null}
<label> </>
Van
<select value={earlierDatasetId} onChange={(event) => { setEarlierDatasetId(event.target.value); clearTemporalComparison() }} disabled={activeTemporalSeries.length < 2}>
{activeTemporalSeries.map((dataset) => (
<option key={dataset.id} value={dataset.id}>{formatObservationDate(dataset.observed_at)}</option>
))}
</select>
</label>
<label>
Naar
<select value={laterDatasetId} onChange={(event) => { setLaterDatasetId(event.target.value); clearTemporalComparison() }} disabled={activeTemporalSeries.length < 2}>
{activeTemporalSeries.map((dataset) => (
<option key={dataset.id} value={dataset.id}>{formatObservationDate(dataset.observed_at)}</option>
))}
</select>
</label>
<button
className="primary-action"
type="button"
disabled={!mapSelectionBbox || !earlierDatasetId || !laterDatasetId || temporalComparisonLoading}
onClick={runTemporalComparison}
>
{temporalComparisonLoading ? 'Vergelijken…' : 'Vergelijk periode'}
</button>
</div>
) : null} ) : null}
<label className="geo-scope-select"> <label className="geo-scope-select">
+5 -4
View File
@@ -236,11 +236,9 @@ def build_partition_features(
source_urls.append(source_url) source_urls.append(source_url)
for source_feature in payload.get("features") or []: for source_feature in payload.get("features") or []:
bbox_feature_count += 1 bbox_feature_count += 1
feature_id = str(source_feature.get("id") or "") feature_id = str(source_feature.get("id") or "").strip()
if not feature_id: if not feature_id:
feature_id = hashlib.sha256( raise RuntimeError("GRB building feature is missing an official source identity")
json.dumps(source_feature.get("geometry"), sort_keys=True).encode("utf-8")
).hexdigest()
if feature_id in seen_ids: if feature_id in seen_ids:
continue continue
seen_ids.add(feature_id) seen_ids.add(feature_id)
@@ -661,6 +659,9 @@ def provision_dataset(
"feature_count": manifest["feature_count"], "feature_count": manifest["feature_count"],
"partition_count": len(partition_paths), "partition_count": len(partition_paths),
"partition_strategy": manifest["partition_strategy"], "partition_strategy": manifest["partition_strategy"],
"identity_stable": True,
"identity_scheme": "grb_ogc_feature_id",
"identity_prefixes": ["GBG."],
"selection_aggregation": { "selection_aggregation": {
"metric_key": "footprint_area", "metric_key": "footprint_area",
"method": "intersection_area", "method": "intersection_area",
+9 -5
View File
@@ -10,7 +10,6 @@ VectorFeatureService; this operator never writes directly to vector_features.
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import hashlib
import json import json
import os import os
import sys import sys
@@ -340,11 +339,11 @@ def build_partition_features(
pages_by_collection[collection.name] += 1 pages_by_collection[collection.name] += 1
for source_feature in payload.get("features") or []: for source_feature in payload.get("features") or []:
bbox_feature_count += 1 bbox_feature_count += 1
raw_id = str(source_feature.get("id") or "") raw_id = str(source_feature.get("id") or "").strip()
if not raw_id: if not raw_id:
raw_id = hashlib.sha256( raise RuntimeError(
json.dumps(source_feature.get("geometry"), sort_keys=True).encode("utf-8") f"GRB {collection.name} feature is missing an official source identity"
).hexdigest() )
feature_id = f"{collection.name}:{raw_id}" feature_id = f"{collection.name}:{raw_id}"
if feature_id in seen_ids: if feature_id in seen_ids:
continue continue
@@ -736,6 +735,11 @@ def provision_dataset(
"feature_count": manifest["feature_count"], "feature_count": manifest["feature_count"],
"partition_count": len(partition_paths), "partition_count": len(partition_paths),
"partition_strategy": manifest["partition_strategy"], "partition_strategy": manifest["partition_strategy"],
"identity_stable": True,
"identity_scheme": "grb_ogc_feature_id",
"identity_prefixes": [
f"{item.name}:{item.name}." for item in definition.collections
],
"selection_aggregation": { "selection_aggregation": {
"metric_key": definition.metric_key, "metric_key": definition.metric_key,
"method": definition.metric_method, "method": definition.metric_method,