diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e439e2d..7dd8fa00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,23 @@ # Changelog +## Sprint 206 Governed Buildings and Addresses Register snapshot (2026-07-15) + +- Added an explicit operator for the current official Digitaal Vlaanderen + building, building-unit and address OGC collections with complete pagination, + safety limits, retries and retained raw SHA256 evidence. +- Added exact EPSG:31370 Area clipping, EPSG:4326 building persistence and + classified reconciliation against checksummed persisted GRB partitions. +- Persisted only building lifecycle data and aggregate unit/address counts; + full addresses, street names and house/box numbers remain outside queryable + output, and ambiguous address/GRB relations are never forced. +- Added exact footprint, lifecycle, unit, address-status and confirmed-GRB + selection metrics, including correct filtered `feature_count` execution. +- Made the Map workspace prefer the richer Mol register snapshot only for the + matching Mol Area and retain regional GRB coverage everywhere else. +- Added source-inventory presentation, runtime packaging and focused operator, + privacy, reconciliation, metric and UI tests. + ## Sprint 205 Governed agricultural-use parcel history (2026-07-15) - Added an explicit ALZ operator for the definitive 2008-2025 annual diff --git a/backend/README.md b/backend/README.md index 1b207a22..77923b4a 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1132,6 +1132,28 @@ transport region. Every annual source ZIP and crop code list remains under the storage volume. PostGIS computes exact hectares for drawn rectangles and persisted Areas; parcel identities are deliberately unavailable for lineage. +## Buildings and Addresses Register snapshot + +After the Mol Area and regional GRB buildings have been provisioned, prepare +the official register evidence with: + +```bash +docker exec geointel python /app/scripts/provision_buildings_addresses_register.py --fetch-only +``` + +Review the generated manifest and then persist through DatasetService: + +```bash +docker exec geointel python /app/scripts/provision_buildings_addresses_register.py +``` + +The resulting `building_registry` Dataset uses ordinary EPSG:4326 +`vector_features`; no register-specific table or direct operator database write +exists. Exact PostGIS selection exposes footprint hectares, lifecycle counts, +aggregate unit/address counts and GRB reconciliation counts. Raw address pages +are checksummed storage evidence only. Address labels and house/box numbers are +not copied into queryable properties. + ## Helpful repository scripts - `bash scripts/backend_install.sh` diff --git a/backend/app/services/vector_feature_service.py b/backend/app/services/vector_feature_service.py index 81340537..04636cc2 100644 --- a/backend/app/services/vector_feature_service.py +++ b/backend/app/services/vector_feature_service.py @@ -26,6 +26,7 @@ FULL_AREA_CLIPPED_OPERATOR_TOOLS = { "provision_waterinfo_station_history.py", "provision_mol_bwk_natura2000.py", "provision_agricultural_parcel_history.py", + "provision_buildings_addresses_register.py", } @@ -144,6 +145,7 @@ class VectorFeatureService: "agricultural": "agriculture", "landbouw": "agriculture", "landbouwgebruik": "agriculture", + "building_registry": "buildings", } for candidate in candidates: if not isinstance(candidate, str) or not candidate.strip(): @@ -538,6 +540,10 @@ class VectorFeatureService: is_estimate = bool(config.get("is_estimate", False)) if not is_estimate and config.get("warning_only_when_estimate", True): warning = None + elif method == "feature_count" and (filter_property or dimension in {1, 2}): + metric_value = float( + db.query(func.count(VectorFeature.id)).filter(*metric_filter).scalar() or 0 + ) elif method != "feature_count": raise AppError( code="INVALID_SELECTION_AGGREGATION", diff --git a/backend/tests/test_sprint206_buildings_addresses_register.py b/backend/tests/test_sprint206_buildings_addresses_register.py new file mode 100644 index 00000000..18bad727 --- /dev/null +++ b/backend/tests/test_sprint206_buildings_addresses_register.py @@ -0,0 +1,356 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from uuid import uuid4 + +from shapely.geometry import Point, box, mapping +from shapely.ops import transform as transform_geometry + +from app.models import Dataset +from app.schemas.operations import VectorSelectionSummary +from app.services.vector_feature_service import VectorFeatureService + + +ROOT = Path(__file__).resolve().parents[2] +BBOX = {"min_x": 5.0, "min_y": 51.1, "max_x": 5.3, "max_y": 51.4, "crs": "EPSG:4326"} + + +def load_operator(): + script_path = ROOT / "scripts" / "provision_buildings_addresses_register.py" + spec = importlib.util.spec_from_file_location("buildings_addresses_register_operator", script_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def building_feature(object_id: str, geometry, status: str = "Gerealiseerd") -> dict: # noqa: ANN001 + return { + "type": "Feature", + "id": f"Gebouw.{object_id}", + "geometry": mapping(geometry), + "properties": { + "ObjectId": int(object_id), + "VersieId": "2026-07-15T08:00:00+02:00", + "GeometrieMethode": "IngemetenGRB", + "GebouwStatus": status, + }, + } + + +def unit_feature(object_id: str, building_id: str, point: Point) -> dict: + return { + "type": "Feature", + "id": f"Gebouweenheid.{object_id}", + "geometry": mapping(point), + "properties": { + "ObjectId": int(object_id), + "GebouwObjectId": int(building_id), + "GebouweenheidStatus": "Gerealiseerd", + "Functie": "NietGekend", + }, + } + + +def address_feature(object_id: str, point: Point) -> dict: + return { + "type": "Feature", + "id": f"Adres.{object_id}", + "geometry": mapping(point), + "properties": { + "ObjectId": int(object_id), + "AdresStatus": "InGebruik", + "PositieSpecificatie": "Gebouweenheid", + "VolledigAdres": "Teststraat 1 bus 2, 2400 Mol", + "Straatnaam": "Teststraat", + "Huisnummer": "1", + "Busnummer": "2", + }, + } + + +class ScalarQuery: + def __init__(self, value: float): + self.value = value + + def filter(self, *args): # noqa: ANN002, ARG002 + return self + + def scalar(self): + return self.value + + +class SequenceScalarSession: + def __init__(self, values: list[float]): + self.values = iter(values) + + def query(self, *args): # noqa: ANN002, ARG002 + return ScalarQuery(next(self.values)) + + +class OfficialResponse: + status_code = 200 + + def __init__(self, payload: dict, url: str): + self.payload = payload + self.url = url + self.content = json.dumps(payload).encode("utf-8") + + def json(self): + return self.payload + + def raise_for_status(self): + return None + + +class TwoPageOfficialSession: + def __init__(self): + self.calls = 0 + + def get(self, url, *, params, timeout): # noqa: ANN001, ARG002 + self.calls += 1 + if self.calls == 1: + payload = { + "type": "FeatureCollection", + "features": [building_feature("1", box(5.10, 51.20, 5.101, 51.201))], + "links": [{"rel": "next", "href": f"{url}?startIndex=1"}], + } + else: + payload = { + "type": "FeatureCollection", + "features": [building_feature("2", box(5.102, 51.20, 5.103, 51.201))], + "links": [], + } + return OfficialResponse(payload, f"{url}?page={self.calls}") + + +def normalized_fixture(module): # noqa: ANN001 + boundary_wgs84 = box(5.09, 51.19, 5.12, 51.22) + boundary_lambert72 = transform_geometry(module.TO_LAMBERT72.transform, boundary_wgs84) + polygon = box(5.10, 51.20, 5.105, 51.205) + buildings, summary = module.normalize_buildings( + [building_feature("100", polygon)], + boundary_lambert72, + ) + return boundary_wgs84, boundary_lambert72, polygon, buildings, summary + + +def test_official_collection_pagination_retains_checksummed_pages(tmp_path: Path) -> None: + module = load_operator() + session = TwoPageOfficialSession() + raw_dir = tmp_path / "raw" + + features, summary = module.fetch_collection( + session, + url=module.BUILDING_ITEMS_URL, + name="buildings", + bbox=(5.0, 51.0, 5.2, 51.2), + raw_dir=raw_dir, + page_limit=1, + max_features=10, + timeout=30, + ) + + assert [feature["properties"]["ObjectId"] for feature in features] == [1, 2] + assert summary["page_count"] == 2 + assert all((tmp_path / page["path"]).is_file() for page in summary["pages"]) + assert all(len(page["sha256"]) == 64 for page in summary["pages"]) + + +def test_buildings_are_clipped_in_lambert72_and_keep_lifecycle_status() -> None: + module = load_operator() + boundary = box(5.10, 51.20, 5.11, 51.21) + boundary_lambert72 = transform_geometry(module.TO_LAMBERT72.transform, boundary) + source = box(5.095, 51.195, 5.105, 51.205) + + buildings, summary = module.normalize_buildings( + [building_feature("100", source, "InAanbouw")], + boundary_lambert72, + ) + + assert summary == {"rejected_or_outside_count": 0, "clipped_count": 1} + record = buildings["100"] + assert record["status_key"] == "under_construction" + assert record["was_clipped"] is True + assert record["geometry_wgs84"].difference(boundary.buffer(1e-7)).area < 1e-12 + assert record["area_ha"] > 0 + + +def test_official_unit_relation_and_exact_address_position_are_aggregated_without_labels() -> None: + module = load_operator() + _, boundary_lambert72, polygon, buildings, _ = normalized_fixture(module) + point = polygon.centroid + units, unit_summary = module.normalize_units( + [unit_feature("200", "100", point)], + boundary_lambert72, + buildings, + ) + address_counts, address_summary = module.link_addresses( + [address_feature("300", point)], + boundary_lambert72, + buildings, + units, + ) + module.reconcile_with_grb( + buildings, + [{"source_feature_id": "GRB.1", "geometry_wgs84": polygon}], + ) + output, totals = module.build_output_features( + buildings, + units, + address_counts, + observed_date=module.date(2026, 7, 15), + area_name="Gemeente Mol - officiële grens", + ) + + assert unit_summary["orphan_building_count"] == 0 + assert address_summary["match_method_counts"] == {"unit_position_exact": 1} + assert totals["linked_unit_count"] == 1 + assert totals["linked_address_count"] == 1 + properties = output[0]["properties"] + assert properties["unit_count"] == 1 + assert properties["active_address_count"] == 1 + assert properties["grb_match_status"] == "matched" + for prohibited in ("VolledigAdres", "Straatnaam", "Huisnummer", "Busnummer", "HuisnummerLabel"): + assert prohibited not in properties + + +def test_ambiguous_unit_position_is_reported_and_never_forced() -> None: + module = load_operator() + boundary = box(5.09, 51.19, 5.12, 51.22) + boundary_lambert72 = transform_geometry(module.TO_LAMBERT72.transform, boundary) + point = Point(5.105, 51.205) + buildings, _ = module.normalize_buildings( + [ + building_feature("100", box(5.10, 51.20, 5.106, 51.21)), + building_feature("101", box(5.104, 51.20, 5.11, 51.21)), + ], + boundary_lambert72, + ) + units, _ = module.normalize_units( + [unit_feature("200", "100", point), unit_feature("201", "101", point)], + boundary_lambert72, + buildings, + ) + + counts, summary = module.link_addresses( + [address_feature("300", point)], + boundary_lambert72, + buildings, + units, + ) + + assert summary["ambiguous_address_count"] == 1 + assert summary["matched_address_count"] == 0 + assert not counts + + +def test_grb_reconciliation_distinguishes_exact_and_unmatched_geometry() -> None: + module = load_operator() + _, _, polygon, buildings, _ = normalized_fixture(module) + buildings["101"] = { + **buildings["100"], + "object_id": "101", + "geometry_wgs84": box(5.11, 51.21, 5.115, 51.215), + "geometry_lambert72": transform_geometry( + module.TO_LAMBERT72.transform, + box(5.11, 51.21, 5.115, 51.215), + ), + } + + summary = module.reconcile_with_grb( + buildings, + [{"source_feature_id": "GRB.1", "geometry_wgs84": polygon}], + ) + + assert buildings["100"]["grb_match_method"] == "exact_geometry" + assert buildings["100"]["grb_match_confidence"] == 1.0 + assert buildings["101"]["grb_match_status"] == "unmatched" + assert summary["match_status_counts"] == {"matched": 1, "unmatched": 1} + assert summary["match_rate"] == 0.5 + + +def test_status_and_relation_metrics_use_filtered_server_owned_aggregations() -> None: + module = load_operator() + metrics = module.selection_metrics() + assert {item["metric_key"] for item in metrics} >= { + "registered_building_count", + "realized_building_count", + "building_unit_count", + "linked_address_count", + "active_address_count", + "grb_matched_building_count", + } + status_metrics = [item for item in metrics if item["metric_key"].endswith("building_count")] + assert any(item.get("filter_property") == "building_status_key" for item in status_metrics) + assert "huishoudens" in next(item for item in metrics if item["metric_key"] == "linked_address_count")["warning"] + + +def test_filtered_feature_count_and_numeric_relations_validate_as_selection_summary() -> None: + module = load_operator() + dataset = Dataset( + id=uuid4(), + project_id=uuid4(), + name="buildings_addresses_register.geojson", + dataset_type="vector", + dataset_role="reference", + source_name=module.SOURCE_NAME, + reference_layer_name="building_registry", + source_metadata={ + "theme": "buildings", + "semantic_metrics": False, + "selection_aggregation": { + "metric_key": "building_footprint_area", + "method": "intersection_area", + "label": "Gebouwgrondoppervlakte", + "unit": "ha", + "geometry_dimension": 2, + }, + "selection_metrics": module.selection_metrics(), + }, + ) + session = SequenceScalarSession([100_000, 2, 0, 1, 0, 4, 3, 4, 3, 2]) + + result = VectorFeatureService.summarize_features_by_bbox( + session, + dataset=dataset, + bbox=BBOX, + total_feature_count=3, + full_dataset_area=True, + ) + + metrics = {item["metric_key"]: item for item in result["metrics"]} + assert result["metric_value"] == 10.0 + assert metrics["registered_building_count"]["metric_value"] == 3 + assert metrics["realized_building_count"]["metric_value"] == 2 + assert metrics["building_unit_count"]["metric_value"] == 4 + assert metrics["active_address_count"]["metric_value"] == 3 + assert metrics["grb_matched_building_count"]["metric_value"] == 2 + VectorSelectionSummary(**result) + + +def test_operator_is_canonical_packaged_and_mol_scoped_in_explorer() -> None: + operator = (ROOT / "scripts/provision_buildings_addresses_register.py").read_text(encoding="utf-8") + service = (ROOT / "backend/app/services/vector_feature_service.py").read_text(encoding="utf-8") + dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8") + readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8") + workspace = (ROOT / "frontend/src/components/map/MapWorkspace.tsx").read_text(encoding="utf-8") + catalog = (ROOT / "frontend/src/components/datasets/SourceCatalogPanel.tsx").read_text(encoding="utf-8") + display = (ROOT / "frontend/src/lib/datasetDisplay.ts").read_text(encoding="utf-8") + + assert "/datasets/upload" in operator + assert "VectorFeature" not in operator + assert "INSERT INTO vector_features" not in operator + assert "VolledigAdres" in operator and '"VolledigAdres", "Straatnaam"' in operator + assert '"provision_buildings_addresses_register.py"' in service + assert "COPY scripts/provision_buildings_addresses_register.py" in dockerfile + assert "py_compile scripts/provision_buildings_addresses_register.py" in readiness + assert "datasetCoversSelectedArea" in workspace + assert "Gebouwen- en Adressenregister" in catalog + assert "building_registry: 'Gebouwenregister'" in display + assert "digitaal_vlaanderen_buildings_addresses_register: 'Digitaal Vlaanderen'" in display diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index eada6986..418893ab 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -80,6 +80,7 @@ COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_off COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py +COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_buildings_addresses_register.py COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py COPY scripts/provision_geographic_scope.py /app/scripts/provision_geographic_scope.py diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index b3d380a8..018d47fe 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -1639,6 +1639,21 @@ Datasets share one scope-specific temporal series, but declare `identity_stable=false`; their temporal response compares area totals and returns no parcel-level added/removed/modified claims. +For `reference_layer_name=building_registry`, the primary metric is exact +intersected building-footprint area in hectares. Supplemental server-owned +metrics expose register building/status counts, aggregate building-unit and +address-status counts, and confirmed GRB matches. Filtered `feature_count` +metrics apply their configured property filter in PostGIS just like filtered +area/sum metrics. Address labels and house/box numbers are never part of the +queryable Feature properties or response contract. + +The governed snapshot is scoped to its persisted `area_id`. The frontend may +prefer it over the regional GRB building layer only when that exact Area is +active; another municipality or the full region must continue to use the +regional GRB Dataset. No new register-specific API endpoint exists: upload, +GeoJSON, exact selection and temporal provenance use the existing Dataset +contracts. + ## Local GeoIntel assistant The assistant is an optional read-only language interface over persisted diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index eb2fef6c..4a2c2daa 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -329,12 +329,44 @@ quality metrics. ## Gebouwenregister -- Naam: Gebouwenregister Vlaanderen -- Type: vector/API/metadata -- Gebruik: gebouwmetadata en statusinformatie -- Toegang: nader te bepalen -- Cache: PostGIS/metadata tabellen -- Prioriteit: V2 +The governed operator `scripts/provision_buildings_addresses_register.py` +reads the official Digitaal Vlaanderen OGC API Features collections +`Gebouw`, `Gebouweenheid` and `Adres`. The source is continuously updated; +GeoIntel therefore creates a dated snapshot rather than claiming an annual +historical series. Raw response pages, request URLs and SHA256 checksums are +retained as operator evidence. + +Building polygons are clipped against the exact persisted Area in +EPSG:31370, transformed to EPSG:4326 and persisted through the normal Dataset +upload route with `source_name=digitaal_vlaanderen_buildings_addresses_register` +and `reference_layer_name=building_registry`. Register lifecycle state remains +separate from GRB geometry. Each register building receives an explicit +`matched`, `review`, `ambiguous` or `unmatched` GRB reconciliation result; a +low-confidence or duplicate match is never silently promoted. + +Building units use the official `GebouwObjectId` relation. The public address +collection does not expose that relation directly, so addresses are linked +only through an exact, unambiguous unit position or unambiguous polygon +containment. Ambiguous and unmatched rows are counted in the manifest and are +never forced to the nearest building. + +The queryable layer contains building polygons, lifecycle status and aggregate +unit/address counts only. Street names, full addresses, house numbers and box +numbers are excluded. Address counts are not households, dwellings, residents +or population. Raw source pages remain restricted operator evidence and are +not exposed by the API or map. + +Official endpoints and catalogues: + +- https://geo.api.vlaanderen.be/Gebouwenregister/ogc/features/v1/collections/Gebouw/items +- https://geo.api.vlaanderen.be/Gebouwenregister/ogc/features/v1/collections/Gebouweenheid/items +- https://geo.api.vlaanderen.be/Adressenregister/ogc/features/v1/collections/Adres/items +- https://www.vlaanderen.be/datavindplaats/catalogus/gebouwen-en-adressenregister +- https://www.vlaanderen.be/datavindplaats/catalogus/gebouwenregister + +Digitaal Vlaanderen is transitioning the download products in summer 2026. +This operator uses the current production OGC API and does not depend on the +retiring `/v2/extract` path. ## Lokale demo datasets diff --git a/docs/DATA_SPECIFICATION.md b/docs/DATA_SPECIFICATION.md index 2a28e8a1..dbe17977 100644 --- a/docs/DATA_SPECIFICATION.md +++ b/docs/DATA_SPECIFICATION.md @@ -61,22 +61,43 @@ GRB-gebouwpolygonen worden gebruikt als ground-truth proxy. Niet absoluut perfec ### Rol -Aanvullende gebouwinformatie bij geometrieën. +Authoritative register lifecycle and relation metadata attached to a dated +building snapshot. It complements GRB; it does not replace the separate GRB +footprint evidence or its QA role. ### Gebruik -- gebouwmetadata -- identificatie -- status -- koppeling met GRB-gebouwpolygonen indien mogelijk +- stable building object and version identity +- official lifecycle status and geometry method +- aggregate registered building-unit counts by lifecycle status +- aggregate address counts by register status +- classified reconciliation against persisted GRB geometry ### Type -Vector/API/metadata. +Official OGC API Features input, normalized polygon GeoJSON upload and +PostGIS `vector_features` output. -### Prioriteit +### Persisted contract -V2 of V1.5. +- `dataset_role=reference` +- `source_name=digitaal_vlaanderen_buildings_addresses_register` +- `reference_layer_name=building_registry` +- `temporal_granularity=snapshot` +- stable building identity within the register, but no fabricated historical + observation between snapshots +- polygon geometry clipped in EPSG:31370 and persisted as EPSG:4326 +- selection metrics: exact footprint hectares, building lifecycle counts, + unit counts, address-status counts and confirmed GRB match counts + +### Privacy and semantics + +Queryable properties may contain register object ids, lifecycle status, +geometry method, aggregate unit/address counts and GRB reconciliation evidence. +They must not contain `VolledigAdres`, `Straatnaam`, `Huisnummer`, +`HuisnummerLabel` or `Busnummer`. Address and unit counts may not be labelled +as population, residents, households or dwellings. Building footprint is +ground area, not floor area, height or volume. ## OpenStreetMap diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 7543f77d..4888e45e 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -133,6 +133,27 @@ artifact; Dataset and vector_feature rows remain the queryable PostGIS state. The manifest binds source, crop-code list and upload artifact checksums. A checksum conflict with an existing annual Dataset fails closed. +Buildings and Addresses Register snapshot evidence lives under: + +```text +storage/operator-evidence/buildings-addresses-register/mol/{observed-date}/ + raw/ + buildings_page_*.json + building_units_page_*.json + addresses_page_*.json + buildings_addresses_register.geojson + buildings_addresses_register.manifest.json +``` + +Raw pages contain the unmodified official response and are retained only as +checksummed operator evidence. They can contain address labels and must never +be served as a map/API artifact. The normalized GeoJSON deliberately contains +one polygon per register building with lifecycle state, aggregate relation +counts and classified GRB reconciliation only. It enters PostGIS exclusively +through DatasetService and ordinary `vector_features`; the operator never +writes database rows directly. The manifest binds all raw pages, the normalized +artifact, exact Area boundary and every GRB partition used for reconciliation. + Offline demo export artifacts can be inspected and cleaned with: ```bash diff --git a/frontend/README.md b/frontend/README.md index 6cbcf86f..b72403f1 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -484,6 +484,14 @@ Object additions/removals stay hidden because annual parcel identity is not stable. The Sources inventory only labels the series available after real Datasets exist. +When the governed Buildings and Addresses Register snapshot is loaded for Mol, +the `Bebouwing` theme automatically prefers that richer Dataset only while the +exact Mol Area is active. It shows register lifecycle, unit/address aggregate +metrics and GRB reconciliation without exposing address labels. Selecting +another municipality or the complete Kempen scope falls back to the complete +regional GRB building layer. This avoids presenting a Mol-only snapshot as +regional coverage. + ## Useful repository scripts - `bash scripts/frontend_install.sh` diff --git a/frontend/src/components/datasets/SourceCatalogPanel.tsx b/frontend/src/components/datasets/SourceCatalogPanel.tsx index d106a7fe..457dd782 100644 --- a/frontend/src/components/datasets/SourceCatalogPanel.tsx +++ b/frontend/src/components/datasets/SourceCatalogPanel.tsx @@ -108,6 +108,12 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E ) const bwkDatasets = ready.filter((dataset) => dataset.source_name === 'inbo_bwk_natura2000') const agricultureDatasets = ready.filter((dataset) => dataset.source_name === 'agentschap_landbouw_zeevisserij_agricultural_parcels') + const buildingsRegisterDatasets = ready.filter( + (dataset) => dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register', + ) + const latestBuildingsRegister = [...buildingsRegisterDatasets].sort( + (left, right) => new Date(right.observed_at ?? 0).getTime() - new Date(left.observed_at ?? 0).getTime(), + )[0] const agricultureYears = agricultureDatasets .flatMap((dataset) => dataset.observed_at ? [new Date(dataset.observed_at).getUTCFullYear()] : []) const pendingSources = AVAILABLE_SOURCES.filter((source) => { @@ -115,6 +121,7 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E if (source.key === 'historical_orthophoto') return historicalOrthophotos.length === 0 if (source.key === 'bwk') return bwkDatasets.length === 0 if (source.key === 'agriculture') return agricultureDatasets.length === 0 + if (source.key === 'buildings_register') return buildingsRegisterDatasets.length === 0 return true }) const themes = Object.keys(THEME_LABELS).map((theme) => { @@ -174,7 +181,7 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E ))} - {waterinfoDatasets.length > 0 || historicalOrthophotos.length > 0 || bwkDatasets.length > 0 || agricultureDatasets.length > 0 ? ( + {waterinfoDatasets.length > 0 || historicalOrthophotos.length > 0 || bwkDatasets.length > 0 || agricultureDatasets.length > 0 || buildingsRegisterDatasets.length > 0 ? (
{waterinfoDatasets.length > 0 ? (
@@ -204,6 +211,17 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E

Oppervlakte en officiële hoofdteeltgroepen zijn historisch vergelijkbaar; perceelidentiteiten blijven bewust niet gekoppeld tussen jaren.

) : null} + {latestBuildingsRegister ? ( +
+ Gebouwen- en Adressenregister + + {(latestBuildingsRegister.feature_count ?? 0).toLocaleString('nl-BE')} gebouwen · {' '} + {Number(latestBuildingsRegister.source_metadata?.['building_unit_count'] ?? 0).toLocaleString('nl-BE')} eenheden · {' '} + {Number(latestBuildingsRegister.source_metadata?.['linked_address_count'] ?? 0).toLocaleString('nl-BE')} gekoppelde adressen + +

Registerstatus en geaggregeerde koppelingen voor Mol; adreslabels en persoonsgegevens worden niet in de kaartlaag getoond.

+
+ ) : null}
) : null} diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index e000b2e8..8337a406 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -120,8 +120,22 @@ function datasetMatchesTheme(dataset: DatasetCreateResponse, theme: DataTheme): return theme.tokens.some((token) => searchText.includes(token)) } -function pickThemeDataset(datasets: DatasetCreateResponse[], theme: DataTheme): DatasetCreateResponse | null { - const candidates = datasets.filter((dataset) => datasetMatchesTheme(dataset, theme)) +function datasetCoversSelectedArea(dataset: DatasetCreateResponse, selectedAreaId: string | null): boolean { + const coverageScope = String(dataset.source_metadata?.['coverage_scope'] ?? '') + if (coverageScope !== 'municipality' || !dataset.area_id) { + return true + } + return Boolean(selectedAreaId) && dataset.area_id === selectedAreaId +} + +function pickThemeDataset( + datasets: DatasetCreateResponse[], + theme: DataTheme, + selectedAreaId: string | null, +): DatasetCreateResponse | null { + const candidates = datasets.filter( + (dataset) => datasetMatchesTheme(dataset, theme) && datasetCoversSelectedArea(dataset, selectedAreaId), + ) candidates.sort((left, right) => { const score = (dataset: DatasetCreateResponse) => (dataset.reference_layer_name && theme.tokens.includes(dataset.reference_layer_name.toLowerCase()) ? 1_000_000 : 0) + @@ -129,6 +143,7 @@ function pickThemeDataset(datasets: DatasetCreateResponse[], theme: DataTheme): (dataset.source_name === 'department_omgeving_land_use' ? 90_000 : 0) + (dataset.source_name === 'inbo_bwk_natura2000' ? 95_000 : 0) + (dataset.source_name === 'agentschap_landbouw_zeevisserij_agricultural_parcels' ? 98_000 : 0) + + (dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register' ? 120_000 : 0) + (dataset.dataset_role === 'reference' ? 10_000 : 0) + (dataset.observed_at ? new Date(dataset.observed_at).getTime() / 100_000_000 : 0) + (dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0) @@ -636,9 +651,9 @@ export function MapWorkspace({ const themeDatasetMap = useMemo( () => Object.fromEntries( - DATA_THEMES.map((theme) => [theme.id, pickThemeDataset(availableMapDatasets, theme)]), + DATA_THEMES.map((theme) => [theme.id, pickThemeDataset(availableMapDatasets, theme, selectedMapAreaId)]), ) as Record, - [availableMapDatasets], + [availableMapDatasets, selectedMapAreaId], ) const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0] const activeThemeMapStyle = DATA_THEME_MAP_STYLES[activeTheme.id] @@ -740,7 +755,7 @@ export function MapWorkspace({ }, [activeTemporalSeries]) useEffect(() => { - if (advancedMode || !activeThemeDataset || (selectedMapDataset && datasetMatchesTheme(selectedMapDataset, activeTheme))) { + if (advancedMode || !activeThemeDataset || selectedMapDataset?.id === activeThemeDataset.id) { return } onOpenDatasetInMap(activeThemeDataset) diff --git a/frontend/src/lib/datasetDisplay.ts b/frontend/src/lib/datasetDisplay.ts index ea408047..91d3ab27 100644 --- a/frontend/src/lib/datasetDisplay.ts +++ b/frontend/src/lib/datasetDisplay.ts @@ -9,6 +9,7 @@ const DATASET_LABEL_BY_LAYER: Record = { forest: 'Bos en groen', nature_value: 'Natuurwaarde', agriculture: 'Landbouwgebruikspercelen', + building_registry: 'Gebouwenregister', regional_boundary: 'Grens vervoerregio Kempen', municipality_boundaries: 'Gemeentegrenzen Kempen', } @@ -17,6 +18,7 @@ const DATASET_SOURCE_LABELS: Record = { department_omgeving_land_use: 'Departement Omgeving', agentschap_landbouw_zeevisserij_agricultural_parcels: 'Agentschap Landbouw en Zeevisserij', digitaal_vlaanderen_orthophoto: 'Digitaal Vlaanderen', + digitaal_vlaanderen_buildings_addresses_register: 'Digitaal Vlaanderen', grb: 'GRB', historical_landuse: 'Digitaal Vlaanderen', inbo_bwk_natura2000: 'INBO', diff --git a/scripts/README.md b/scripts/README.md index 94ef97b5..86c6cf92 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1481,6 +1481,33 @@ Datasets. `--force` refreshes retained evidence but cannot silently replace a conflicting persisted annual checksum. Use `--scope mol` for an independent municipal series. +## Buildings and Addresses Register snapshot + +Prepare and audit the current official Mol snapshot without persistence: + +```bash +docker exec geointel python /app/scripts/provision_buildings_addresses_register.py --fetch-only +``` + +Import the audited artifact through the canonical Dataset upload route: + +```bash +docker exec geointel python /app/scripts/provision_buildings_addresses_register.py +``` + +The operator requires a persisted Mol Area and the complete regional GRB +buildings Dataset with valid manifest/partition checksums. It reads only the +official `Gebouw`, `Gebouweenheid` and `Adres` OGC collections, clips in +EPSG:31370 and retains every raw response page under the storage volume. +`--force` refetches evidence; it cannot overwrite a conflicting snapshot for +the same Area/date. Safety limits are configurable with `--page-limit`, +`--max-buildings`, `--max-units` and `--max-addresses`. + +Only aggregate unit/address counts enter the queryable building layer. Review +`address_relations`, `grb_reconciliation`, checksums and limitations in the +manifest before accepting a broader import. Raw address response pages are +operator evidence and must not be published. + ## Tower deployment Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime: diff --git a/scripts/provision_buildings_addresses_register.py b/scripts/provision_buildings_addresses_register.py new file mode 100644 index 00000000..090e82f1 --- /dev/null +++ b/scripts/provision_buildings_addresses_register.py @@ -0,0 +1,1197 @@ +"""Provision a privacy-minimized Buildings and Addresses Register snapshot. + +The explicit operator reads the official Digitaal Vlaanderen OGC API Features +collections for buildings, building units and addresses. It retains raw, +checksummed source pages as operator evidence, but the queryable Dataset only +contains building polygons and aggregate relation counts. Street names, house +numbers, box numbers and complete address labels are never copied into +``vector_features``. + +Building-unit relations use the official ``GebouwObjectId``. Address-to- +building relations use the official address position and are classified as an +exact unit-position match, an unambiguous containing-building match, ambiguous +or unmatched. GRB reconciliation uses the latest persisted regional GRB +manifest and records exact/high-IoU/review/unmatched outcomes per building. +All persistence flows through the canonical Dataset upload route. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +import unicodedata +from collections import Counter, defaultdict +from datetime import date, datetime, time, timezone +from pathlib import Path +from typing import Any, Iterable + +import requests +from pyproj import Transformer +from requests.adapters import HTTPAdapter +from shapely.geometry import MultiPolygon, Point, Polygon, mapping, shape +from shapely.ops import transform as transform_geometry +from shapely.ops import unary_union +from shapely.strtree import STRtree +from shapely.validation import make_valid +from urllib3.util.retry import Retry + + +BUILDING_ITEMS_URL = ( + "https://geo.api.vlaanderen.be/Gebouwenregister/ogc/features/v1/collections/Gebouw/items" +) +UNIT_ITEMS_URL = ( + "https://geo.api.vlaanderen.be/Gebouwenregister/ogc/features/v1/collections/Gebouweenheid/items" +) +ADDRESS_ITEMS_URL = ( + "https://geo.api.vlaanderen.be/Adressenregister/ogc/features/v1/collections/Adres/items" +) +CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/gebouwen-en-adressenregister" +BUILDING_CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/gebouwenregister" +CHANGE_NOTICE_URL = ( + "https://www.vlaanderen.be/digitaal-vlaanderen/" + "belangrijke-wijzigingen-bij-het-gebouwen-en-adressenregister-in-de-zomer-van-2026" +) +ATTRIBUTION = "Bron: Digitaal Vlaanderen" +SOURCE_NAME = "digitaal_vlaanderen_buildings_addresses_register" +DEFAULT_PROJECT_NAME = "Kempen Regional Workbench" +DEFAULT_AREA_NAME = "Gemeente Mol - officiele grens" +DEFAULT_API_URL = "http://127.0.0.1:8000" +DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/buildings-addresses-register/mol") +DEFAULT_PAGE_LIMIT = 1000 +DEFAULT_MAX_BUILDINGS = 100_000 +DEFAULT_MAX_UNITS = 200_000 +DEFAULT_MAX_ADDRESSES = 200_000 +SCHEMA_VERSION = 1 +TO_LAMBERT72 = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) +TO_WGS84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True) + +BUILDING_STATUS_KEYS = { + "gerealiseerd": "realized", + "gepland": "planned", + "nietgerealiseerd": "not_realized", + "inaanbouw": "under_construction", + "gehistoreerd": "historical", +} +BUILDING_STATUS_LABELS = { + "realized": "Gerealiseerd", + "planned": "Gepland", + "not_realized": "Niet gerealiseerd", + "under_construction": "In aanbouw", + "historical": "Gehistoreerd", + "unknown": "Onbekend", +} +UNIT_STATUS_KEYS = { + "gerealiseerd": "realized", + "gepland": "planned", + "nietgerealiseerd": "not_realized", + "gehistoreerd": "historical", +} +ADDRESS_STATUS_KEYS = { + "ingebruik": "in_use", + "voorgesteld": "proposed", + "gehistoreerd": "historical", + "afgekeurd": "rejected", + "inonderzoek": "under_review", +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Provision an official Buildings and Addresses Register snapshot for a persisted Area." + ) + parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME) + parser.add_argument("--area-name", default=DEFAULT_AREA_NAME) + parser.add_argument("--observed-date", type=date.fromisoformat, default=date.today()) + parser.add_argument("--base-url", default=DEFAULT_API_URL) + parser.add_argument("--output-root", type=Path, default=DEFAULT_OUTPUT_ROOT) + parser.add_argument("--page-limit", type=int, default=DEFAULT_PAGE_LIMIT) + parser.add_argument("--max-buildings", type=int, default=DEFAULT_MAX_BUILDINGS) + parser.add_argument("--max-units", type=int, default=DEFAULT_MAX_UNITS) + parser.add_argument("--max-addresses", type=int, default=DEFAULT_MAX_ADDRESSES) + parser.add_argument("--request-timeout", type=int, default=180) + parser.add_argument("--api-timeout", type=int, default=300) + parser.add_argument("--force", action="store_true") + parser.add_argument("--fetch-only", action="store_true") + return parser.parse_args() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def observed_at(value: date) -> str: + return datetime.combine(value, time.min, tzinfo=timezone.utc).isoformat() + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_bytes_atomic(path: Path, value: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(f"{path.suffix}.partial") + temporary.write_bytes(value) + temporary.replace(path) + + +def write_json_atomic(path: Path, value: Any, *, pretty: bool = False) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(f"{path.suffix}.partial") + temporary.write_text( + json.dumps( + value, + ensure_ascii=False, + indent=2 if pretty else None, + separators=None if pretty else (",", ":"), + sort_keys=pretty, + ), + encoding="utf-8", + ) + temporary.replace(path) + + +def source_session() -> requests.Session: + retry = Retry( + total=5, + connect=5, + read=5, + status=5, + backoff_factor=1.0, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=frozenset({"GET"}), + raise_on_status=True, + ) + adapter = HTTPAdapter(max_retries=retry) + session = requests.Session() + session.headers.update({"User-Agent": "GeoIntel-Buildings-Addresses-Register-Operator/1.0"}) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +def response_data(response: requests.Response) -> Any: + try: + payload = response.json() + except ValueError as exc: + raise RuntimeError( + f"GeoIntel API returned non-JSON ({response.status_code}): {response.text[:300]}" + ) from exc + if not response.ok: + raise RuntimeError( + f"GeoIntel API failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:800]}" + ) + if not isinstance(payload, dict) or "data" not in payload: + raise RuntimeError("GeoIntel API response does not use the canonical data envelope") + return payload["data"] + + +def paginated_items(session: requests.Session, url: str, *, timeout: int) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + offset = 0 + total: int | None = None + while total is None or offset < total: + page = response_data(session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout)) + page_items = list(page.get("items") or []) + page_total = int(page.get("total") or 0) + if total is None: + total = page_total + elif page_total != total: + raise RuntimeError("GeoIntel pagination total changed while reading the workspace") + items.extend(page_items) + if not page_items: + break + offset += len(page_items) + if total is not None and len(items) != total: + raise RuntimeError(f"GeoIntel pagination returned {len(items)} of {total} items") + return items + + +def polygonal_geometry(geometry): + if geometry is None or geometry.is_empty: + return None + if not geometry.is_valid: + geometry = make_valid(geometry) + polygons: list[Polygon] = [] + + def collect(candidate) -> None: + if candidate is None or candidate.is_empty: + return + if isinstance(candidate, Polygon): + polygons.append(candidate) + elif isinstance(candidate, MultiPolygon): + polygons.extend(part for part in candidate.geoms if not part.is_empty) + elif hasattr(candidate, "geoms"): + for part in candidate.geoms: + collect(part) + + collect(geometry) + if not polygons: + return None + result = unary_union(polygons) + if not result.is_valid: + result = make_valid(result) + return result if not result.is_empty and result.is_valid else None + + +def normalize_area_name(value: str) -> str: + return unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii").casefold() + + +def locate_workspace( + session: requests.Session, + base_url: str, + project_name: str, + area_name: str, + timeout: int, +) -> tuple[str, str, Any, list[dict[str, Any]]]: + projects = paginated_items(session, f"{base_url}/api/v1/projects", timeout=timeout) + project = next((item for item in projects if item.get("name") == project_name), None) + if not project: + raise RuntimeError(f"Project {project_name!r} is missing") + project_id = str(project["id"]) + areas = paginated_items(session, f"{base_url}/api/v1/projects/{project_id}/areas", timeout=timeout) + target = normalize_area_name(area_name) + area = next((item for item in areas if normalize_area_name(str(item.get("name") or "")) == target), None) + if not area or not area.get("geometry"): + raise RuntimeError(f"Persisted Area {area_name!r} with geometry is missing") + boundary = polygonal_geometry(shape(area["geometry"])) + if boundary is None: + raise RuntimeError("Persisted Area geometry is invalid or not polygonal") + datasets = paginated_items(session, f"{base_url}/api/v1/projects/{project_id}/datasets", timeout=timeout) + return project_id, str(area["id"]), boundary, datasets + + +def next_page_url(payload: dict[str, Any]) -> str | None: + for link in payload.get("links") or []: + if str(link.get("rel") or "").lower() == "next" and link.get("href"): + return str(link["href"]) + return None + + +def fetch_collection( + session: requests.Session, + *, + url: str, + name: str, + bbox: tuple[float, float, float, float], + raw_dir: Path, + page_limit: int, + max_features: int, + timeout: int, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + request_url: str | None = url + params: dict[str, str] | None = { + "f": "application/geo+json", + "bbox": ",".join(f"{value:.8f}" for value in bbox), + "limit": str(page_limit), + } + seen_urls: set[str] = set() + seen_ids: set[str] = set() + features: list[dict[str, Any]] = [] + pages: list[dict[str, Any]] = [] + duplicate_count = 0 + while request_url: + request_key = requests.Request("GET", request_url, params=params).prepare().url or request_url + if request_key in seen_urls: + raise RuntimeError(f"{name} pagination loop detected") + seen_urls.add(request_key) + response = session.get(request_url, params=params, timeout=timeout) + params = None + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection": + raise RuntimeError(f"{name} returned an invalid FeatureCollection") + raw_bytes = response.content + page_path = raw_dir / f"{name}_page_{len(pages) + 1:05d}.json" + write_bytes_atomic(page_path, raw_bytes) + page_features = list(payload.get("features") or []) + pages.append( + { + "path": str(page_path.relative_to(raw_dir.parent)), + "sha256": sha256_bytes(raw_bytes), + "size_bytes": len(raw_bytes), + "feature_count": len(page_features), + "source_url": response.url, + } + ) + for feature in page_features: + identity = str(feature.get("id") or (feature.get("properties") or {}).get("ObjectId") or "") + if not identity: + raise RuntimeError(f"{name} returned a feature without stable identity") + if identity in seen_ids: + duplicate_count += 1 + continue + seen_ids.add(identity) + features.append(feature) + if len(features) > max_features: + raise RuntimeError( + f"{name} exceeds the {max_features} feature safety limit; refusing truncated output" + ) + request_url = next_page_url(payload) + if not features: + raise RuntimeError(f"{name} returned no source features for the Area bbox") + return features, { + "collection": name, + "page_count": len(pages), + "bbox_feature_count": len(features), + "duplicate_count": duplicate_count, + "pages": pages, + } + + +def compact_key(value: Any) -> str: + normalized = unicodedata.normalize("NFKD", str(value or "")).encode("ascii", "ignore").decode("ascii") + return "".join(character.lower() for character in normalized if character.isalnum()) + + +def normalize_buildings( + source_features: Iterable[dict[str, Any]], + boundary_lambert72, +) -> tuple[dict[str, dict[str, Any]], dict[str, int]]: + records: dict[str, dict[str, Any]] = {} + rejected = 0 + clipped = 0 + for feature in source_features: + properties = dict(feature.get("properties") or {}) + object_id = str(properties.get("ObjectId") or "").strip() + geometry_payload = feature.get("geometry") + source_wgs84 = polygonal_geometry(shape(geometry_payload)) if geometry_payload else None + source_lambert72 = ( + polygonal_geometry(transform_geometry(TO_LAMBERT72.transform, source_wgs84)) + if source_wgs84 is not None + else None + ) + if not object_id or source_lambert72 is None or not source_lambert72.intersects(boundary_lambert72): + rejected += 1 + continue + was_clipped = not source_lambert72.within(boundary_lambert72) + clipped_lambert72 = polygonal_geometry(source_lambert72.intersection(boundary_lambert72)) + if clipped_lambert72 is None or clipped_lambert72.area <= 0: + rejected += 1 + continue + if was_clipped: + clipped += 1 + clipped_wgs84 = polygonal_geometry(transform_geometry(TO_WGS84.transform, clipped_lambert72)) + if clipped_wgs84 is None: + rejected += 1 + continue + raw_status = str(properties.get("GebouwStatus") or "").strip() + status_key = BUILDING_STATUS_KEYS.get(compact_key(raw_status), "unknown") + records[object_id] = { + "object_id": object_id, + "version_id": str(properties.get("VersieId") or ""), + "geometry_method": str(properties.get("GeometrieMethode") or ""), + "status_key": status_key, + "status_label": BUILDING_STATUS_LABELS[status_key], + "geometry_wgs84": clipped_wgs84, + "geometry_lambert72": clipped_lambert72, + "area_ha": float(clipped_lambert72.area) / 10_000.0, + "was_clipped": was_clipped, + } + if not records: + raise RuntimeError("No valid Buildings Register polygons intersect the persisted Area") + return records, {"rejected_or_outside_count": rejected, "clipped_count": clipped} + + +def normalize_units( + source_features: Iterable[dict[str, Any]], + boundary_lambert72, + buildings: dict[str, dict[str, Any]], +) -> tuple[list[dict[str, Any]], dict[str, int]]: + units: list[dict[str, Any]] = [] + outside = 0 + orphan = 0 + for feature in source_features: + properties = dict(feature.get("properties") or {}) + geometry_payload = feature.get("geometry") + point_wgs84 = shape(geometry_payload) if geometry_payload else None + if not isinstance(point_wgs84, Point) or point_wgs84.is_empty: + outside += 1 + continue + point_lambert72 = transform_geometry(TO_LAMBERT72.transform, point_wgs84) + if not boundary_lambert72.covers(point_lambert72): + outside += 1 + continue + building_id = str(properties.get("GebouwObjectId") or "").strip() + if building_id not in buildings: + orphan += 1 + status_key = UNIT_STATUS_KEYS.get(compact_key(properties.get("GebouweenheidStatus")), "unknown") + units.append( + { + "object_id": str(properties.get("ObjectId") or ""), + "building_id": building_id, + "status_key": status_key, + "function_key": compact_key(properties.get("Functie")) or "unknown", + "point_wgs84": point_wgs84, + } + ) + return units, {"outside_count": outside, "orphan_building_count": orphan} + + +def coordinate_key(point: Point) -> tuple[float, float]: + return round(float(point.x), 7), round(float(point.y), 7) + + +def link_addresses( + source_features: Iterable[dict[str, Any]], + boundary_lambert72, + buildings: dict[str, dict[str, Any]], + units: list[dict[str, Any]], +) -> tuple[dict[str, Counter[str]], dict[str, Any]]: + unit_positions: dict[tuple[float, float], list[dict[str, Any]]] = defaultdict(list) + for unit in units: + if unit["building_id"] in buildings: + unit_positions[coordinate_key(unit["point_wgs84"])].append(unit) + + building_ids = list(buildings) + building_geometries = [buildings[building_id]["geometry_wgs84"] for building_id in building_ids] + building_tree = STRtree(building_geometries) + counts: dict[str, Counter[str]] = defaultdict(Counter) + method_counts: Counter[str] = Counter() + status_counts: Counter[str] = Counter() + retained_address_count = 0 + for feature in source_features: + properties = dict(feature.get("properties") or {}) + geometry_payload = feature.get("geometry") + point_wgs84 = shape(geometry_payload) if geometry_payload else None + if not isinstance(point_wgs84, Point) or point_wgs84.is_empty: + continue + point_lambert72 = transform_geometry(TO_LAMBERT72.transform, point_wgs84) + if not boundary_lambert72.covers(point_lambert72): + continue + retained_address_count += 1 + status_key = ADDRESS_STATUS_KEYS.get(compact_key(properties.get("AdresStatus")), "unknown") + status_counts[status_key] += 1 + specificity = compact_key(properties.get("PositieSpecificatie")) + matched_building_id: str | None = None + method = "unmatched" + if specificity == "gebouweenheid": + candidates = unit_positions.get(coordinate_key(point_wgs84), []) + candidate_buildings = {candidate["building_id"] for candidate in candidates} + if len(candidate_buildings) == 1: + matched_building_id = next(iter(candidate_buildings)) + method = "unit_position_exact" if len(candidates) == 1 else "unit_position_building_unambiguous" + elif len(candidate_buildings) > 1: + method = "ambiguous_unit_position" + + if matched_building_id is None and method == "unmatched": + containing = [ + int(index) + for index in building_tree.query(point_wgs84) + if building_geometries[int(index)].covers(point_wgs84) + ] + if len(containing) == 1: + matched_building_id = building_ids[containing[0]] + method = "building_contains" + elif len(containing) > 1: + method = "ambiguous_building_contains" + + method_counts[method] += 1 + if matched_building_id is not None: + counts[matched_building_id]["address_count"] += 1 + counts[matched_building_id][f"address_status_{status_key}"] += 1 + counts[matched_building_id][f"address_match_{method}"] += 1 + + return counts, { + "retained_address_count": retained_address_count, + "match_method_counts": dict(sorted(method_counts.items())), + "status_counts": dict(sorted(status_counts.items())), + "matched_address_count": sum( + value for key, value in method_counts.items() if key in { + "unit_position_exact", "unit_position_building_unambiguous", "building_contains" + } + ), + "ambiguous_address_count": sum(value for key, value in method_counts.items() if key.startswith("ambiguous")), + "unmatched_address_count": int(method_counts.get("unmatched", 0)), + "privacy_field_violations": 0, + } + + +def find_grb_dataset(datasets: list[dict[str, Any]]) -> dict[str, Any]: + candidates = [ + dataset + for dataset in datasets + if dataset.get("status") == "ready" + and dataset.get("source_name") == "grb" + and dataset.get("reference_layer_name") == "buildings" + and (dataset.get("provenance_metadata") or {}).get("manifest_path") + ] + if not candidates: + raise RuntimeError("A persisted regional GRB buildings Dataset with manifest evidence is required") + candidates.sort(key=lambda item: str(item.get("observed_at") or item.get("imported_at") or ""), reverse=True) + return candidates[0] + + +def load_grb_reference( + grb_dataset: dict[str, Any], + boundary_wgs84, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + provenance = grb_dataset.get("provenance_metadata") or {} + manifest_path = Path(str(provenance.get("manifest_path") or "")) + if not manifest_path.is_file(): + raise RuntimeError(f"Persisted GRB manifest is missing: {manifest_path}") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("status") != "complete" or manifest.get("reference_truncated") is not False: + raise RuntimeError("Persisted GRB manifest is incomplete or truncated") + artifact_path = manifest_path.parent / str(manifest.get("artifact_filename") or "") + if not artifact_path.is_file() or sha256_file(artifact_path) != manifest.get("artifact_sha256"): + raise RuntimeError("Persisted GRB combined artifact checksum is invalid") + + records: list[dict[str, Any]] = [] + partition_checksums: dict[str, str] = {} + partition_dir = manifest_path.parent / "partitions" + for summary in manifest.get("partitions") or []: + partition_path = partition_dir / str(summary.get("filename") or "") + expected_checksum = str(summary.get("sha256") or "") + if not partition_path.is_file() or sha256_file(partition_path) != expected_checksum: + raise RuntimeError(f"Persisted GRB partition checksum is invalid: {partition_path.name}") + partition_checksums[partition_path.name] = expected_checksum + payload = json.loads(partition_path.read_text(encoding="utf-8")) + for feature in payload.get("features") or []: + geometry_payload = feature.get("geometry") + geometry = polygonal_geometry(shape(geometry_payload)) if geometry_payload else None + if geometry is None or not geometry.intersects(boundary_wgs84): + continue + properties = feature.get("properties") or {} + identity = str(feature.get("id") or properties.get("source_feature_id") or "") + if identity: + records.append({"source_feature_id": identity, "geometry_wgs84": geometry}) + if not records: + raise RuntimeError("Persisted GRB evidence contains no buildings intersecting the Area") + return records, { + "dataset_id": str(grb_dataset["id"]), + "observed_at": grb_dataset.get("observed_at"), + "manifest_path": str(manifest_path), + "artifact_sha256": manifest["artifact_sha256"], + "partition_checksums": partition_checksums, + "feature_count": len(records), + } + + +def reconcile_with_grb( + buildings: dict[str, dict[str, Any]], + grb_records: list[dict[str, Any]], +) -> dict[str, Any]: + grb_metric = [ + polygonal_geometry(transform_geometry(TO_LAMBERT72.transform, record["geometry_wgs84"])) + for record in grb_records + ] + valid_indexes = [index for index, geometry in enumerate(grb_metric) if geometry is not None] + geometries = [grb_metric[index] for index in valid_indexes] + tree = STRtree(geometries) + exact_lookup: dict[str, list[int]] = defaultdict(list) + for tree_index, geometry in enumerate(geometries): + exact_lookup[geometry.wkb_hex].append(tree_index) + + target_to_buildings: dict[str, list[str]] = defaultdict(list) + match_counts: Counter[str] = Counter() + for building_id, building in buildings.items(): + geometry = building["geometry_lambert72"] + best_tree_index: int | None = None + best_iou = 0.0 + method = "unmatched" + exact = exact_lookup.get(geometry.wkb_hex, []) + if len(exact) == 1 and geometries[exact[0]].equals(geometry): + best_tree_index = exact[0] + best_iou = 1.0 + method = "exact_geometry" + else: + for candidate in tree.query(geometry): + tree_index = int(candidate) + candidate_geometry = geometries[tree_index] + intersection_area = geometry.intersection(candidate_geometry).area + if intersection_area <= 0: + continue + union_area = geometry.union(candidate_geometry).area + iou = float(intersection_area / union_area) if union_area > 0 else 0.0 + if iou > best_iou: + best_iou = iou + best_tree_index = tree_index + if ( + best_tree_index is not None + and best_iou >= 0.99999999 + and geometry.hausdorff_distance(geometries[best_tree_index]) <= 0.001 + ): + method = "exact_geometry" + best_iou = 1.0 + elif best_iou >= 0.98: + method = "high_iou" + elif best_iou >= 0.80: + method = "review_iou" + else: + best_tree_index = None + best_iou = 0.0 + + if best_tree_index is None: + building["grb_match_status"] = "unmatched" + building["grb_match_method"] = "unmatched" + building["grb_match_confidence"] = 0.0 + building["grb_source_feature_id"] = None + continue + source_index = valid_indexes[best_tree_index] + target_id = grb_records[source_index]["source_feature_id"] + building["grb_match_status"] = "matched" if method != "review_iou" else "review" + building["grb_match_method"] = method + building["grb_match_confidence"] = round(best_iou, 8) + building["grb_source_feature_id"] = target_id + target_to_buildings[target_id].append(building_id) + + duplicate_targets = {target for target, ids in target_to_buildings.items() if len(ids) > 1} + for target_id in duplicate_targets: + for building_id in target_to_buildings[target_id]: + building = buildings[building_id] + building["grb_match_status"] = "ambiguous" + building["grb_match_method"] = "duplicate_grb_target" + + for building in buildings.values(): + match_counts[building["grb_match_status"]] += 1 + matched_targets = { + building["grb_source_feature_id"] + for building in buildings.values() + if building["grb_match_status"] == "matched" and building["grb_source_feature_id"] + } + realized = [building for building in buildings.values() if building["status_key"] == "realized"] + realized_matched = sum(1 for building in realized if building["grb_match_status"] == "matched") + return { + "match_status_counts": dict(sorted(match_counts.items())), + "matched_grb_feature_count": len(matched_targets), + "unmatched_grb_feature_count": max(0, len(grb_records) - len(matched_targets)), + "duplicate_grb_target_count": len(duplicate_targets), + "match_rate": round(match_counts.get("matched", 0) / len(buildings), 8), + "realized_match_rate": round(realized_matched / len(realized), 8) if realized else 0.0, + } + + +def build_output_features( + buildings: dict[str, dict[str, Any]], + units: list[dict[str, Any]], + address_counts: dict[str, Counter[str]], + *, + observed_date: date, + area_name: str, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + unit_counts: dict[str, Counter[str]] = defaultdict(Counter) + for unit in units: + building_id = unit["building_id"] + if building_id not in buildings: + continue + unit_counts[building_id]["unit_count"] += 1 + unit_counts[building_id][f"unit_status_{unit['status_key']}"] += 1 + + features: list[dict[str, Any]] = [] + status_counts: Counter[str] = Counter() + total_area_ha = 0.0 + total_units = 0 + total_addresses = 0 + prohibited_output_fields = {"VolledigAdres", "Straatnaam", "Huisnummer", "Busnummer", "HuisnummerLabel"} + for building_id, building in buildings.items(): + units_for_building = unit_counts[building_id] + addresses_for_building = address_counts[building_id] + properties = { + "source_name": SOURCE_NAME, + "source_feature_id": f"Gebouwenregister:Gebouw:{building_id}", + "reference_layer_name": "building_registry", + "theme": "buildings", + "authority_level": "authoritative", + "coverage_scope": "municipality", + "coverage_area": area_name, + "observed_at": observed_date.isoformat(), + "attribution": ATTRIBUTION, + "building_object_id": building_id, + "building_version_id": building["version_id"], + "building_status_key": building["status_key"], + "building_status_label": building["status_label"], + "building_geometry_method": building["geometry_method"], + "clipped_to_area": building["was_clipped"], + "building_area_ha": round(building["area_ha"], 8), + "unit_count": int(units_for_building.get("unit_count", 0)), + "realized_unit_count": int(units_for_building.get("unit_status_realized", 0)), + "planned_unit_count": int(units_for_building.get("unit_status_planned", 0)), + "historical_unit_count": int(units_for_building.get("unit_status_historical", 0)), + "not_realized_unit_count": int(units_for_building.get("unit_status_not_realized", 0)), + "unknown_unit_count": int(units_for_building.get("unit_status_unknown", 0)), + "address_count": int(addresses_for_building.get("address_count", 0)), + "active_address_count": int(addresses_for_building.get("address_status_in_use", 0)), + "grb_match_status": building["grb_match_status"], + "grb_match_method": building["grb_match_method"], + "grb_match_confidence": building["grb_match_confidence"], + "grb_source_feature_id": building["grb_source_feature_id"], + "privacy_profile": "aggregate_counts_only", + } + if prohibited_output_fields.intersection(properties): + raise RuntimeError("Privacy-minimized output unexpectedly contains address label fields") + feature_id = properties["source_feature_id"] + features.append( + { + "type": "Feature", + "id": feature_id, + "geometry": mapping(building["geometry_wgs84"]), + "properties": properties, + } + ) + status_counts[building["status_key"]] += 1 + total_area_ha += building["area_ha"] + total_units += properties["unit_count"] + total_addresses += properties["address_count"] + return features, { + "building_status_counts": dict(sorted(status_counts.items())), + "building_area_ha": round(total_area_ha, 6), + "linked_unit_count": total_units, + "linked_address_count": total_addresses, + } + + +def reusable_snapshot(snapshot_dir: Path) -> tuple[Path, Path, dict[str, Any]] | None: + artifact_path = snapshot_dir / "buildings_addresses_register.geojson" + manifest_path = snapshot_dir / "buildings_addresses_register.manifest.json" + if not artifact_path.is_file() or not manifest_path.is_file(): + return None + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + if manifest.get("schema_version") != SCHEMA_VERSION: + return None + if manifest.get("artifact_sha256") != sha256_file(artifact_path): + return None + for collection in manifest.get("collections") or []: + for page in collection.get("pages") or []: + page_path = snapshot_dir / str(page.get("path") or "") + if not page_path.is_file() or page.get("sha256") != sha256_file(page_path): + return None + for filename, checksum in (manifest.get("grb_reference") or {}).get("partition_checksums", {}).items(): + grb_manifest = Path(str((manifest.get("grb_reference") or {}).get("manifest_path") or "")) + partition_path = grb_manifest.parent / "partitions" / filename + if not partition_path.is_file() or sha256_file(partition_path) != checksum: + return None + return artifact_path, manifest_path, manifest + + +def prepare_snapshot( + session: requests.Session, + *, + boundary_wgs84, + datasets: list[dict[str, Any]], + snapshot_dir: Path, + area_name: str, + observed_date: date, + page_limit: int, + max_buildings: int, + max_units: int, + max_addresses: int, + timeout: int, + force: bool, +) -> tuple[Path, Path, dict[str, Any]]: + if not force: + reusable = reusable_snapshot(snapshot_dir) + if reusable: + return reusable + snapshot_dir.mkdir(parents=True, exist_ok=True) + raw_dir = snapshot_dir / "raw" + raw_dir.mkdir(parents=True, exist_ok=True) + boundary_lambert72 = polygonal_geometry(transform_geometry(TO_LAMBERT72.transform, boundary_wgs84)) + if boundary_lambert72 is None: + raise RuntimeError("Area boundary could not be transformed to EPSG:31370") + + buildings_raw, buildings_source = fetch_collection( + session, + url=BUILDING_ITEMS_URL, + name="buildings", + bbox=boundary_wgs84.bounds, + raw_dir=raw_dir, + page_limit=page_limit, + max_features=max_buildings, + timeout=timeout, + ) + units_raw, units_source = fetch_collection( + session, + url=UNIT_ITEMS_URL, + name="building_units", + bbox=boundary_wgs84.bounds, + raw_dir=raw_dir, + page_limit=page_limit, + max_features=max_units, + timeout=timeout, + ) + addresses_raw, addresses_source = fetch_collection( + session, + url=ADDRESS_ITEMS_URL, + name="addresses", + bbox=boundary_wgs84.bounds, + raw_dir=raw_dir, + page_limit=page_limit, + max_features=max_addresses, + timeout=timeout, + ) + buildings, building_filter = normalize_buildings(buildings_raw, boundary_lambert72) + units, unit_filter = normalize_units(units_raw, boundary_lambert72, buildings) + address_counts, address_relations = link_addresses( + addresses_raw, + boundary_lambert72, + buildings, + units, + ) + if address_relations["privacy_field_violations"]: + raise RuntimeError("Address privacy output contract failed") + + grb_dataset = find_grb_dataset(datasets) + grb_records, grb_reference = load_grb_reference(grb_dataset, boundary_wgs84) + grb_reconciliation = reconcile_with_grb(buildings, grb_records) + output_features, output_summary = build_output_features( + buildings, + units, + address_counts, + observed_date=observed_date, + area_name=area_name, + ) + generated_at = utc_now() + artifact = { + "type": "FeatureCollection", + "name": f"Gebouwen- en Adressenregister - {area_name} - {observed_date.isoformat()}", + "features": output_features, + "source": "Digitaal Vlaanderen Buildings and Addresses Register OGC API Features", + "source_urls": [BUILDING_ITEMS_URL, UNIT_ITEMS_URL, ADDRESS_ITEMS_URL], + "attribution": ATTRIBUTION, + "catalog_url": CATALOG_URL, + "observed_at": observed_date.isoformat(), + "coverage_area": area_name, + "privacy_profile": "aggregate_counts_only", + "reference_truncated": False, + "generated_at": generated_at, + } + artifact_path = snapshot_dir / "buildings_addresses_register.geojson" + write_json_atomic(artifact_path, artifact) + manifest = { + "schema_version": SCHEMA_VERSION, + "status": "complete", + "observed_at": observed_date.isoformat(), + "generated_at": generated_at, + "coverage_area": area_name, + "crs_source": "EPSG:4326", + "crs_clip": "EPSG:31370", + "crs_persisted": "EPSG:4326", + "boundary_sha256": sha256_bytes(json.dumps(mapping(boundary_wgs84), sort_keys=True).encode("utf-8")), + "boundary_bbox_wgs84": list(boundary_wgs84.bounds), + "collections": [buildings_source, units_source, addresses_source], + "building_filter": building_filter, + "unit_filter": unit_filter, + "address_relations": address_relations, + "grb_reference": grb_reference, + "grb_reconciliation": grb_reconciliation, + "feature_count": len(output_features), + **output_summary, + "privacy_profile": { + "queryable_output": "building polygons with aggregate relation counts", + "excluded_fields": ["VolledigAdres", "Straatnaam", "Huisnummer", "Busnummer", "HuisnummerLabel"], + "personal_data_exposed": False, + }, + "reference_truncated": False, + "catalog_url": CATALOG_URL, + "building_catalog_url": BUILDING_CATALOG_URL, + "change_notice_url": CHANGE_NOTICE_URL, + "attribution": ATTRIBUTION, + "artifact_path": str(artifact_path), + "artifact_sha256": sha256_file(artifact_path), + "artifact_size_bytes": artifact_path.stat().st_size, + "limitations": [ + "This is a continuously updated register captured as a dated snapshot, not an historical annual series.", + "Address-to-building links are classified by exact unit position or polygon containment; ambiguous and unmatched addresses are never forced.", + "Address counts are not households, residents, dwellings or population.", + "Building-unit counts describe registered functional units and do not imply residential use.", + "GRB matching is evidence for geometric reconciliation; register lifecycle status remains separate from GRB footprint geometry.", + ], + } + manifest_path = snapshot_dir / "buildings_addresses_register.manifest.json" + write_json_atomic(manifest_path, manifest, pretty=True) + return artifact_path, manifest_path, manifest + + +def selection_metrics() -> list[dict[str, Any]]: + return [ + { + "metric_key": "registered_building_count", + "method": "feature_count", + "label": "Gebouwen in het register", + "unit": "gebouwen", + }, + { + "metric_key": "realized_building_count", + "method": "feature_count", + "label": "Gerealiseerde gebouwen", + "unit": "gebouwen", + "filter_property": "building_status_key", + "filter_values": ["realized"], + }, + { + "metric_key": "under_construction_building_count", + "method": "feature_count", + "label": "Gebouwen in aanbouw", + "unit": "gebouwen", + "filter_property": "building_status_key", + "filter_values": ["under_construction"], + }, + { + "metric_key": "planned_building_count", + "method": "feature_count", + "label": "Geplande gebouwen", + "unit": "gebouwen", + "filter_property": "building_status_key", + "filter_values": ["planned"], + }, + { + "metric_key": "historical_building_count", + "method": "feature_count", + "label": "Gehistoreerde gebouwen", + "unit": "gebouwen", + "filter_property": "building_status_key", + "filter_values": ["historical"], + }, + { + "metric_key": "building_unit_count", + "method": "sum", + "property": "unit_count", + "label": "Geregistreerde gebouweenheden", + "unit": "eenheden", + "warning": "Gebouweenheden zijn functionele registereenheden en niet automatisch woningen.", + }, + { + "metric_key": "realized_building_unit_count", + "method": "sum", + "property": "realized_unit_count", + "label": "Gerealiseerde gebouweenheden", + "unit": "eenheden", + }, + { + "metric_key": "linked_address_count", + "method": "sum", + "property": "address_count", + "label": "Gekoppelde adressen", + "unit": "adressen", + "warning": "Adressen zijn geen huishoudens, woningen, inwoners of bevolkingsmeting.", + }, + { + "metric_key": "active_address_count", + "method": "sum", + "property": "active_address_count", + "label": "Adressen in gebruik", + "unit": "adressen", + "warning": "Adresstatus beschrijft het registerobject en zegt niets over bewoning.", + }, + { + "metric_key": "grb_matched_building_count", + "method": "feature_count", + "label": "Gebouwen met bevestigde GRB-match", + "unit": "gebouwen", + "filter_property": "grb_match_status", + "filter_values": ["matched"], + }, + ] + + +def upload_snapshot( + session: requests.Session, + *, + base_url: str, + project_id: str, + area_id: str, + artifact_path: Path, + manifest_path: Path, + manifest: dict[str, Any], + observed_date: date, + timeout: int, +) -> dict[str, Any]: + warning = ( + "Gebouwoppervlakte is grondvlak, geen vloeroppervlakte of volume. Adressen zijn geen huishoudens, " + "woningen, inwoners of bevolkingsmeting." + ) + source_metadata = { + "provider": "Digitaal Vlaanderen", + "theme": "buildings", + "layer_name": "Gebouwen- en Adressenregister", + "layer_type": "building_registry", + "authority_level": "authoritative", + "coverage_scope": "municipality", + "coverage_area": manifest["coverage_area"], + "feature_count": manifest["feature_count"], + "building_unit_count": manifest["linked_unit_count"], + "linked_address_count": manifest["linked_address_count"], + "address_match_method_counts": manifest["address_relations"]["match_method_counts"], + "unmatched_address_count": manifest["address_relations"]["unmatched_address_count"], + "ambiguous_address_count": manifest["address_relations"]["ambiguous_address_count"], + "grb_match_rate": manifest["grb_reconciliation"]["match_rate"], + "realized_grb_match_rate": manifest["grb_reconciliation"]["realized_match_rate"], + "geometry_clipped_to_area": True, + "identity_stable": True, + "semantic_metrics": False, + "privacy_profile": "aggregate_counts_only", + "attribution": ATTRIBUTION, + "catalog_url": CATALOG_URL, + "selection_aggregation": { + "metric_key": "building_footprint_area", + "method": "intersection_area", + "label": "Gebouwgrondoppervlakte", + "unit": "ha", + "geometry_dimension": 2, + "warning": warning, + }, + "selection_metrics": selection_metrics(), + } + provenance_metadata = { + "operator_tool": "provision_buildings_addresses_register.py", + "operator_explicit_fetch": True, + "geometry_clipped_to_area": True, + "source_urls": [BUILDING_ITEMS_URL, UNIT_ITEMS_URL, ADDRESS_ITEMS_URL], + "catalog_url": CATALOG_URL, + "building_catalog_url": BUILDING_CATALOG_URL, + "change_notice_url": CHANGE_NOTICE_URL, + "manifest_path": str(manifest_path), + "artifact_sha256": manifest["artifact_sha256"], + "raw_page_checksums": { + page["path"]: page["sha256"] + for collection in manifest["collections"] + for page in collection["pages"] + }, + "grb_reference": manifest["grb_reference"], + "grb_reconciliation": manifest["grb_reconciliation"], + "privacy_profile": manifest["privacy_profile"], + "reference_truncated": False, + "generated_at": manifest["generated_at"], + "limitations": manifest["limitations"], + } + with artifact_path.open("rb") as handle: + response = session.post( + f"{base_url}/api/v1/projects/{project_id}/datasets/upload", + data={ + "dataset_type": "vector", + "source": "operator_official_import", + "dataset_role": "reference", + "source_name": SOURCE_NAME, + "reference_layer_name": "building_registry", + "source_metadata_json": json.dumps(source_metadata, ensure_ascii=False), + "provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False), + "area_id": area_id, + "temporal_series_key": f"buildings-addresses-register:{area_id}", + "observed_at": observed_at(observed_date), + "temporal_granularity": "snapshot", + "source_version": observed_date.isoformat(), + }, + files={"file": (artifact_path.name, handle, "application/geo+json")}, + timeout=timeout, + ) + return response_data(response) + + +def main() -> int: + args = parse_args() + if ( + args.page_limit < 1 + or args.page_limit > 5000 + or min(args.max_buildings, args.max_units, args.max_addresses) < args.page_limit + ): + print(json.dumps({"status": "error", "message": "Invalid page or feature safety limits"}), file=sys.stderr) + return 2 + base_url = args.base_url.rstrip("/") + snapshot_dir = args.output_root / args.observed_date.isoformat() + try: + with requests.Session() as api_session: + project_id, area_id, boundary, datasets = locate_workspace( + api_session, + base_url, + args.project_name, + args.area_name, + args.api_timeout, + ) + with source_session() as official_session: + artifact_path, manifest_path, manifest = prepare_snapshot( + official_session, + boundary_wgs84=boundary, + datasets=datasets, + snapshot_dir=snapshot_dir, + area_name=args.area_name, + observed_date=args.observed_date, + page_limit=args.page_limit, + max_buildings=args.max_buildings, + max_units=args.max_units, + max_addresses=args.max_addresses, + timeout=args.request_timeout, + force=args.force, + ) + existing = next( + ( + item + for item in datasets + if item.get("source_name") == SOURCE_NAME + and item.get("source_version") == args.observed_date.isoformat() + and str(item.get("area_id") or "") == area_id + ), + None, + ) + if existing: + persisted_checksum = str(existing.get("checksum_sha256") or "") + if persisted_checksum and persisted_checksum != manifest["artifact_sha256"]: + raise RuntimeError( + "A different register snapshot is already persisted for this date and Area" + ) + persistence = { + "status": "existing", + "dataset_id": str(existing["id"]), + "feature_count": existing.get("feature_count") or manifest["feature_count"], + } + elif args.fetch_only: + persistence = {"status": "prepared", "dataset_id": None, "feature_count": manifest["feature_count"]} + else: + dataset = upload_snapshot( + api_session, + base_url=base_url, + project_id=project_id, + area_id=area_id, + artifact_path=artifact_path, + manifest_path=manifest_path, + manifest=manifest, + observed_date=args.observed_date, + timeout=args.api_timeout, + ) + persistence = { + "status": "created", + "dataset_id": str(dataset["id"]), + "feature_count": dataset.get("feature_count") or manifest["feature_count"], + } + except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc: + print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr) + return 1 + print( + json.dumps( + { + "status": "ok", + "mode": "fetch_only" if args.fetch_only else "provisioned", + "observed_at": args.observed_date.isoformat(), + "area": args.area_name, + "feature_count": manifest["feature_count"], + "building_area_ha": manifest["building_area_ha"], + "linked_unit_count": manifest["linked_unit_count"], + "linked_address_count": manifest["linked_address_count"], + "address_relations": manifest["address_relations"], + "grb_reconciliation": manifest["grb_reconciliation"], + "artifact_path": str(artifact_path), + "manifest_path": str(manifest_path), + "persistence": persistence, + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 32016db0..79f0fd5c 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -50,6 +50,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py ${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py +${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_timeseries.py ${PYTHON_BIN} -m py_compile scripts/geographic_scopes.py ${PYTHON_BIN} -m py_compile scripts/provision_geographic_scope.py