From 0e36d750c09cb8514c55e81808efadeb12bee05e Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 17:36:33 +0200 Subject: [PATCH] Complete live Mol register validation --- CHANGELOG.md | 3 + ..._sprint206_buildings_addresses_register.py | 7 ++ docs/CODEX_EXECUTION_LOG.md | 71 +++++++++++++++++++ docs/STORAGE_ARCHITECTURE.md | 5 +- docs/TODO.md | 8 +-- .../provision_buildings_addresses_register.py | 42 +++++++++-- 6 files changed, 126 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12f7a305..be5c3d58 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,9 @@ matching Mol Area and retain regional GRB coverage everywhere else. - Added source-inventory presentation, runtime packaging and focused operator, privacy, reconciliation, metric and UI tests. +- Live Mol validation retained 43,945 exact building polygons, 30,382 units + and 21,388 unambiguous address relations. It found a 98.45% realized-building + GRB match rate, zero invalid geometries and zero prohibited address fields. ## Sprint 205 Governed agricultural-use parcel history (2026-07-15) diff --git a/backend/tests/test_sprint206_buildings_addresses_register.py b/backend/tests/test_sprint206_buildings_addresses_register.py index a5403269..c288494f 100644 --- a/backend/tests/test_sprint206_buildings_addresses_register.py +++ b/backend/tests/test_sprint206_buildings_addresses_register.py @@ -187,6 +187,13 @@ def test_buildings_are_clipped_in_lambert72_and_keep_lifecycle_status() -> None: assert record["area_ha"] > 0 +def test_area_evidence_paths_are_isolated_per_municipality() -> None: + module = load_operator() + + assert module.area_storage_key("Gemeente Mol - officiële grens") == "mol" + assert module.area_storage_key("Gemeente Geel - officiële grens") == "geel" + + def test_official_unit_relation_and_exact_address_position_are_aggregated_without_labels() -> None: module = load_operator() _, boundary_lambert72, polygon, buildings, _ = normalized_fixture(module) diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index fa4c7080..2fb51ccc 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -8613,3 +8613,74 @@ Next: - Implement P3 Buildings and Addresses Register through a governed snapshot operator, preserving register lifecycle semantics separately from GRB footprint geometry and excluding personal data. + +## Sprint 206 - Governed Buildings and Addresses Register snapshot (2026-07-15) + +Implemented: +- Added `scripts/provision_buildings_addresses_register.py` for the official + Digitaal Vlaanderen `Gebouw`, `Gebouweenheid` and `Adres` OGC API Features + collections. It retains raw pages and SHA256 checksums, uses retries and + limits, clips exactly in EPSG:31370 and persists only through DatasetService. +- Added service-window-safe pagination. Live audit proved that the address + collection omitted `next` after 52,000 bbox rows while offset 52,000 still + returned data. The first incomplete snapshot was rejected and transactionally + removed; the corrected operator continued with `startIndex` to a short final + page at 52,110 rows. +- Kept building lifecycle status separate from GRB footprint evidence. Official + building-unit relations use `GebouwObjectId`; address relations are accepted + only for an exact unambiguous unit position or unambiguous building + containment. Ambiguous and unmatched relations remain diagnostics. +- Persisted building polygons with aggregate relation counts only. Full address, + street, house-number and box-number fields remain in restricted raw operator + evidence and never enter queryable feature properties. +- Added exact footprint, lifecycle, unit, address-status and GRB-match metrics. + Fixed filtered `feature_count` execution without changing the existing + unfiltered count contract. +- Added Map and Sources presentation. The exact Mol Area prefers the register; + another municipality and the full Kempen scope retain complete regional GRB. + Evidence paths and manifest reuse are isolated by Area key and boundary hash. + +Live evidence: +- Source bbox acquisition completed with 111,696 buildings (112 pages), 67,239 + building units (68 pages) and 52,110 addresses (53 pages), with zero duplicate + source ids. Exact Mol clipping retained 43,945 buildings and 23,788 addresses. +- Dataset `c3a47680-cea1-406c-aebb-8ebfa8d7a359` is the only ready Mol register + snapshot for 2026-07-15. It has one DatasetVersion, 43,945 vector rows, + 43,945 distinct source ids, SRID 4326, polygonal geometry only and zero invalid + geometries. +- Lifecycle counts are 37,159 realized, 14 under construction, 155 planned, + 22 not realized and 6,595 historical buildings. Exact footprint area is + 502.1434 ha. +- Aggregate relations contain 30,382 building units, 21,388 linked addresses + and 19,513 addresses in use. There are 1,687 ambiguous and 713 unmatched + address positions. Queryable properties contain zero prohibited address + label/number fields. +- GRB reconciliation produced 36,588 matched, 5 review, 664 ambiguous and 6,688 + unmatched register buildings. There are 353 GRB footprints without a confirmed + target and 329 duplicate-target groups. Realized-building match rate is + 98.4526%; total register match rate is lower because historical and other + lifecycle records are retained. +- Live exact-Area API returned all eleven governed metrics. Browser validation + confirmed Mol register selection, Geel GRB fallback, readable metric/warning + output, source inventory, MapLibre rendering, no horizontal overflow and no + browser console warnings/errors. + +Validation evidence: +- Full local readiness passed backend compilation, 656 backend tests, one + Alembic head, frontend TypeScript typecheck/build and shell syntax gates. +- Tower deployment passed all-in-one health, PostGIS 3.6 migration/schema smoke, + frontend proxy and icon verification. + +Known limitations: +- This is a current continuously updated snapshot, not a fabricated historical + series. GRB remains the complete regional geometry source outside Mol. +- Address counts are not households, dwellings, residents or population. + Building footprint is ground area, not floor area, height or volume. +- Ambiguous address links, duplicate GRB targets and review/unmatched geometry + remain explicit evidence for later regional expansion; no nearest match is + forced. + +Next: +- Implement P4 Digitaal Hoogtemodel Vlaanderen with a governed DTM/DSM product, + vertical reference, bounded raster storage and measured elevation/slope + statistics. Do not infer water depth or volume from terrain height alone. diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 4888e45e..5504c2b2 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -136,7 +136,7 @@ checksum conflict with an existing annual Dataset fails closed. Buildings and Addresses Register snapshot evidence lives under: ```text -storage/operator-evidence/buildings-addresses-register/mol/{observed-date}/ +storage/operator-evidence/buildings-addresses-register/{area-key}/{observed-date}/ raw/ buildings_page_*.json building_units_page_*.json @@ -153,6 +153,9 @@ counts and classified GRB reconciliation only. It enters PostGIS exclusively through DatasetService and ordinary `vector_features`; the operator never writes database rows directly. The manifest binds all raw pages, the normalized artifact, exact Area boundary and every GRB partition used for reconciliation. +The Area key is deterministic (`mol`, `geel`, and so on). Reuse also requires +the same Area label, observation date and boundary checksum, preventing one +municipality's evidence from being imported under another Area. Offline demo export artifacts can be inspected and cleaned with: diff --git a/docs/TODO.md b/docs/TODO.md index 4372dda6..3385175e 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -53,10 +53,10 @@ pass live Mol validation before regional expansion. ### P3 - Buildings and Addresses Register -- [ ] Define a governed snapshot operator and relation mapping between building unit, building object, address and GRB geometry. -- [ ] Keep register lifecycle/status semantics separate from GRB footprint geometry and document match confidence/unmatched rows. -- [ ] Add current building-status/address metrics without exposing personal data or treating addresses as households/population. -- [ ] Add Mol reconciliation tests and a live mismatch report before regional import. +- [x] Define a governed snapshot operator and relation mapping between building unit, building object, address and GRB geometry. +- [x] Keep register lifecycle/status semantics separate from GRB footprint geometry and document match confidence/unmatched rows. +- [x] Add current building-status/address metrics without exposing personal data or treating addresses as households/population. +- [x] Add Mol reconciliation tests and a live mismatch report before regional import. ### P4 - Digitaal Hoogtemodel Vlaanderen diff --git a/scripts/provision_buildings_addresses_register.py b/scripts/provision_buildings_addresses_register.py index 958cf2de..cfbeaa42 100644 --- a/scripts/provision_buildings_addresses_register.py +++ b/scripts/provision_buildings_addresses_register.py @@ -58,7 +58,7 @@ SOURCE_NAME = "digitaal_vlaanderen_buildings_addresses_register" DEFAULT_PROJECT_NAME = "Kempen Regional Workbench" DEFAULT_AREA_NAME = "Gemeente Mol - officiele grens" DEFAULT_API_URL = "http://127.0.0.1:8000" -DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/buildings-addresses-register/mol") +DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/buildings-addresses-register") DEFAULT_PAGE_LIMIT = 1000 DEFAULT_MAX_BUILDINGS = 150_000 DEFAULT_MAX_UNITS = 200_000 @@ -247,6 +247,22 @@ def normalize_area_name(value: str) -> str: return unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii").casefold() +def area_storage_key(value: str) -> str: + normalized = normalize_area_name(value) + if normalized.startswith("gemeente "): + municipality = normalized.removeprefix("gemeente ").split(" - ", maxsplit=1)[0].strip() + if municipality: + return "-".join(municipality.split()) + key = "-".join(part for part in compact_key(value).split() if part) + return key or "area" + + +def boundary_checksum(boundary_wgs84) -> str: + return sha256_bytes( + json.dumps(mapping(boundary_wgs84), sort_keys=True).encode("utf-8") + ) + + def locate_workspace( session: requests.Session, base_url: str, @@ -763,7 +779,13 @@ def build_output_features( } -def reusable_snapshot(snapshot_dir: Path) -> tuple[Path, Path, dict[str, Any]] | None: +def reusable_snapshot( + snapshot_dir: Path, + *, + area_name: str, + observed_date: date, + expected_boundary_checksum: str, +) -> tuple[Path, Path, dict[str, Any]] | None: artifact_path = snapshot_dir / "buildings_addresses_register.geojson" manifest_path = snapshot_dir / "buildings_addresses_register.manifest.json" if not artifact_path.is_file() or not manifest_path.is_file(): @@ -774,6 +796,10 @@ def reusable_snapshot(snapshot_dir: Path) -> tuple[Path, Path, dict[str, Any]] | return None if manifest.get("schema_version") != SCHEMA_VERSION: return None + if manifest.get("coverage_area") != area_name or manifest.get("observed_at") != observed_date.isoformat(): + return None + if manifest.get("boundary_sha256") != expected_boundary_checksum: + return None if manifest.get("artifact_sha256") != sha256_file(artifact_path): return None for collection in manifest.get("collections") or []: @@ -804,8 +830,14 @@ def prepare_snapshot( timeout: int, force: bool, ) -> tuple[Path, Path, dict[str, Any]]: + expected_boundary_checksum = boundary_checksum(boundary_wgs84) if not force: - reusable = reusable_snapshot(snapshot_dir) + reusable = reusable_snapshot( + snapshot_dir, + area_name=area_name, + observed_date=observed_date, + expected_boundary_checksum=expected_boundary_checksum, + ) if reusable: return reusable snapshot_dir.mkdir(parents=True, exist_ok=True) @@ -892,7 +924,7 @@ def prepare_snapshot( "crs_source": "EPSG:4326", "crs_clip": "EPSG:31370", "crs_persisted": "EPSG:4326", - "boundary_sha256": sha256_bytes(json.dumps(mapping(boundary_wgs84), sort_keys=True).encode("utf-8")), + "boundary_sha256": expected_boundary_checksum, "boundary_bbox_wgs84": list(boundary_wgs84.bounds), "collections": [buildings_source, units_source, addresses_source], "building_filter": building_filter, @@ -1113,7 +1145,7 @@ def main() -> int: print(json.dumps({"status": "error", "message": "Invalid page or feature safety limits"}), file=sys.stderr) return 2 base_url = args.base_url.rstrip("/") - snapshot_dir = args.output_root / args.observed_date.isoformat() + snapshot_dir = args.output_root / area_storage_key(args.area_name) / args.observed_date.isoformat() try: with requests.Session() as api_session: project_id, area_id, boundary, datasets = locate_workspace(