Complete live Mol register validation
This commit is contained in:
@@ -26,6 +26,9 @@
|
||||
matching Mol Area and retain regional GRB coverage everywhere else.
|
||||
- Added source-inventory presentation, runtime packaging and focused operator,
|
||||
privacy, reconciliation, metric and UI tests.
|
||||
- Live Mol validation retained 43,945 exact building polygons, 30,382 units
|
||||
and 21,388 unambiguous address relations. It found a 98.45% realized-building
|
||||
GRB match rate, zero invalid geometries and zero prohibited address fields.
|
||||
|
||||
## Sprint 205 Governed agricultural-use parcel history (2026-07-15)
|
||||
|
||||
|
||||
@@ -187,6 +187,13 @@ def test_buildings_are_clipped_in_lambert72_and_keep_lifecycle_status() -> None:
|
||||
assert record["area_ha"] > 0
|
||||
|
||||
|
||||
def test_area_evidence_paths_are_isolated_per_municipality() -> None:
|
||||
module = load_operator()
|
||||
|
||||
assert module.area_storage_key("Gemeente Mol - officiële grens") == "mol"
|
||||
assert module.area_storage_key("Gemeente Geel - officiële grens") == "geel"
|
||||
|
||||
|
||||
def test_official_unit_relation_and_exact_address_position_are_aggregated_without_labels() -> None:
|
||||
module = load_operator()
|
||||
_, boundary_lambert72, polygon, buildings, _ = normalized_fixture(module)
|
||||
|
||||
@@ -8613,3 +8613,74 @@ Next:
|
||||
- Implement P3 Buildings and Addresses Register through a governed snapshot
|
||||
operator, preserving register lifecycle semantics separately from GRB
|
||||
footprint geometry and excluding personal data.
|
||||
|
||||
## Sprint 206 - Governed Buildings and Addresses Register snapshot (2026-07-15)
|
||||
|
||||
Implemented:
|
||||
- Added `scripts/provision_buildings_addresses_register.py` for the official
|
||||
Digitaal Vlaanderen `Gebouw`, `Gebouweenheid` and `Adres` OGC API Features
|
||||
collections. It retains raw pages and SHA256 checksums, uses retries and
|
||||
limits, clips exactly in EPSG:31370 and persists only through DatasetService.
|
||||
- Added service-window-safe pagination. Live audit proved that the address
|
||||
collection omitted `next` after 52,000 bbox rows while offset 52,000 still
|
||||
returned data. The first incomplete snapshot was rejected and transactionally
|
||||
removed; the corrected operator continued with `startIndex` to a short final
|
||||
page at 52,110 rows.
|
||||
- Kept building lifecycle status separate from GRB footprint evidence. Official
|
||||
building-unit relations use `GebouwObjectId`; address relations are accepted
|
||||
only for an exact unambiguous unit position or unambiguous building
|
||||
containment. Ambiguous and unmatched relations remain diagnostics.
|
||||
- Persisted building polygons with aggregate relation counts only. Full address,
|
||||
street, house-number and box-number fields remain in restricted raw operator
|
||||
evidence and never enter queryable feature properties.
|
||||
- Added exact footprint, lifecycle, unit, address-status and GRB-match metrics.
|
||||
Fixed filtered `feature_count` execution without changing the existing
|
||||
unfiltered count contract.
|
||||
- Added Map and Sources presentation. The exact Mol Area prefers the register;
|
||||
another municipality and the full Kempen scope retain complete regional GRB.
|
||||
Evidence paths and manifest reuse are isolated by Area key and boundary hash.
|
||||
|
||||
Live evidence:
|
||||
- Source bbox acquisition completed with 111,696 buildings (112 pages), 67,239
|
||||
building units (68 pages) and 52,110 addresses (53 pages), with zero duplicate
|
||||
source ids. Exact Mol clipping retained 43,945 buildings and 23,788 addresses.
|
||||
- Dataset `c3a47680-cea1-406c-aebb-8ebfa8d7a359` is the only ready Mol register
|
||||
snapshot for 2026-07-15. It has one DatasetVersion, 43,945 vector rows,
|
||||
43,945 distinct source ids, SRID 4326, polygonal geometry only and zero invalid
|
||||
geometries.
|
||||
- Lifecycle counts are 37,159 realized, 14 under construction, 155 planned,
|
||||
22 not realized and 6,595 historical buildings. Exact footprint area is
|
||||
502.1434 ha.
|
||||
- Aggregate relations contain 30,382 building units, 21,388 linked addresses
|
||||
and 19,513 addresses in use. There are 1,687 ambiguous and 713 unmatched
|
||||
address positions. Queryable properties contain zero prohibited address
|
||||
label/number fields.
|
||||
- GRB reconciliation produced 36,588 matched, 5 review, 664 ambiguous and 6,688
|
||||
unmatched register buildings. There are 353 GRB footprints without a confirmed
|
||||
target and 329 duplicate-target groups. Realized-building match rate is
|
||||
98.4526%; total register match rate is lower because historical and other
|
||||
lifecycle records are retained.
|
||||
- Live exact-Area API returned all eleven governed metrics. Browser validation
|
||||
confirmed Mol register selection, Geel GRB fallback, readable metric/warning
|
||||
output, source inventory, MapLibre rendering, no horizontal overflow and no
|
||||
browser console warnings/errors.
|
||||
|
||||
Validation evidence:
|
||||
- Full local readiness passed backend compilation, 656 backend tests, one
|
||||
Alembic head, frontend TypeScript typecheck/build and shell syntax gates.
|
||||
- Tower deployment passed all-in-one health, PostGIS 3.6 migration/schema smoke,
|
||||
frontend proxy and icon verification.
|
||||
|
||||
Known limitations:
|
||||
- This is a current continuously updated snapshot, not a fabricated historical
|
||||
series. GRB remains the complete regional geometry source outside Mol.
|
||||
- Address counts are not households, dwellings, residents or population.
|
||||
Building footprint is ground area, not floor area, height or volume.
|
||||
- Ambiguous address links, duplicate GRB targets and review/unmatched geometry
|
||||
remain explicit evidence for later regional expansion; no nearest match is
|
||||
forced.
|
||||
|
||||
Next:
|
||||
- Implement P4 Digitaal Hoogtemodel Vlaanderen with a governed DTM/DSM product,
|
||||
vertical reference, bounded raster storage and measured elevation/slope
|
||||
statistics. Do not infer water depth or volume from terrain height alone.
|
||||
|
||||
@@ -136,7 +136,7 @@ checksum conflict with an existing annual Dataset fails closed.
|
||||
Buildings and Addresses Register snapshot evidence lives under:
|
||||
|
||||
```text
|
||||
storage/operator-evidence/buildings-addresses-register/mol/{observed-date}/
|
||||
storage/operator-evidence/buildings-addresses-register/{area-key}/{observed-date}/
|
||||
raw/
|
||||
buildings_page_*.json
|
||||
building_units_page_*.json
|
||||
@@ -153,6 +153,9 @@ counts and classified GRB reconciliation only. It enters PostGIS exclusively
|
||||
through DatasetService and ordinary `vector_features`; the operator never
|
||||
writes database rows directly. The manifest binds all raw pages, the normalized
|
||||
artifact, exact Area boundary and every GRB partition used for reconciliation.
|
||||
The Area key is deterministic (`mol`, `geel`, and so on). Reuse also requires
|
||||
the same Area label, observation date and boundary checksum, preventing one
|
||||
municipality's evidence from being imported under another Area.
|
||||
|
||||
Offline demo export artifacts can be inspected and cleaned with:
|
||||
|
||||
|
||||
+4
-4
@@ -53,10 +53,10 @@ pass live Mol validation before regional expansion.
|
||||
|
||||
### P3 - Buildings and Addresses Register
|
||||
|
||||
- [ ] Define a governed snapshot operator and relation mapping between building unit, building object, address and GRB geometry.
|
||||
- [ ] Keep register lifecycle/status semantics separate from GRB footprint geometry and document match confidence/unmatched rows.
|
||||
- [ ] Add current building-status/address metrics without exposing personal data or treating addresses as households/population.
|
||||
- [ ] Add Mol reconciliation tests and a live mismatch report before regional import.
|
||||
- [x] Define a governed snapshot operator and relation mapping between building unit, building object, address and GRB geometry.
|
||||
- [x] Keep register lifecycle/status semantics separate from GRB footprint geometry and document match confidence/unmatched rows.
|
||||
- [x] Add current building-status/address metrics without exposing personal data or treating addresses as households/population.
|
||||
- [x] Add Mol reconciliation tests and a live mismatch report before regional import.
|
||||
|
||||
### P4 - Digitaal Hoogtemodel Vlaanderen
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ SOURCE_NAME = "digitaal_vlaanderen_buildings_addresses_register"
|
||||
DEFAULT_PROJECT_NAME = "Kempen Regional Workbench"
|
||||
DEFAULT_AREA_NAME = "Gemeente Mol - officiele grens"
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/buildings-addresses-register/mol")
|
||||
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/buildings-addresses-register")
|
||||
DEFAULT_PAGE_LIMIT = 1000
|
||||
DEFAULT_MAX_BUILDINGS = 150_000
|
||||
DEFAULT_MAX_UNITS = 200_000
|
||||
@@ -247,6 +247,22 @@ def normalize_area_name(value: str) -> str:
|
||||
return unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii").casefold()
|
||||
|
||||
|
||||
def area_storage_key(value: str) -> str:
|
||||
normalized = normalize_area_name(value)
|
||||
if normalized.startswith("gemeente "):
|
||||
municipality = normalized.removeprefix("gemeente ").split(" - ", maxsplit=1)[0].strip()
|
||||
if municipality:
|
||||
return "-".join(municipality.split())
|
||||
key = "-".join(part for part in compact_key(value).split() if part)
|
||||
return key or "area"
|
||||
|
||||
|
||||
def boundary_checksum(boundary_wgs84) -> str:
|
||||
return sha256_bytes(
|
||||
json.dumps(mapping(boundary_wgs84), sort_keys=True).encode("utf-8")
|
||||
)
|
||||
|
||||
|
||||
def locate_workspace(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
@@ -763,7 +779,13 @@ def build_output_features(
|
||||
}
|
||||
|
||||
|
||||
def reusable_snapshot(snapshot_dir: Path) -> tuple[Path, Path, dict[str, Any]] | None:
|
||||
def reusable_snapshot(
|
||||
snapshot_dir: Path,
|
||||
*,
|
||||
area_name: str,
|
||||
observed_date: date,
|
||||
expected_boundary_checksum: str,
|
||||
) -> tuple[Path, Path, dict[str, Any]] | None:
|
||||
artifact_path = snapshot_dir / "buildings_addresses_register.geojson"
|
||||
manifest_path = snapshot_dir / "buildings_addresses_register.manifest.json"
|
||||
if not artifact_path.is_file() or not manifest_path.is_file():
|
||||
@@ -774,6 +796,10 @@ def reusable_snapshot(snapshot_dir: Path) -> tuple[Path, Path, dict[str, Any]] |
|
||||
return None
|
||||
if manifest.get("schema_version") != SCHEMA_VERSION:
|
||||
return None
|
||||
if manifest.get("coverage_area") != area_name or manifest.get("observed_at") != observed_date.isoformat():
|
||||
return None
|
||||
if manifest.get("boundary_sha256") != expected_boundary_checksum:
|
||||
return None
|
||||
if manifest.get("artifact_sha256") != sha256_file(artifact_path):
|
||||
return None
|
||||
for collection in manifest.get("collections") or []:
|
||||
@@ -804,8 +830,14 @@ def prepare_snapshot(
|
||||
timeout: int,
|
||||
force: bool,
|
||||
) -> tuple[Path, Path, dict[str, Any]]:
|
||||
expected_boundary_checksum = boundary_checksum(boundary_wgs84)
|
||||
if not force:
|
||||
reusable = reusable_snapshot(snapshot_dir)
|
||||
reusable = reusable_snapshot(
|
||||
snapshot_dir,
|
||||
area_name=area_name,
|
||||
observed_date=observed_date,
|
||||
expected_boundary_checksum=expected_boundary_checksum,
|
||||
)
|
||||
if reusable:
|
||||
return reusable
|
||||
snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -892,7 +924,7 @@ def prepare_snapshot(
|
||||
"crs_source": "EPSG:4326",
|
||||
"crs_clip": "EPSG:31370",
|
||||
"crs_persisted": "EPSG:4326",
|
||||
"boundary_sha256": sha256_bytes(json.dumps(mapping(boundary_wgs84), sort_keys=True).encode("utf-8")),
|
||||
"boundary_sha256": expected_boundary_checksum,
|
||||
"boundary_bbox_wgs84": list(boundary_wgs84.bounds),
|
||||
"collections": [buildings_source, units_source, addresses_source],
|
||||
"building_filter": building_filter,
|
||||
@@ -1113,7 +1145,7 @@ def main() -> int:
|
||||
print(json.dumps({"status": "error", "message": "Invalid page or feature safety limits"}), file=sys.stderr)
|
||||
return 2
|
||||
base_url = args.base_url.rstrip("/")
|
||||
snapshot_dir = args.output_root / args.observed_date.isoformat()
|
||||
snapshot_dir = args.output_root / area_storage_key(args.area_name) / args.observed_date.isoformat()
|
||||
try:
|
||||
with requests.Session() as api_session:
|
||||
project_id, area_id, boundary, datasets = locate_workspace(
|
||||
|
||||
Reference in New Issue
Block a user