Complete live Mol register validation
This commit is contained in:
@@ -26,6 +26,9 @@
|
|||||||
matching Mol Area and retain regional GRB coverage everywhere else.
|
matching Mol Area and retain regional GRB coverage everywhere else.
|
||||||
- Added source-inventory presentation, runtime packaging and focused operator,
|
- Added source-inventory presentation, runtime packaging and focused operator,
|
||||||
privacy, reconciliation, metric and UI tests.
|
privacy, reconciliation, metric and UI tests.
|
||||||
|
- Live Mol validation retained 43,945 exact building polygons, 30,382 units
|
||||||
|
and 21,388 unambiguous address relations. It found a 98.45% realized-building
|
||||||
|
GRB match rate, zero invalid geometries and zero prohibited address fields.
|
||||||
|
|
||||||
## Sprint 205 Governed agricultural-use parcel history (2026-07-15)
|
## Sprint 205 Governed agricultural-use parcel history (2026-07-15)
|
||||||
|
|
||||||
|
|||||||
@@ -187,6 +187,13 @@ def test_buildings_are_clipped_in_lambert72_and_keep_lifecycle_status() -> None:
|
|||||||
assert record["area_ha"] > 0
|
assert record["area_ha"] > 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_area_evidence_paths_are_isolated_per_municipality() -> None:
|
||||||
|
module = load_operator()
|
||||||
|
|
||||||
|
assert module.area_storage_key("Gemeente Mol - officiële grens") == "mol"
|
||||||
|
assert module.area_storage_key("Gemeente Geel - officiële grens") == "geel"
|
||||||
|
|
||||||
|
|
||||||
def test_official_unit_relation_and_exact_address_position_are_aggregated_without_labels() -> None:
|
def test_official_unit_relation_and_exact_address_position_are_aggregated_without_labels() -> None:
|
||||||
module = load_operator()
|
module = load_operator()
|
||||||
_, boundary_lambert72, polygon, buildings, _ = normalized_fixture(module)
|
_, boundary_lambert72, polygon, buildings, _ = normalized_fixture(module)
|
||||||
|
|||||||
@@ -8613,3 +8613,74 @@ Next:
|
|||||||
- Implement P3 Buildings and Addresses Register through a governed snapshot
|
- Implement P3 Buildings and Addresses Register through a governed snapshot
|
||||||
operator, preserving register lifecycle semantics separately from GRB
|
operator, preserving register lifecycle semantics separately from GRB
|
||||||
footprint geometry and excluding personal data.
|
footprint geometry and excluding personal data.
|
||||||
|
|
||||||
|
## Sprint 206 - Governed Buildings and Addresses Register snapshot (2026-07-15)
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
- Added `scripts/provision_buildings_addresses_register.py` for the official
|
||||||
|
Digitaal Vlaanderen `Gebouw`, `Gebouweenheid` and `Adres` OGC API Features
|
||||||
|
collections. It retains raw pages and SHA256 checksums, uses retries and
|
||||||
|
limits, clips exactly in EPSG:31370 and persists only through DatasetService.
|
||||||
|
- Added service-window-safe pagination. Live audit proved that the address
|
||||||
|
collection omitted `next` after 52,000 bbox rows while offset 52,000 still
|
||||||
|
returned data. The first incomplete snapshot was rejected and transactionally
|
||||||
|
removed; the corrected operator continued with `startIndex` to a short final
|
||||||
|
page at 52,110 rows.
|
||||||
|
- Kept building lifecycle status separate from GRB footprint evidence. Official
|
||||||
|
building-unit relations use `GebouwObjectId`; address relations are accepted
|
||||||
|
only for an exact unambiguous unit position or unambiguous building
|
||||||
|
containment. Ambiguous and unmatched relations remain diagnostics.
|
||||||
|
- Persisted building polygons with aggregate relation counts only. Full address,
|
||||||
|
street, house-number and box-number fields remain in restricted raw operator
|
||||||
|
evidence and never enter queryable feature properties.
|
||||||
|
- Added exact footprint, lifecycle, unit, address-status and GRB-match metrics.
|
||||||
|
Fixed filtered `feature_count` execution without changing the existing
|
||||||
|
unfiltered count contract.
|
||||||
|
- Added Map and Sources presentation. The exact Mol Area prefers the register;
|
||||||
|
another municipality and the full Kempen scope retain complete regional GRB.
|
||||||
|
Evidence paths and manifest reuse are isolated by Area key and boundary hash.
|
||||||
|
|
||||||
|
Live evidence:
|
||||||
|
- Source bbox acquisition completed with 111,696 buildings (112 pages), 67,239
|
||||||
|
building units (68 pages) and 52,110 addresses (53 pages), with zero duplicate
|
||||||
|
source ids. Exact Mol clipping retained 43,945 buildings and 23,788 addresses.
|
||||||
|
- Dataset `c3a47680-cea1-406c-aebb-8ebfa8d7a359` is the only ready Mol register
|
||||||
|
snapshot for 2026-07-15. It has one DatasetVersion, 43,945 vector rows,
|
||||||
|
43,945 distinct source ids, SRID 4326, polygonal geometry only and zero invalid
|
||||||
|
geometries.
|
||||||
|
- Lifecycle counts are 37,159 realized, 14 under construction, 155 planned,
|
||||||
|
22 not realized and 6,595 historical buildings. Exact footprint area is
|
||||||
|
502.1434 ha.
|
||||||
|
- Aggregate relations contain 30,382 building units, 21,388 linked addresses
|
||||||
|
and 19,513 addresses in use. There are 1,687 ambiguous and 713 unmatched
|
||||||
|
address positions. Queryable properties contain zero prohibited address
|
||||||
|
label/number fields.
|
||||||
|
- GRB reconciliation produced 36,588 matched, 5 review, 664 ambiguous and 6,688
|
||||||
|
unmatched register buildings. There are 353 GRB footprints without a confirmed
|
||||||
|
target and 329 duplicate-target groups. Realized-building match rate is
|
||||||
|
98.4526%; total register match rate is lower because historical and other
|
||||||
|
lifecycle records are retained.
|
||||||
|
- Live exact-Area API returned all eleven governed metrics. Browser validation
|
||||||
|
confirmed Mol register selection, Geel GRB fallback, readable metric/warning
|
||||||
|
output, source inventory, MapLibre rendering, no horizontal overflow and no
|
||||||
|
browser console warnings/errors.
|
||||||
|
|
||||||
|
Validation evidence:
|
||||||
|
- Full local readiness passed backend compilation, 656 backend tests, one
|
||||||
|
Alembic head, frontend TypeScript typecheck/build and shell syntax gates.
|
||||||
|
- Tower deployment passed all-in-one health, PostGIS 3.6 migration/schema smoke,
|
||||||
|
frontend proxy and icon verification.
|
||||||
|
|
||||||
|
Known limitations:
|
||||||
|
- This is a current continuously updated snapshot, not a fabricated historical
|
||||||
|
series. GRB remains the complete regional geometry source outside Mol.
|
||||||
|
- Address counts are not households, dwellings, residents or population.
|
||||||
|
Building footprint is ground area, not floor area, height or volume.
|
||||||
|
- Ambiguous address links, duplicate GRB targets and review/unmatched geometry
|
||||||
|
remain explicit evidence for later regional expansion; no nearest match is
|
||||||
|
forced.
|
||||||
|
|
||||||
|
Next:
|
||||||
|
- Implement P4 Digitaal Hoogtemodel Vlaanderen with a governed DTM/DSM product,
|
||||||
|
vertical reference, bounded raster storage and measured elevation/slope
|
||||||
|
statistics. Do not infer water depth or volume from terrain height alone.
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ checksum conflict with an existing annual Dataset fails closed.
|
|||||||
Buildings and Addresses Register snapshot evidence lives under:
|
Buildings and Addresses Register snapshot evidence lives under:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
storage/operator-evidence/buildings-addresses-register/mol/{observed-date}/
|
storage/operator-evidence/buildings-addresses-register/{area-key}/{observed-date}/
|
||||||
raw/
|
raw/
|
||||||
buildings_page_*.json
|
buildings_page_*.json
|
||||||
building_units_page_*.json
|
building_units_page_*.json
|
||||||
@@ -153,6 +153,9 @@ counts and classified GRB reconciliation only. It enters PostGIS exclusively
|
|||||||
through DatasetService and ordinary `vector_features`; the operator never
|
through DatasetService and ordinary `vector_features`; the operator never
|
||||||
writes database rows directly. The manifest binds all raw pages, the normalized
|
writes database rows directly. The manifest binds all raw pages, the normalized
|
||||||
artifact, exact Area boundary and every GRB partition used for reconciliation.
|
artifact, exact Area boundary and every GRB partition used for reconciliation.
|
||||||
|
The Area key is deterministic (`mol`, `geel`, and so on). Reuse also requires
|
||||||
|
the same Area label, observation date and boundary checksum, preventing one
|
||||||
|
municipality's evidence from being imported under another Area.
|
||||||
|
|
||||||
Offline demo export artifacts can be inspected and cleaned with:
|
Offline demo export artifacts can be inspected and cleaned with:
|
||||||
|
|
||||||
|
|||||||
+4
-4
@@ -53,10 +53,10 @@ pass live Mol validation before regional expansion.
|
|||||||
|
|
||||||
### P3 - Buildings and Addresses Register
|
### P3 - Buildings and Addresses Register
|
||||||
|
|
||||||
- [ ] Define a governed snapshot operator and relation mapping between building unit, building object, address and GRB geometry.
|
- [x] Define a governed snapshot operator and relation mapping between building unit, building object, address and GRB geometry.
|
||||||
- [ ] Keep register lifecycle/status semantics separate from GRB footprint geometry and document match confidence/unmatched rows.
|
- [x] Keep register lifecycle/status semantics separate from GRB footprint geometry and document match confidence/unmatched rows.
|
||||||
- [ ] Add current building-status/address metrics without exposing personal data or treating addresses as households/population.
|
- [x] Add current building-status/address metrics without exposing personal data or treating addresses as households/population.
|
||||||
- [ ] Add Mol reconciliation tests and a live mismatch report before regional import.
|
- [x] Add Mol reconciliation tests and a live mismatch report before regional import.
|
||||||
|
|
||||||
### P4 - Digitaal Hoogtemodel Vlaanderen
|
### P4 - Digitaal Hoogtemodel Vlaanderen
|
||||||
|
|
||||||
|
|||||||
@@ -58,7 +58,7 @@ SOURCE_NAME = "digitaal_vlaanderen_buildings_addresses_register"
|
|||||||
DEFAULT_PROJECT_NAME = "Kempen Regional Workbench"
|
DEFAULT_PROJECT_NAME = "Kempen Regional Workbench"
|
||||||
DEFAULT_AREA_NAME = "Gemeente Mol - officiele grens"
|
DEFAULT_AREA_NAME = "Gemeente Mol - officiele grens"
|
||||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||||
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/buildings-addresses-register/mol")
|
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/buildings-addresses-register")
|
||||||
DEFAULT_PAGE_LIMIT = 1000
|
DEFAULT_PAGE_LIMIT = 1000
|
||||||
DEFAULT_MAX_BUILDINGS = 150_000
|
DEFAULT_MAX_BUILDINGS = 150_000
|
||||||
DEFAULT_MAX_UNITS = 200_000
|
DEFAULT_MAX_UNITS = 200_000
|
||||||
@@ -247,6 +247,22 @@ def normalize_area_name(value: str) -> str:
|
|||||||
return unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii").casefold()
|
return unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii").casefold()
|
||||||
|
|
||||||
|
|
||||||
|
def area_storage_key(value: str) -> str:
|
||||||
|
normalized = normalize_area_name(value)
|
||||||
|
if normalized.startswith("gemeente "):
|
||||||
|
municipality = normalized.removeprefix("gemeente ").split(" - ", maxsplit=1)[0].strip()
|
||||||
|
if municipality:
|
||||||
|
return "-".join(municipality.split())
|
||||||
|
key = "-".join(part for part in compact_key(value).split() if part)
|
||||||
|
return key or "area"
|
||||||
|
|
||||||
|
|
||||||
|
def boundary_checksum(boundary_wgs84) -> str:
|
||||||
|
return sha256_bytes(
|
||||||
|
json.dumps(mapping(boundary_wgs84), sort_keys=True).encode("utf-8")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def locate_workspace(
|
def locate_workspace(
|
||||||
session: requests.Session,
|
session: requests.Session,
|
||||||
base_url: str,
|
base_url: str,
|
||||||
@@ -763,7 +779,13 @@ def build_output_features(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def reusable_snapshot(snapshot_dir: Path) -> tuple[Path, Path, dict[str, Any]] | None:
|
def reusable_snapshot(
|
||||||
|
snapshot_dir: Path,
|
||||||
|
*,
|
||||||
|
area_name: str,
|
||||||
|
observed_date: date,
|
||||||
|
expected_boundary_checksum: str,
|
||||||
|
) -> tuple[Path, Path, dict[str, Any]] | None:
|
||||||
artifact_path = snapshot_dir / "buildings_addresses_register.geojson"
|
artifact_path = snapshot_dir / "buildings_addresses_register.geojson"
|
||||||
manifest_path = snapshot_dir / "buildings_addresses_register.manifest.json"
|
manifest_path = snapshot_dir / "buildings_addresses_register.manifest.json"
|
||||||
if not artifact_path.is_file() or not manifest_path.is_file():
|
if not artifact_path.is_file() or not manifest_path.is_file():
|
||||||
@@ -774,6 +796,10 @@ def reusable_snapshot(snapshot_dir: Path) -> tuple[Path, Path, dict[str, Any]] |
|
|||||||
return None
|
return None
|
||||||
if manifest.get("schema_version") != SCHEMA_VERSION:
|
if manifest.get("schema_version") != SCHEMA_VERSION:
|
||||||
return None
|
return None
|
||||||
|
if manifest.get("coverage_area") != area_name or manifest.get("observed_at") != observed_date.isoformat():
|
||||||
|
return None
|
||||||
|
if manifest.get("boundary_sha256") != expected_boundary_checksum:
|
||||||
|
return None
|
||||||
if manifest.get("artifact_sha256") != sha256_file(artifact_path):
|
if manifest.get("artifact_sha256") != sha256_file(artifact_path):
|
||||||
return None
|
return None
|
||||||
for collection in manifest.get("collections") or []:
|
for collection in manifest.get("collections") or []:
|
||||||
@@ -804,8 +830,14 @@ def prepare_snapshot(
|
|||||||
timeout: int,
|
timeout: int,
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> tuple[Path, Path, dict[str, Any]]:
|
) -> tuple[Path, Path, dict[str, Any]]:
|
||||||
|
expected_boundary_checksum = boundary_checksum(boundary_wgs84)
|
||||||
if not force:
|
if not force:
|
||||||
reusable = reusable_snapshot(snapshot_dir)
|
reusable = reusable_snapshot(
|
||||||
|
snapshot_dir,
|
||||||
|
area_name=area_name,
|
||||||
|
observed_date=observed_date,
|
||||||
|
expected_boundary_checksum=expected_boundary_checksum,
|
||||||
|
)
|
||||||
if reusable:
|
if reusable:
|
||||||
return reusable
|
return reusable
|
||||||
snapshot_dir.mkdir(parents=True, exist_ok=True)
|
snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||||
@@ -892,7 +924,7 @@ def prepare_snapshot(
|
|||||||
"crs_source": "EPSG:4326",
|
"crs_source": "EPSG:4326",
|
||||||
"crs_clip": "EPSG:31370",
|
"crs_clip": "EPSG:31370",
|
||||||
"crs_persisted": "EPSG:4326",
|
"crs_persisted": "EPSG:4326",
|
||||||
"boundary_sha256": sha256_bytes(json.dumps(mapping(boundary_wgs84), sort_keys=True).encode("utf-8")),
|
"boundary_sha256": expected_boundary_checksum,
|
||||||
"boundary_bbox_wgs84": list(boundary_wgs84.bounds),
|
"boundary_bbox_wgs84": list(boundary_wgs84.bounds),
|
||||||
"collections": [buildings_source, units_source, addresses_source],
|
"collections": [buildings_source, units_source, addresses_source],
|
||||||
"building_filter": building_filter,
|
"building_filter": building_filter,
|
||||||
@@ -1113,7 +1145,7 @@ def main() -> int:
|
|||||||
print(json.dumps({"status": "error", "message": "Invalid page or feature safety limits"}), file=sys.stderr)
|
print(json.dumps({"status": "error", "message": "Invalid page or feature safety limits"}), file=sys.stderr)
|
||||||
return 2
|
return 2
|
||||||
base_url = args.base_url.rstrip("/")
|
base_url = args.base_url.rstrip("/")
|
||||||
snapshot_dir = args.output_root / args.observed_date.isoformat()
|
snapshot_dir = args.output_root / area_storage_key(args.area_name) / args.observed_date.isoformat()
|
||||||
try:
|
try:
|
||||||
with requests.Session() as api_session:
|
with requests.Session() as api_session:
|
||||||
project_id, area_id, boundary, datasets = locate_workspace(
|
project_id, area_id, boundary, datasets = locate_workspace(
|
||||||
|
|||||||
Reference in New Issue
Block a user