diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dd8fa00..12f7a305 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ - Added an explicit operator for the current official Digitaal Vlaanderen building, building-unit and address OGC collections with complete pagination, safety limits, retries and retained raw SHA256 evidence. +- Hardened completeness against the production address service's omitted + `next` link by continuing full pages with an explicit `startIndex` until a + short terminal page is observed. - Added exact EPSG:31370 Area clipping, EPSG:4326 building persistence and classified reconciliation against checksummed persisted GRB partitions. - Persisted only building lifecycle data and aggregate unit/address counts; diff --git a/backend/tests/test_sprint206_buildings_addresses_register.py b/backend/tests/test_sprint206_buildings_addresses_register.py index 18bad727..658c2575 100644 --- a/backend/tests/test_sprint206_buildings_addresses_register.py +++ b/backend/tests/test_sprint206_buildings_addresses_register.py @@ -111,21 +111,25 @@ class OfficialResponse: class TwoPageOfficialSession: def __init__(self): self.calls = 0 + self.params = [] def get(self, url, *, params, timeout): # noqa: ANN001, ARG002 self.calls += 1 + self.params.append(params) if self.calls == 1: payload = { "type": "FeatureCollection", "features": [building_feature("1", box(5.10, 51.20, 5.101, 51.201))], "links": [{"rel": "next", "href": f"{url}?startIndex=1"}], } - else: + elif self.calls == 2: payload = { "type": "FeatureCollection", "features": [building_feature("2", box(5.102, 51.20, 5.103, 51.201))], "links": [], } + else: + payload = {"type": "FeatureCollection", "features": [], "links": []} return OfficialResponse(payload, f"{url}?page={self.calls}") @@ -157,7 +161,9 @@ def test_official_collection_pagination_retains_checksummed_pages(tmp_path: Path ) assert [feature["properties"]["ObjectId"] for feature in features] == [1, 2] - assert summary["page_count"] == 2 + assert summary["page_count"] == 3 + assert summary["pagination_fallback_count"] == 1 + assert session.params[2]["startIndex"] == "2" assert all((tmp_path / page["path"]).is_file() for page in summary["pages"]) assert all(len(page["sha256"]) == 64 for page in summary["pages"]) @@ -351,6 +357,7 @@ def test_operator_is_canonical_packaged_and_mol_scoped_in_explorer() -> None: assert "COPY scripts/provision_buildings_addresses_register.py" in dockerfile assert "py_compile scripts/provision_buildings_addresses_register.py" in readiness assert "datasetCoversSelectedArea" in workspace + assert "digitaal_vlaanderen_buildings_addresses_register' ? 1_200_000" in workspace assert "Gebouwen- en Adressenregister" in catalog assert "building_registry: 'Gebouwenregister'" in display assert "digitaal_vlaanderen_buildings_addresses_register: 'Digitaal Vlaanderen'" in display diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 4a2c2daa..afe8fc32 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -335,6 +335,11 @@ reads the official Digitaal Vlaanderen OGC API Features collections GeoIntel therefore creates a dated snapshot rather than claiming an annual historical series. Raw response pages, request URLs and SHA256 checksums are retained as operator evidence. +The operator follows advertised `next` links and, when a full page omits that +link, continues with an explicit `startIndex`. This is required because the +production address collection currently stops advertising `next` at a +service-side result window even though later offsets remain available. A short +final page is required before the snapshot is considered complete. Building polygons are clipped against the exact persisted Area in EPSG:31370, transformed to EPSG:4326 and persisted through the normal Dataset diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index 8337a406..1bc0530b 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -143,7 +143,7 @@ function pickThemeDataset( (dataset.source_name === 'department_omgeving_land_use' ? 90_000 : 0) + (dataset.source_name === 'inbo_bwk_natura2000' ? 95_000 : 0) + (dataset.source_name === 'agentschap_landbouw_zeevisserij_agricultural_parcels' ? 98_000 : 0) + - (dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register' ? 120_000 : 0) + + (dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register' ? 1_200_000 : 0) + (dataset.dataset_role === 'reference' ? 10_000 : 0) + (dataset.observed_at ? new Date(dataset.observed_at).getTime() / 100_000_000 : 0) + (dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0) diff --git a/scripts/README.md b/scripts/README.md index 86c6cf92..556fbe72 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1499,9 +1499,14 @@ The operator requires a persisted Mol Area and the complete regional GRB buildings Dataset with valid manifest/partition checksums. It reads only the official `Gebouw`, `Gebouweenheid` and `Adres` OGC collections, clips in EPSG:31370 and retains every raw response page under the storage volume. +Advertised `next` links are followed; a full page without `next` is continued +with `startIndex` until a short final page proves completeness. `--force` refetches evidence; it cannot overwrite a conflicting snapshot for the same Area/date. Safety limits are configurable with `--page-limit`, `--max-buildings`, `--max-units` and `--max-addresses`. +The 150,000-building default is a bbox acquisition ceiling: exact Mol clipping +retains far fewer rows, but the rectangular source request also intersects +neighbouring municipalities. Only aggregate unit/address counts enter the queryable building layer. Review `address_relations`, `grb_reconciliation`, checksums and limitations in the diff --git a/scripts/provision_buildings_addresses_register.py b/scripts/provision_buildings_addresses_register.py index 090e82f1..958cf2de 100644 --- a/scripts/provision_buildings_addresses_register.py +++ b/scripts/provision_buildings_addresses_register.py @@ -60,7 +60,7 @@ DEFAULT_AREA_NAME = "Gemeente Mol - officiele grens" DEFAULT_API_URL = "http://127.0.0.1:8000" DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/buildings-addresses-register/mol") DEFAULT_PAGE_LIMIT = 1000 -DEFAULT_MAX_BUILDINGS = 100_000 +DEFAULT_MAX_BUILDINGS = 150_000 DEFAULT_MAX_UNITS = 200_000 DEFAULT_MAX_ADDRESSES = 200_000 SCHEMA_VERSION = 1 @@ -290,16 +290,19 @@ def fetch_collection( timeout: int, ) -> tuple[list[dict[str, Any]], dict[str, Any]]: request_url: str | None = url - params: dict[str, str] | None = { + base_params: dict[str, str] = { "f": "application/geo+json", "bbox": ",".join(f"{value:.8f}" for value in bbox), "limit": str(page_limit), } + params: dict[str, str] | None = dict(base_params) seen_urls: set[str] = set() seen_ids: set[str] = set() features: list[dict[str, Any]] = [] pages: list[dict[str, Any]] = [] duplicate_count = 0 + retrieved_count = 0 + pagination_fallback_count = 0 while request_url: request_key = requests.Request("GET", request_url, params=params).prepare().url or request_url if request_key in seen_urls: @@ -315,6 +318,7 @@ def fetch_collection( page_path = raw_dir / f"{name}_page_{len(pages) + 1:05d}.json" write_bytes_atomic(page_path, raw_bytes) page_features = list(payload.get("features") or []) + retrieved_count += len(page_features) pages.append( { "path": str(page_path.relative_to(raw_dir.parent)), @@ -338,6 +342,13 @@ def fetch_collection( f"{name} exceeds the {max_features} feature safety limit; refusing truncated output" ) request_url = next_page_url(payload) + if request_url is None and len(page_features) == page_limit: + # The production Address Register currently stops emitting `next` + # after a service-side window while higher startIndex values remain + # available. A full page is therefore not proof of completion. + request_url = url + params = {**base_params, "startIndex": str(retrieved_count)} + pagination_fallback_count += 1 if not features: raise RuntimeError(f"{name} returned no source features for the Area bbox") return features, { @@ -345,6 +356,7 @@ def fetch_collection( "page_count": len(pages), "bbox_feature_count": len(features), "duplicate_count": duplicate_count, + "pagination_fallback_count": pagination_fallback_count, "pages": pages, }