Harden register pagination and Mol source selection

CHANGELOG.md

@@ -12,6 +12,9 @@
 - Added an explicit operator for the current official Digitaal Vlaanderen
   building, building-unit and address OGC collections with complete pagination,
   safety limits, retries and retained raw SHA256 evidence.
+- Hardened completeness against the production address service's omitted
+  `next` link by continuing full pages with an explicit `startIndex` until a
+  short terminal page is observed.
 - Added exact EPSG:31370 Area clipping, EPSG:4326 building persistence and
   classified reconciliation against checksummed persisted GRB partitions.
 - Persisted only building lifecycle data and aggregate unit/address counts;

backend/tests/test_sprint206_buildings_addresses_register.py

@@ -111,21 +111,25 @@ class OfficialResponse:
 class TwoPageOfficialSession:
     def __init__(self):
         self.calls = 0
+        self.params = []
 
     def get(self, url, *, params, timeout):  # noqa: ANN001, ARG002
         self.calls += 1
+        self.params.append(params)
         if self.calls == 1:
             payload = {
                 "type": "FeatureCollection",
                 "features": [building_feature("1", box(5.10, 51.20, 5.101, 51.201))],
                 "links": [{"rel": "next", "href": f"{url}?startIndex=1"}],
             }
-        else:
+        elif self.calls == 2:
             payload = {
                 "type": "FeatureCollection",
                 "features": [building_feature("2", box(5.102, 51.20, 5.103, 51.201))],
                 "links": [],
             }
+        else:
+            payload = {"type": "FeatureCollection", "features": [], "links": []}
         return OfficialResponse(payload, f"{url}?page={self.calls}")
 
 
@@ -157,7 +161,9 @@ def test_official_collection_pagination_retains_checksummed_pages(tmp_path: Path
     )
 
     assert [feature["properties"]["ObjectId"] for feature in features] == [1, 2]
-    assert summary["page_count"] == 2
+    assert summary["page_count"] == 3
+    assert summary["pagination_fallback_count"] == 1
+    assert session.params[2]["startIndex"] == "2"
     assert all((tmp_path / page["path"]).is_file() for page in summary["pages"])
     assert all(len(page["sha256"]) == 64 for page in summary["pages"])
 
@@ -351,6 +357,7 @@ def test_operator_is_canonical_packaged_and_mol_scoped_in_explorer() -> None:
     assert "COPY scripts/provision_buildings_addresses_register.py" in dockerfile
     assert "py_compile scripts/provision_buildings_addresses_register.py" in readiness
     assert "datasetCoversSelectedArea" in workspace
+    assert "digitaal_vlaanderen_buildings_addresses_register' ? 1_200_000" in workspace
     assert "Gebouwen- en Adressenregister" in catalog
     assert "building_registry: 'Gebouwenregister'" in display
     assert "digitaal_vlaanderen_buildings_addresses_register: 'Digitaal Vlaanderen'" in display

docs/DATA_SOURCES.md

@@ -335,6 +335,11 @@ reads the official Digitaal Vlaanderen OGC API Features collections
 GeoIntel therefore creates a dated snapshot rather than claiming an annual
 historical series. Raw response pages, request URLs and SHA256 checksums are
 retained as operator evidence.
+The operator follows advertised `next` links and, when a full page omits that
+link, continues with an explicit `startIndex`. This is required because the
+production address collection currently stops advertising `next` at a
+service-side result window even though later offsets remain available. A short
+final page is required before the snapshot is considered complete.
 
 Building polygons are clipped against the exact persisted Area in
 EPSG:31370, transformed to EPSG:4326 and persisted through the normal Dataset

frontend/src/components/map/MapWorkspace.tsx

@@ -143,7 +143,7 @@ function pickThemeDataset(
       (dataset.source_name === 'department_omgeving_land_use' ? 90_000 : 0) +
       (dataset.source_name === 'inbo_bwk_natura2000' ? 95_000 : 0) +
       (dataset.source_name === 'agentschap_landbouw_zeevisserij_agricultural_parcels' ? 98_000 : 0) +
-      (dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register' ? 120_000 : 0) +
+      (dataset.source_name === 'digitaal_vlaanderen_buildings_addresses_register' ? 1_200_000 : 0) +
       (dataset.dataset_role === 'reference' ? 10_000 : 0) +
       (dataset.observed_at ? new Date(dataset.observed_at).getTime() / 100_000_000 : 0) +
       (dataset.feature_count ?? dataset.vector_summary?.feature_count ?? 0)

scripts/README.md

@@ -1499,9 +1499,14 @@ The operator requires a persisted Mol Area and the complete regional GRB
 buildings Dataset with valid manifest/partition checksums. It reads only the
 official `Gebouw`, `Gebouweenheid` and `Adres` OGC collections, clips in
 EPSG:31370 and retains every raw response page under the storage volume.
+Advertised `next` links are followed; a full page without `next` is continued
+with `startIndex` until a short final page proves completeness.
 `--force` refetches evidence; it cannot overwrite a conflicting snapshot for
 the same Area/date. Safety limits are configurable with `--page-limit`,
 `--max-buildings`, `--max-units` and `--max-addresses`.
+The 150,000-building default is a bbox acquisition ceiling: exact Mol clipping
+retains far fewer rows, but the rectangular source request also intersects
+neighbouring municipalities.
 
 Only aggregate unit/address counts enter the queryable building layer. Review
 `address_relations`, `grb_reconciliation`, checksums and limitations in the

scripts/provision_buildings_addresses_register.py

@@ -60,7 +60,7 @@ DEFAULT_AREA_NAME = "Gemeente Mol - officiele grens"
 DEFAULT_API_URL = "http://127.0.0.1:8000"
 DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/buildings-addresses-register/mol")
 DEFAULT_PAGE_LIMIT = 1000
-DEFAULT_MAX_BUILDINGS = 100_000
+DEFAULT_MAX_BUILDINGS = 150_000
 DEFAULT_MAX_UNITS = 200_000
 DEFAULT_MAX_ADDRESSES = 200_000
 SCHEMA_VERSION = 1
@@ -290,16 +290,19 @@ def fetch_collection(
     timeout: int,
 ) -> tuple[list[dict[str, Any]], dict[str, Any]]:
     request_url: str | None = url
-    params: dict[str, str] | None = {
+    base_params: dict[str, str] = {
         "f": "application/geo+json",
         "bbox": ",".join(f"{value:.8f}" for value in bbox),
         "limit": str(page_limit),
     }
+    params: dict[str, str] | None = dict(base_params)
     seen_urls: set[str] = set()
     seen_ids: set[str] = set()
     features: list[dict[str, Any]] = []
     pages: list[dict[str, Any]] = []
     duplicate_count = 0
+    retrieved_count = 0
+    pagination_fallback_count = 0
     while request_url:
         request_key = requests.Request("GET", request_url, params=params).prepare().url or request_url
         if request_key in seen_urls:
@@ -315,6 +318,7 @@ def fetch_collection(
         page_path = raw_dir / f"{name}_page_{len(pages) + 1:05d}.json"
         write_bytes_atomic(page_path, raw_bytes)
         page_features = list(payload.get("features") or [])
+        retrieved_count += len(page_features)
         pages.append(
             {
                 "path": str(page_path.relative_to(raw_dir.parent)),
@@ -338,6 +342,13 @@ def fetch_collection(
                     f"{name} exceeds the {max_features} feature safety limit; refusing truncated output"
                 )
         request_url = next_page_url(payload)
+        if request_url is None and len(page_features) == page_limit:
+            # The production Address Register currently stops emitting `next`
+            # after a service-side window while higher startIndex values remain
+            # available. A full page is therefore not proof of completion.
+            request_url = url
+            params = {**base_params, "startIndex": str(retrieved_count)}
+            pagination_fallback_count += 1
     if not features:
         raise RuntimeError(f"{name} returned no source features for the Area bbox")
     return features, {
@@ -345,6 +356,7 @@ def fetch_collection(
         "page_count": len(pages),
         "bbox_feature_count": len(features),
         "duplicate_count": duplicate_count,
+        "pagination_fallback_count": pagination_fallback_count,
         "pages": pages,
     }