Harden register pagination and Mol source selection

scripts/provision_buildings_addresses_register.py

@@ -60,7 +60,7 @@ DEFAULT_AREA_NAME = "Gemeente Mol - officiele grens"
 DEFAULT_API_URL = "http://127.0.0.1:8000"
 DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-evidence/buildings-addresses-register/mol")
 DEFAULT_PAGE_LIMIT = 1000
-DEFAULT_MAX_BUILDINGS = 100_000
+DEFAULT_MAX_BUILDINGS = 150_000
 DEFAULT_MAX_UNITS = 200_000
 DEFAULT_MAX_ADDRESSES = 200_000
 SCHEMA_VERSION = 1
@@ -290,16 +290,19 @@ def fetch_collection(
     timeout: int,
 ) -> tuple[list[dict[str, Any]], dict[str, Any]]:
     request_url: str | None = url
-    params: dict[str, str] | None = {
+    base_params: dict[str, str] = {
         "f": "application/geo+json",
         "bbox": ",".join(f"{value:.8f}" for value in bbox),
         "limit": str(page_limit),
     }
+    params: dict[str, str] | None = dict(base_params)
     seen_urls: set[str] = set()
     seen_ids: set[str] = set()
     features: list[dict[str, Any]] = []
     pages: list[dict[str, Any]] = []
     duplicate_count = 0
+    retrieved_count = 0
+    pagination_fallback_count = 0
     while request_url:
         request_key = requests.Request("GET", request_url, params=params).prepare().url or request_url
         if request_key in seen_urls:
@@ -315,6 +318,7 @@ def fetch_collection(
         page_path = raw_dir / f"{name}_page_{len(pages) + 1:05d}.json"
         write_bytes_atomic(page_path, raw_bytes)
         page_features = list(payload.get("features") or [])
+        retrieved_count += len(page_features)
         pages.append(
             {
                 "path": str(page_path.relative_to(raw_dir.parent)),
@@ -338,6 +342,13 @@ def fetch_collection(
                     f"{name} exceeds the {max_features} feature safety limit; refusing truncated output"
                 )
         request_url = next_page_url(payload)
+        if request_url is None and len(page_features) == page_limit:
+            # The production Address Register currently stops emitting `next`
+            # after a service-side window while higher startIndex values remain
+            # available. A full page is therefore not proof of completion.
+            request_url = url
+            params = {**base_params, "startIndex": str(retrieved_count)}
+            pagination_fallback_count += 1
     if not features:
         raise RuntimeError(f"{name} returned no source features for the Area bbox")
     return features, {
@@ -345,6 +356,7 @@ def fetch_collection(
         "page_count": len(pages),
         "bbox_feature_count": len(features),
         "duplicate_count": duplicate_count,
+        "pagination_fallback_count": pagination_fallback_count,
         "pages": pages,
     }