Page GRB references for operator samples

2026-07-09 15:02:12 +02:00
parent 77c45937c6
commit a63d4eaadc
8 changed files with 254 additions and 12 deletions
@@ -20,6 +20,8 @@ from typing import Any
 WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms"
 GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
 DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data")
+DEFAULT_GRB_PAGE_LIMIT = 1000
+DEFAULT_GRB_MAX_FEATURES = 100000
 requests: Any = None
 rasterio: Any = None
 Transformer: Any = None
@@ -214,6 +216,18 @@ def parse_args() -> argparse.Namespace:
        default=float(os.environ.get("OPERATOR_SAMPLE_HALF_SIZE_SCALE", "1")),
        help="Multiplier applied to each documented AOI half-size in meters.",
    )
+    parser.add_argument(
+        "--reference-page-limit",
+        type=int,
+        default=int(os.environ.get("OPERATOR_GRB_PAGE_LIMIT", str(DEFAULT_GRB_PAGE_LIMIT))),
+        help="GRB OGC API Features page size for reference buildings.",
+    )
+    parser.add_argument(
+        "--reference-max-features",
+        type=int,
+        default=int(os.environ.get("OPERATOR_GRB_MAX_FEATURES", str(DEFAULT_GRB_MAX_FEATURES))),
+        help="Safety cap for paged GRB reference features per sample.",
+    )
    return parser.parse_args()


@@ -316,6 +330,45 @@ def sample_artifact_paths(sample: OperatorSample, output_dir: Path) -> tuple[Pat
    return ortho_path, reference_path


+def next_geojson_link(payload: dict[str, Any]) -> str | None:
+    for link in payload.get("links") or []:
+        if link.get("rel") == "next" and "geo+json" in str(link.get("type", "")).lower():
+            href = link.get("href")
+            if href:
+                return str(href)
+    for link in payload.get("links") or []:
+        if link.get("rel") == "next":
+            href = link.get("href")
+            if href:
+                return str(href)
+    return None
+
+
+def merge_reference_page_features(
+    pages: list[dict[str, Any]],
+    *,
+    max_features: int,
+) -> tuple[list[dict[str, Any]], bool]:
+    features: list[dict[str, Any]] = []
+    seen_feature_keys: set[str] = set()
+    truncated = False
+
+    for page in pages:
+        for feature in page.get("features") or []:
+            feature_key = str(feature.get("id") or json.dumps(feature.get("geometry"), sort_keys=True))
+            if feature_key in seen_feature_keys:
+                continue
+            if len(features) >= max_features:
+                truncated = True
+                break
+            seen_feature_keys.add(feature_key)
+            features.append(feature)
+        if truncated:
+            break
+
+    return features, truncated
+
+
 def fetch_orthophoto(sample: OperatorSample, ortho_path: Path, lambert_bbox: tuple[float, float, float, float]) -> str:
    minx, miny, maxx, maxy = lambert_bbox
    wms_params = {
@@ -362,27 +415,69 @@ def fetch_orthophoto(sample: OperatorSample, ortho_path: Path, lambert_bbox: tup
    return prepared_url(WMS_URL, wms_params)


-def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list[float]) -> tuple[str, int]:
+def fetch_reference(
+    sample: OperatorSample,
+    reference_path: Path,
+    geo_bbox: list[float],
+    *,
+    page_limit: int = DEFAULT_GRB_PAGE_LIMIT,
+    max_features: int = DEFAULT_GRB_MAX_FEATURES,
+) -> tuple[str, int]:
+    if page_limit <= 0:
+        raise SystemExit("--reference-page-limit must be a positive integer")
+    if max_features <= 0:
+        raise SystemExit("--reference-max-features must be a positive integer")
+
    ogc_params = {
        "f": "application/geo+json",
-        "limit": "1000",
+        "limit": str(page_limit),
        "bbox": ",".join(f"{value:.8f}" for value in geo_bbox),
    }
+    pages: list[dict[str, Any]] = []
+    page_urls = [prepared_url(GRB_GBG_URL, ogc_params)]
    response = requests.get(GRB_GBG_URL, params=ogc_params, timeout=120)
-    response.raise_for_status()
-    reference = response.json()
-    features = reference.get("features") or []
+
+    seen_next_urls: set[str] = set()
+    stopped_at_feature_cap = False
+    while True:
+        response.raise_for_status()
+        page = response.json()
+        pages.append(page)
+        next_url = next_geojson_link(page)
+        if not next_url:
+            break
+        if next_url in seen_next_urls:
+            raise SystemExit(f"GRB GBG pagination loop detected for {sample.slug}: {next_url}")
+        if sum(len(current_page.get("features") or []) for current_page in pages) >= max_features:
+            stopped_at_feature_cap = True
+            break
+        seen_next_urls.add(next_url)
+        page_urls.append(next_url)
+        response = requests.get(next_url, params=None, timeout=120)
+
+    reference = pages[0] if pages else {"type": "FeatureCollection", "features": []}
+    features, truncated = merge_reference_page_features(pages, max_features=max_features)
+    truncated = truncated or stopped_at_feature_cap
    if not features and not sample.allow_empty_reference:
        raise SystemExit(f"GRB GBG returned no building features for {sample.slug} bbox {geo_bbox}")

+    reference["features"] = features
    reference["name"] = f"GRB GBG buildings - {sample.display_name} sample AOI"
    reference["source"] = "Digitaal Vlaanderen GRB OGC API Features collection GBG"
    reference["source_url"] = prepared_url(GRB_GBG_URL, ogc_params)
+    reference["source_urls"] = page_urls
    reference["attribution"] = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen"
    reference["bbox"] = geo_bbox
    reference["sample_slug"] = sample.slug
    reference["sample_role"] = sample.sample_role
    reference["allow_empty_reference"] = sample.allow_empty_reference
+    reference["reference_page_limit"] = page_limit
+    reference["reference_max_features"] = max_features
+    reference["reference_pages_fetched"] = len(pages)
+    reference["reference_truncated"] = truncated
+    reference["numberReturned"] = len(features)
+    if "links" in reference:
+        reference["links"] = [link for link in reference.get("links") or [] if link.get("rel") != "next"]
    for feature in features:
        props = feature.setdefault("properties", {})
        props.setdefault("source_name", "grb")
@@ -394,7 +489,14 @@ def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list
    return prepared_url(GRB_GBG_URL, ogc_params), len(features)


-def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dict[str, Any]:
+def prepare_sample(
+    sample: OperatorSample,
+    output_dir: Path,
+    force: bool,
+    *,
+    reference_page_limit: int = DEFAULT_GRB_PAGE_LIMIT,
+    reference_max_features: int = DEFAULT_GRB_MAX_FEATURES,
+) -> dict[str, Any]:
    ortho_path, reference_path = sample_artifact_paths(sample, output_dir)
    lambert_bbox, geo_bbox = sample_bounds(sample)
    skip_existing = ortho_path.exists() and reference_path.exists() and not force
@@ -402,7 +504,13 @@ def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dic
    source_urls: dict[str, str | None] = {"orthophoto": None, "reference": None}
    if not skip_existing:
        source_urls["orthophoto"] = fetch_orthophoto(sample, ortho_path, lambert_bbox)
-        source_urls["reference"], reference_feature_count = fetch_reference(sample, reference_path, geo_bbox)
+        source_urls["reference"], reference_feature_count = fetch_reference(
+            sample,
+            reference_path,
+            geo_bbox,
+            page_limit=reference_page_limit,
+            max_features=reference_max_features,
+        )
    else:
        reference_feature_count = geojson_feature_count(reference_path)

@@ -424,6 +532,8 @@ def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dic
        "epsg31370_bbox": list(lambert_bbox),
        "skip_existing": skip_existing,
        "source_urls": source_urls,
+        "reference_page_limit": reference_page_limit,
+        "reference_max_features": reference_max_features,
        "attribution": {
            "orthophoto": "Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen",
            "reference": "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen",
@@ -470,6 +580,8 @@ def main() -> int:
            ),
            output_dir,
            force=args.force,
+            reference_page_limit=args.reference_page_limit,
+            reference_max_features=args.reference_max_features,
        )
        for sample in selected_samples(args.samples)
    ]
@@ -482,6 +594,8 @@ def main() -> int:
        "sample_width": args.width,
        "sample_height": args.height,
        "half_size_scale": args.half_size_scale,
+        "reference_page_limit": args.reference_page_limit,
+        "reference_max_features": args.reference_max_features,
        "samples": samples,
    }
    manifest_path = output_dir / args.manifest_name