Page GRB references for operator samples

CHANGELOG.md

@@ -7,6 +7,14 @@
 
 # Changelog
 
+## Sprint 152 GRB reference paging for operator samples (2026-07-09)
+
+- Fixed the operator real-data sample preparer so GRB GBG reference GeoJSON is fetched through OGC API `rel=next` pagination links instead of stopping at the first `limit=1000` page.
+- Added `--reference-page-limit` / `OPERATOR_GRB_PAGE_LIMIT` and `--reference-max-features` / `OPERATOR_GRB_MAX_FEATURES` safeguards for dense reference AOIs.
+- Generated reference GeoJSON now records fetched page URLs, page count, truncation state and paging limits for auditability.
+- Added regression coverage for paged GRB responses and the new CLI help options.
+- No application provider endpoint, migration, API contract, live GRB product import, model download or active YOLO model changed.
+
 ## Sprint 151 runtime GIS upload and AOI1024 YOLO candidate (2026-07-09)
 
 - Fixed the operator YOLO training wrapper so the all-in-one runtime defaults to `/opt/geointel/venv/bin/python` when present, while still falling back to `python3` for local shells.

backend/tests/test_sprint127_operator_sample_quality_matrix.py

@@ -46,6 +46,8 @@ def test_prepare_operator_real_data_samples_help_does_not_require_gis_dependenci
     assert "--width" in result.stdout
     assert "--height" in result.stdout
     assert "--half-size-scale" in result.stdout
+    assert "--reference-page-limit" in result.stdout
+    assert "--reference-max-features" in result.stdout
 
 
 def test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sample() -> None:

backend/tests/test_sprint131_operator_sample_expansion.py

@@ -123,6 +123,81 @@ def test_background_candidate_can_write_empty_reference_geojson(tmp_path: Path,
     assert '"sample_role": "background_candidate"' in payload
 
 
+def test_fetch_reference_follows_grb_next_links_until_complete(tmp_path: Path, monkeypatch) -> None:
+    module = load_sample_preparer()
+    requested: list[tuple[str, dict | None]] = []
+
+    def feature(feature_id: str) -> dict:
+        return {
+            "type": "Feature",
+            "id": feature_id,
+            "geometry": {"type": "Polygon", "coordinates": []},
+            "properties": {},
+        }
+
+    class FeatureResponse:
+        def __init__(self, payload: dict) -> None:
+            self.payload = payload
+
+        def raise_for_status(self) -> None:
+            return None
+
+        def json(self) -> dict:
+            return self.payload
+
+    class FakeRequests:
+        Request = module.requests.Request if module.requests else object
+
+        @staticmethod
+        def get(url, params=None, timeout=120):
+            requested.append((url, params))
+            if len(requested) == 1:
+                return FeatureResponse(
+                    {
+                        "type": "FeatureCollection",
+                        "features": [feature("GBG.1")],
+                        "numberReturned": 1,
+                        "links": [
+                            {
+                                "rel": "next",
+                                "type": "application/geo+json",
+                                "href": "https://example.test/grb?page=2",
+                            }
+                        ],
+                    }
+                )
+            return FeatureResponse(
+                {
+                    "type": "FeatureCollection",
+                    "features": [feature("GBG.2")],
+                    "numberReturned": 1,
+                    "links": [],
+                }
+            )
+
+    monkeypatch.setattr(module, "requests", FakeRequests)
+    monkeypatch.setattr(module, "prepared_url", lambda url, params: f"{url}?prepared=true")
+
+    sample = module.OperatorSample(
+        slug="urban",
+        display_name="Urban",
+        center_lon=5.0,
+        center_lat=51.0,
+    )
+
+    source_url, feature_count = module.fetch_reference(sample, tmp_path / "urban.geojson", [4.9, 50.9, 5.1, 51.1])
+    payload = (tmp_path / "urban.geojson").read_text(encoding="utf-8")
+
+    assert source_url.endswith("?prepared=true")
+    assert feature_count == 2
+    assert requested == [
+        (module.GRB_GBG_URL, {"f": "application/geo+json", "limit": "1000", "bbox": "4.90000000,50.90000000,5.10000000,51.10000000"}),
+        ("https://example.test/grb?page=2", None),
+    ]
+    assert '"id": "GBG.1"' in payload
+    assert '"id": "GBG.2"' in payload
+
+
 def test_reference_sample_still_rejects_empty_grb_response(tmp_path: Path, monkeypatch) -> None:
     module = load_sample_preparer()
 

docs/AI_PIPELINES.md

@@ -196,10 +196,15 @@ The helper fetches explicit Digitaal Vlaanderen orthophoto/GRB GBG sample pairs
 for the documented AOIs only and writes `operator_samples_manifest.json`. The
 default corpus includes dense reference AOIs for Geel, Mol, Turnhout, Herentals,
 Balen, Retie and Westerlo plus explicitly marked background candidates for
-Postel-bos, Lommel-heide and Kasterlee-bos. Background candidates may persist
-empty GRB FeatureCollections for negative-tile training; normal reference AOIs
-still fail on empty GRB responses. The application itself still does not perform
-live provider fetching.
+Postel-bos, Lommel-heide, Kasterlee-bos, Dessel-heide, Ravels-bos,
+Meerhout-bos, Geel-Bel, Arendonk-heide and Herenthout-bos. Background
+candidates may persist empty GRB FeatureCollections for negative-tile training;
+normal reference AOIs still fail on empty GRB responses. Dense GRB references
+are fetched through OGC API `rel=next` pagination links instead of trusting only
+the first 1000-feature page. Generated reference GeoJSON records
+`reference_pages_fetched`, `reference_truncated`, `reference_page_limit`,
+`reference_max_features` and `source_urls` for auditability. The application
+itself still does not perform live provider fetching.
 
 For confidence-threshold calibration, use the sweep wrapper:
 

docs/CODEX_EXECUTION_LOG.md

@@ -5938,6 +5938,32 @@ Open:
 
 - Use `yolo-building-tile-uniquehardneg160` as the next safer hard-negative training dataset candidate. Benchmark after training before changing defaults.
 
+# Sprint 152 - GRB reference paging for operator samples
+
+## What changed
+
+- Fixed `scripts/prepare_operator_real_data_samples.py` so GRB GBG reference exports follow OGC API `rel=next` pagination links instead of silently trusting only the first `limit=1000` page.
+- Added operator controls:
+  - `--reference-page-limit` / `OPERATOR_GRB_PAGE_LIMIT`, default `1000`.
+  - `--reference-max-features` / `OPERATOR_GRB_MAX_FEATURES`, default `100000`.
+- Generated reference GeoJSON now records `source_urls`, `reference_pages_fetched`, `reference_truncated`, `reference_page_limit` and `reference_max_features`.
+- Kept the change operator-only: no GeoIntel API route calls this helper, no product provider endpoint changed, no live GRB/OSM import was added, no migration changed and no YOLO model was activated.
+
+## What was tested
+
+- RED: `python -m pytest backend\tests\test_sprint131_operator_sample_expansion.py::test_fetch_reference_follows_grb_next_links_until_complete -q` failed because only the first GRB page was persisted.
+- GREEN: `python -m pytest backend\tests\test_sprint131_operator_sample_expansion.py::test_fetch_reference_follows_grb_next_links_until_complete -q`
+- `python -m pytest backend\tests\test_sprint127_operator_sample_quality_matrix.py backend\tests\test_sprint131_operator_sample_expansion.py -q`
+
+## Known limitations
+
+- Existing Tower AOI1024 artifacts were generated before this fix. Regenerate the operator samples with `--force` before retraining or re-auditing dense AOI labels.
+- This does not make the Sprint 7B GRB provider a live product importer; it only fixes explicit operator sample preparation.
+
+## Next recommended pass
+
+- Regenerate `/app/storage/operator-data/operator-samples-1024` on Tower with the paged script, re-export the AOI1024 YOLO tile dataset, rerun the dataset audit and only then consider another inactive training candidate.
+
 # Sprint 151 - Runtime GIS upload and AOI1024 YOLO candidate
 
 ## What changed

docs/TODO.md

@@ -472,5 +472,6 @@ This file now starts with the current implementation status. Older preparation/b
 - [x] Run AOI1024 background/hard-negative matrix for `geointel-building-yolov8s-aoi1024visible025e50-pt`.
 - [x] Fix promotion-report parsing for `multi_sample_quality_summary.json` inputs.
 - [x] Generate AOI1024 promotion report and keep recommended candidate as `none`.
-- [ ] Add GRB paging or smaller dense AOI sampling before trusting 1000-feature-capped dense reference exports as full ground truth.
+- [x] Add GRB paging before trusting dense reference exports as full ground truth.
+- [ ] Regenerate Tower AOI1024 operator samples with paged GRB references, then re-export and audit labels before any new training attempt.
 - [ ] Keep every local YOLO candidate inactive until positive-AOI and hard-negative promotion reports recommend default activation.

scripts/README.md

@@ -190,6 +190,15 @@ returns no buildings; background candidates are explicitly marked with
 negative-tile training. The helper fetches only the explicit documented AOIs,
 records Digitaal Vlaanderen attribution and reuses existing files by default.
 Use `--force` only when the local runtime artifacts should be regenerated.
+GRB building references are fetched through the provider's OGC API
+`rel=next` pagination links, so dense AOIs are not silently limited to the
+first 1000 features. The default page size is `1000`; override it with
+`--reference-page-limit` or `OPERATOR_GRB_PAGE_LIMIT`. The safety cap defaults
+to `100000` features per sample and can be adjusted with
+`--reference-max-features` or `OPERATOR_GRB_MAX_FEATURES`. Generated reference
+GeoJSON files record `reference_pages_fetched`, `reference_truncated`,
+`reference_page_limit`, `reference_max_features` and every fetched
+`source_urls` page for auditability.
 
 For model-training candidates, prepare a larger operator-only sample manifest so
 tile overlap can create meaningful context instead of one tile per source
@@ -202,6 +211,8 @@ docker exec -it geointel python3 /app/scripts/prepare_operator_real_data_samples
   --width 1024 \
   --height 1024 \
   --half-size-scale 2 \
+  --reference-page-limit 1000 \
+  --reference-max-features 100000 \
   --force
 ```
 

scripts/prepare_operator_real_data_samples.py

@@ -20,6 +20,8 @@ from typing import Any
 WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms"
 GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
 DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data")
+DEFAULT_GRB_PAGE_LIMIT = 1000
+DEFAULT_GRB_MAX_FEATURES = 100000
 requests: Any = None
 rasterio: Any = None
 Transformer: Any = None
@@ -214,6 +216,18 @@ def parse_args() -> argparse.Namespace:
         default=float(os.environ.get("OPERATOR_SAMPLE_HALF_SIZE_SCALE", "1")),
         help="Multiplier applied to each documented AOI half-size in meters.",
     )
+    parser.add_argument(
+        "--reference-page-limit",
+        type=int,
+        default=int(os.environ.get("OPERATOR_GRB_PAGE_LIMIT", str(DEFAULT_GRB_PAGE_LIMIT))),
+        help="GRB OGC API Features page size for reference buildings.",
+    )
+    parser.add_argument(
+        "--reference-max-features",
+        type=int,
+        default=int(os.environ.get("OPERATOR_GRB_MAX_FEATURES", str(DEFAULT_GRB_MAX_FEATURES))),
+        help="Safety cap for paged GRB reference features per sample.",
+    )
     return parser.parse_args()
 
 
@@ -316,6 +330,45 @@ def sample_artifact_paths(sample: OperatorSample, output_dir: Path) -> tuple[Pat
     return ortho_path, reference_path
 
 
+def next_geojson_link(payload: dict[str, Any]) -> str | None:
+    for link in payload.get("links") or []:
+        if link.get("rel") == "next" and "geo+json" in str(link.get("type", "")).lower():
+            href = link.get("href")
+            if href:
+                return str(href)
+    for link in payload.get("links") or []:
+        if link.get("rel") == "next":
+            href = link.get("href")
+            if href:
+                return str(href)
+    return None
+
+
+def merge_reference_page_features(
+    pages: list[dict[str, Any]],
+    *,
+    max_features: int,
+) -> tuple[list[dict[str, Any]], bool]:
+    features: list[dict[str, Any]] = []
+    seen_feature_keys: set[str] = set()
+    truncated = False
+
+    for page in pages:
+        for feature in page.get("features") or []:
+            feature_key = str(feature.get("id") or json.dumps(feature.get("geometry"), sort_keys=True))
+            if feature_key in seen_feature_keys:
+                continue
+            if len(features) >= max_features:
+                truncated = True
+                break
+            seen_feature_keys.add(feature_key)
+            features.append(feature)
+        if truncated:
+            break
+
+    return features, truncated
+
+
 def fetch_orthophoto(sample: OperatorSample, ortho_path: Path, lambert_bbox: tuple[float, float, float, float]) -> str:
     minx, miny, maxx, maxy = lambert_bbox
     wms_params = {
@@ -362,27 +415,69 @@ def fetch_orthophoto(sample: OperatorSample, ortho_path: Path, lambert_bbox: tup
     return prepared_url(WMS_URL, wms_params)
 
 
-def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list[float]) -> tuple[str, int]:
+def fetch_reference(
+    sample: OperatorSample,
+    reference_path: Path,
+    geo_bbox: list[float],
+    *,
+    page_limit: int = DEFAULT_GRB_PAGE_LIMIT,
+    max_features: int = DEFAULT_GRB_MAX_FEATURES,
+) -> tuple[str, int]:
+    if page_limit <= 0:
+        raise SystemExit("--reference-page-limit must be a positive integer")
+    if max_features <= 0:
+        raise SystemExit("--reference-max-features must be a positive integer")
+
     ogc_params = {
         "f": "application/geo+json",
-        "limit": "1000",
+        "limit": str(page_limit),
         "bbox": ",".join(f"{value:.8f}" for value in geo_bbox),
     }
+    pages: list[dict[str, Any]] = []
+    page_urls = [prepared_url(GRB_GBG_URL, ogc_params)]
     response = requests.get(GRB_GBG_URL, params=ogc_params, timeout=120)
-    response.raise_for_status()
-    reference = response.json()
-    features = reference.get("features") or []
+
+    seen_next_urls: set[str] = set()
+    stopped_at_feature_cap = False
+    while True:
+        response.raise_for_status()
+        page = response.json()
+        pages.append(page)
+        next_url = next_geojson_link(page)
+        if not next_url:
+            break
+        if next_url in seen_next_urls:
+            raise SystemExit(f"GRB GBG pagination loop detected for {sample.slug}: {next_url}")
+        if sum(len(current_page.get("features") or []) for current_page in pages) >= max_features:
+            stopped_at_feature_cap = True
+            break
+        seen_next_urls.add(next_url)
+        page_urls.append(next_url)
+        response = requests.get(next_url, params=None, timeout=120)
+
+    reference = pages[0] if pages else {"type": "FeatureCollection", "features": []}
+    features, truncated = merge_reference_page_features(pages, max_features=max_features)
+    truncated = truncated or stopped_at_feature_cap
     if not features and not sample.allow_empty_reference:
         raise SystemExit(f"GRB GBG returned no building features for {sample.slug} bbox {geo_bbox}")
 
+    reference["features"] = features
     reference["name"] = f"GRB GBG buildings - {sample.display_name} sample AOI"
     reference["source"] = "Digitaal Vlaanderen GRB OGC API Features collection GBG"
     reference["source_url"] = prepared_url(GRB_GBG_URL, ogc_params)
+    reference["source_urls"] = page_urls
     reference["attribution"] = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen"
     reference["bbox"] = geo_bbox
     reference["sample_slug"] = sample.slug
     reference["sample_role"] = sample.sample_role
     reference["allow_empty_reference"] = sample.allow_empty_reference
+    reference["reference_page_limit"] = page_limit
+    reference["reference_max_features"] = max_features
+    reference["reference_pages_fetched"] = len(pages)
+    reference["reference_truncated"] = truncated
+    reference["numberReturned"] = len(features)
+    if "links" in reference:
+        reference["links"] = [link for link in reference.get("links") or [] if link.get("rel") != "next"]
     for feature in features:
         props = feature.setdefault("properties", {})
         props.setdefault("source_name", "grb")
@@ -394,7 +489,14 @@ def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list
     return prepared_url(GRB_GBG_URL, ogc_params), len(features)
 
 
-def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dict[str, Any]:
+def prepare_sample(
+    sample: OperatorSample,
+    output_dir: Path,
+    force: bool,
+    *,
+    reference_page_limit: int = DEFAULT_GRB_PAGE_LIMIT,
+    reference_max_features: int = DEFAULT_GRB_MAX_FEATURES,
+) -> dict[str, Any]:
     ortho_path, reference_path = sample_artifact_paths(sample, output_dir)
     lambert_bbox, geo_bbox = sample_bounds(sample)
     skip_existing = ortho_path.exists() and reference_path.exists() and not force
@@ -402,7 +504,13 @@ def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dic
     source_urls: dict[str, str | None] = {"orthophoto": None, "reference": None}
     if not skip_existing:
         source_urls["orthophoto"] = fetch_orthophoto(sample, ortho_path, lambert_bbox)
-        source_urls["reference"], reference_feature_count = fetch_reference(sample, reference_path, geo_bbox)
+        source_urls["reference"], reference_feature_count = fetch_reference(
+            sample,
+            reference_path,
+            geo_bbox,
+            page_limit=reference_page_limit,
+            max_features=reference_max_features,
+        )
     else:
         reference_feature_count = geojson_feature_count(reference_path)
 
@@ -424,6 +532,8 @@ def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dic
         "epsg31370_bbox": list(lambert_bbox),
         "skip_existing": skip_existing,
         "source_urls": source_urls,
+        "reference_page_limit": reference_page_limit,
+        "reference_max_features": reference_max_features,
         "attribution": {
             "orthophoto": "Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen",
             "reference": "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen",
@@ -470,6 +580,8 @@ def main() -> int:
             ),
             output_dir,
             force=args.force,
+            reference_page_limit=args.reference_page_limit,
+            reference_max_features=args.reference_max_features,
         )
         for sample in selected_samples(args.samples)
     ]
@@ -482,6 +594,8 @@ def main() -> int:
         "sample_width": args.width,
         "sample_height": args.height,
         "half_size_scale": args.half_size_scale,
+        "reference_page_limit": args.reference_page_limit,
+        "reference_max_features": args.reference_max_features,
         "samples": samples,
     }
     manifest_path = output_dir / args.manifest_name