diff --git a/CHANGELOG.md b/CHANGELOG.md index 04e8bb6d..1e542952 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ # Changelog +## Sprint 152 GRB reference paging for operator samples (2026-07-09) + +- Fixed the operator real-data sample preparer so GRB GBG reference GeoJSON is fetched through OGC API `rel=next` pagination links instead of stopping at the first `limit=1000` page. +- Added `--reference-page-limit` / `OPERATOR_GRB_PAGE_LIMIT` and `--reference-max-features` / `OPERATOR_GRB_MAX_FEATURES` safeguards for dense reference AOIs. +- Generated reference GeoJSON now records fetched page URLs, page count, truncation state and paging limits for auditability. +- Added regression coverage for paged GRB responses and the new CLI help options. +- No application provider endpoint, migration, API contract, live GRB product import, model download or active YOLO model changed. + ## Sprint 151 runtime GIS upload and AOI1024 YOLO candidate (2026-07-09) - Fixed the operator YOLO training wrapper so the all-in-one runtime defaults to `/opt/geointel/venv/bin/python` when present, while still falling back to `python3` for local shells. diff --git a/backend/tests/test_sprint127_operator_sample_quality_matrix.py b/backend/tests/test_sprint127_operator_sample_quality_matrix.py index eaa57fdc..663ded4c 100644 --- a/backend/tests/test_sprint127_operator_sample_quality_matrix.py +++ b/backend/tests/test_sprint127_operator_sample_quality_matrix.py @@ -46,6 +46,8 @@ def test_prepare_operator_real_data_samples_help_does_not_require_gis_dependenci assert "--width" in result.stdout assert "--height" in result.stdout assert "--half-size-scale" in result.stdout + assert "--reference-page-limit" in result.stdout + assert "--reference-max-features" in result.stdout def test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sample() -> None: diff --git a/backend/tests/test_sprint131_operator_sample_expansion.py b/backend/tests/test_sprint131_operator_sample_expansion.py index cb38d4f4..0bb93095 100644 --- a/backend/tests/test_sprint131_operator_sample_expansion.py +++ b/backend/tests/test_sprint131_operator_sample_expansion.py @@ -123,6 +123,81 @@ def test_background_candidate_can_write_empty_reference_geojson(tmp_path: Path, assert '"sample_role": "background_candidate"' in payload +def test_fetch_reference_follows_grb_next_links_until_complete(tmp_path: Path, monkeypatch) -> None: + module = load_sample_preparer() + requested: list[tuple[str, dict | None]] = [] + + def feature(feature_id: str) -> dict: + return { + "type": "Feature", + "id": feature_id, + "geometry": {"type": "Polygon", "coordinates": []}, + "properties": {}, + } + + class FeatureResponse: + def __init__(self, payload: dict) -> None: + self.payload = payload + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict: + return self.payload + + class FakeRequests: + Request = module.requests.Request if module.requests else object + + @staticmethod + def get(url, params=None, timeout=120): + requested.append((url, params)) + if len(requested) == 1: + return FeatureResponse( + { + "type": "FeatureCollection", + "features": [feature("GBG.1")], + "numberReturned": 1, + "links": [ + { + "rel": "next", + "type": "application/geo+json", + "href": "https://example.test/grb?page=2", + } + ], + } + ) + return FeatureResponse( + { + "type": "FeatureCollection", + "features": [feature("GBG.2")], + "numberReturned": 1, + "links": [], + } + ) + + monkeypatch.setattr(module, "requests", FakeRequests) + monkeypatch.setattr(module, "prepared_url", lambda url, params: f"{url}?prepared=true") + + sample = module.OperatorSample( + slug="urban", + display_name="Urban", + center_lon=5.0, + center_lat=51.0, + ) + + source_url, feature_count = module.fetch_reference(sample, tmp_path / "urban.geojson", [4.9, 50.9, 5.1, 51.1]) + payload = (tmp_path / "urban.geojson").read_text(encoding="utf-8") + + assert source_url.endswith("?prepared=true") + assert feature_count == 2 + assert requested == [ + (module.GRB_GBG_URL, {"f": "application/geo+json", "limit": "1000", "bbox": "4.90000000,50.90000000,5.10000000,51.10000000"}), + ("https://example.test/grb?page=2", None), + ] + assert '"id": "GBG.1"' in payload + assert '"id": "GBG.2"' in payload + + def test_reference_sample_still_rejects_empty_grb_response(tmp_path: Path, monkeypatch) -> None: module = load_sample_preparer() diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 90f077bd..eb084a93 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -196,10 +196,15 @@ The helper fetches explicit Digitaal Vlaanderen orthophoto/GRB GBG sample pairs for the documented AOIs only and writes `operator_samples_manifest.json`. The default corpus includes dense reference AOIs for Geel, Mol, Turnhout, Herentals, Balen, Retie and Westerlo plus explicitly marked background candidates for -Postel-bos, Lommel-heide and Kasterlee-bos. Background candidates may persist -empty GRB FeatureCollections for negative-tile training; normal reference AOIs -still fail on empty GRB responses. The application itself still does not perform -live provider fetching. +Postel-bos, Lommel-heide, Kasterlee-bos, Dessel-heide, Ravels-bos, +Meerhout-bos, Geel-Bel, Arendonk-heide and Herenthout-bos. Background +candidates may persist empty GRB FeatureCollections for negative-tile training; +normal reference AOIs still fail on empty GRB responses. Dense GRB references +are fetched through OGC API `rel=next` pagination links instead of trusting only +the first 1000-feature page. Generated reference GeoJSON records +`reference_pages_fetched`, `reference_truncated`, `reference_page_limit`, +`reference_max_features` and `source_urls` for auditability. The application +itself still does not perform live provider fetching. For confidence-threshold calibration, use the sweep wrapper: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 652e8eff..d40a57e5 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -5938,6 +5938,32 @@ Open: - Use `yolo-building-tile-uniquehardneg160` as the next safer hard-negative training dataset candidate. Benchmark after training before changing defaults. +# Sprint 152 - GRB reference paging for operator samples + +## What changed + +- Fixed `scripts/prepare_operator_real_data_samples.py` so GRB GBG reference exports follow OGC API `rel=next` pagination links instead of silently trusting only the first `limit=1000` page. +- Added operator controls: + - `--reference-page-limit` / `OPERATOR_GRB_PAGE_LIMIT`, default `1000`. + - `--reference-max-features` / `OPERATOR_GRB_MAX_FEATURES`, default `100000`. +- Generated reference GeoJSON now records `source_urls`, `reference_pages_fetched`, `reference_truncated`, `reference_page_limit` and `reference_max_features`. +- Kept the change operator-only: no GeoIntel API route calls this helper, no product provider endpoint changed, no live GRB/OSM import was added, no migration changed and no YOLO model was activated. + +## What was tested + +- RED: `python -m pytest backend\tests\test_sprint131_operator_sample_expansion.py::test_fetch_reference_follows_grb_next_links_until_complete -q` failed because only the first GRB page was persisted. +- GREEN: `python -m pytest backend\tests\test_sprint131_operator_sample_expansion.py::test_fetch_reference_follows_grb_next_links_until_complete -q` +- `python -m pytest backend\tests\test_sprint127_operator_sample_quality_matrix.py backend\tests\test_sprint131_operator_sample_expansion.py -q` + +## Known limitations + +- Existing Tower AOI1024 artifacts were generated before this fix. Regenerate the operator samples with `--force` before retraining or re-auditing dense AOI labels. +- This does not make the Sprint 7B GRB provider a live product importer; it only fixes explicit operator sample preparation. + +## Next recommended pass + +- Regenerate `/app/storage/operator-data/operator-samples-1024` on Tower with the paged script, re-export the AOI1024 YOLO tile dataset, rerun the dataset audit and only then consider another inactive training candidate. + # Sprint 151 - Runtime GIS upload and AOI1024 YOLO candidate ## What changed diff --git a/docs/TODO.md b/docs/TODO.md index 2e3f5f42..c35e79a8 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -472,5 +472,6 @@ This file now starts with the current implementation status. Older preparation/b - [x] Run AOI1024 background/hard-negative matrix for `geointel-building-yolov8s-aoi1024visible025e50-pt`. - [x] Fix promotion-report parsing for `multi_sample_quality_summary.json` inputs. - [x] Generate AOI1024 promotion report and keep recommended candidate as `none`. -- [ ] Add GRB paging or smaller dense AOI sampling before trusting 1000-feature-capped dense reference exports as full ground truth. +- [x] Add GRB paging before trusting dense reference exports as full ground truth. +- [ ] Regenerate Tower AOI1024 operator samples with paged GRB references, then re-export and audit labels before any new training attempt. - [ ] Keep every local YOLO candidate inactive until positive-AOI and hard-negative promotion reports recommend default activation. diff --git a/scripts/README.md b/scripts/README.md index b6c1b91d..e13cc3fd 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -190,6 +190,15 @@ returns no buildings; background candidates are explicitly marked with negative-tile training. The helper fetches only the explicit documented AOIs, records Digitaal Vlaanderen attribution and reuses existing files by default. Use `--force` only when the local runtime artifacts should be regenerated. +GRB building references are fetched through the provider's OGC API +`rel=next` pagination links, so dense AOIs are not silently limited to the +first 1000 features. The default page size is `1000`; override it with +`--reference-page-limit` or `OPERATOR_GRB_PAGE_LIMIT`. The safety cap defaults +to `100000` features per sample and can be adjusted with +`--reference-max-features` or `OPERATOR_GRB_MAX_FEATURES`. Generated reference +GeoJSON files record `reference_pages_fetched`, `reference_truncated`, +`reference_page_limit`, `reference_max_features` and every fetched +`source_urls` page for auditability. For model-training candidates, prepare a larger operator-only sample manifest so tile overlap can create meaningful context instead of one tile per source @@ -202,6 +211,8 @@ docker exec -it geointel python3 /app/scripts/prepare_operator_real_data_samples --width 1024 \ --height 1024 \ --half-size-scale 2 \ + --reference-page-limit 1000 \ + --reference-max-features 100000 \ --force ``` diff --git a/scripts/prepare_operator_real_data_samples.py b/scripts/prepare_operator_real_data_samples.py index 1a9d7ad3..bf97d774 100644 --- a/scripts/prepare_operator_real_data_samples.py +++ b/scripts/prepare_operator_real_data_samples.py @@ -20,6 +20,8 @@ from typing import Any WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms" GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items" DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data") +DEFAULT_GRB_PAGE_LIMIT = 1000 +DEFAULT_GRB_MAX_FEATURES = 100000 requests: Any = None rasterio: Any = None Transformer: Any = None @@ -214,6 +216,18 @@ def parse_args() -> argparse.Namespace: default=float(os.environ.get("OPERATOR_SAMPLE_HALF_SIZE_SCALE", "1")), help="Multiplier applied to each documented AOI half-size in meters.", ) + parser.add_argument( + "--reference-page-limit", + type=int, + default=int(os.environ.get("OPERATOR_GRB_PAGE_LIMIT", str(DEFAULT_GRB_PAGE_LIMIT))), + help="GRB OGC API Features page size for reference buildings.", + ) + parser.add_argument( + "--reference-max-features", + type=int, + default=int(os.environ.get("OPERATOR_GRB_MAX_FEATURES", str(DEFAULT_GRB_MAX_FEATURES))), + help="Safety cap for paged GRB reference features per sample.", + ) return parser.parse_args() @@ -316,6 +330,45 @@ def sample_artifact_paths(sample: OperatorSample, output_dir: Path) -> tuple[Pat return ortho_path, reference_path +def next_geojson_link(payload: dict[str, Any]) -> str | None: + for link in payload.get("links") or []: + if link.get("rel") == "next" and "geo+json" in str(link.get("type", "")).lower(): + href = link.get("href") + if href: + return str(href) + for link in payload.get("links") or []: + if link.get("rel") == "next": + href = link.get("href") + if href: + return str(href) + return None + + +def merge_reference_page_features( + pages: list[dict[str, Any]], + *, + max_features: int, +) -> tuple[list[dict[str, Any]], bool]: + features: list[dict[str, Any]] = [] + seen_feature_keys: set[str] = set() + truncated = False + + for page in pages: + for feature in page.get("features") or []: + feature_key = str(feature.get("id") or json.dumps(feature.get("geometry"), sort_keys=True)) + if feature_key in seen_feature_keys: + continue + if len(features) >= max_features: + truncated = True + break + seen_feature_keys.add(feature_key) + features.append(feature) + if truncated: + break + + return features, truncated + + def fetch_orthophoto(sample: OperatorSample, ortho_path: Path, lambert_bbox: tuple[float, float, float, float]) -> str: minx, miny, maxx, maxy = lambert_bbox wms_params = { @@ -362,27 +415,69 @@ def fetch_orthophoto(sample: OperatorSample, ortho_path: Path, lambert_bbox: tup return prepared_url(WMS_URL, wms_params) -def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list[float]) -> tuple[str, int]: +def fetch_reference( + sample: OperatorSample, + reference_path: Path, + geo_bbox: list[float], + *, + page_limit: int = DEFAULT_GRB_PAGE_LIMIT, + max_features: int = DEFAULT_GRB_MAX_FEATURES, +) -> tuple[str, int]: + if page_limit <= 0: + raise SystemExit("--reference-page-limit must be a positive integer") + if max_features <= 0: + raise SystemExit("--reference-max-features must be a positive integer") + ogc_params = { "f": "application/geo+json", - "limit": "1000", + "limit": str(page_limit), "bbox": ",".join(f"{value:.8f}" for value in geo_bbox), } + pages: list[dict[str, Any]] = [] + page_urls = [prepared_url(GRB_GBG_URL, ogc_params)] response = requests.get(GRB_GBG_URL, params=ogc_params, timeout=120) - response.raise_for_status() - reference = response.json() - features = reference.get("features") or [] + + seen_next_urls: set[str] = set() + stopped_at_feature_cap = False + while True: + response.raise_for_status() + page = response.json() + pages.append(page) + next_url = next_geojson_link(page) + if not next_url: + break + if next_url in seen_next_urls: + raise SystemExit(f"GRB GBG pagination loop detected for {sample.slug}: {next_url}") + if sum(len(current_page.get("features") or []) for current_page in pages) >= max_features: + stopped_at_feature_cap = True + break + seen_next_urls.add(next_url) + page_urls.append(next_url) + response = requests.get(next_url, params=None, timeout=120) + + reference = pages[0] if pages else {"type": "FeatureCollection", "features": []} + features, truncated = merge_reference_page_features(pages, max_features=max_features) + truncated = truncated or stopped_at_feature_cap if not features and not sample.allow_empty_reference: raise SystemExit(f"GRB GBG returned no building features for {sample.slug} bbox {geo_bbox}") + reference["features"] = features reference["name"] = f"GRB GBG buildings - {sample.display_name} sample AOI" reference["source"] = "Digitaal Vlaanderen GRB OGC API Features collection GBG" reference["source_url"] = prepared_url(GRB_GBG_URL, ogc_params) + reference["source_urls"] = page_urls reference["attribution"] = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen" reference["bbox"] = geo_bbox reference["sample_slug"] = sample.slug reference["sample_role"] = sample.sample_role reference["allow_empty_reference"] = sample.allow_empty_reference + reference["reference_page_limit"] = page_limit + reference["reference_max_features"] = max_features + reference["reference_pages_fetched"] = len(pages) + reference["reference_truncated"] = truncated + reference["numberReturned"] = len(features) + if "links" in reference: + reference["links"] = [link for link in reference.get("links") or [] if link.get("rel") != "next"] for feature in features: props = feature.setdefault("properties", {}) props.setdefault("source_name", "grb") @@ -394,7 +489,14 @@ def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list return prepared_url(GRB_GBG_URL, ogc_params), len(features) -def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dict[str, Any]: +def prepare_sample( + sample: OperatorSample, + output_dir: Path, + force: bool, + *, + reference_page_limit: int = DEFAULT_GRB_PAGE_LIMIT, + reference_max_features: int = DEFAULT_GRB_MAX_FEATURES, +) -> dict[str, Any]: ortho_path, reference_path = sample_artifact_paths(sample, output_dir) lambert_bbox, geo_bbox = sample_bounds(sample) skip_existing = ortho_path.exists() and reference_path.exists() and not force @@ -402,7 +504,13 @@ def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dic source_urls: dict[str, str | None] = {"orthophoto": None, "reference": None} if not skip_existing: source_urls["orthophoto"] = fetch_orthophoto(sample, ortho_path, lambert_bbox) - source_urls["reference"], reference_feature_count = fetch_reference(sample, reference_path, geo_bbox) + source_urls["reference"], reference_feature_count = fetch_reference( + sample, + reference_path, + geo_bbox, + page_limit=reference_page_limit, + max_features=reference_max_features, + ) else: reference_feature_count = geojson_feature_count(reference_path) @@ -424,6 +532,8 @@ def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dic "epsg31370_bbox": list(lambert_bbox), "skip_existing": skip_existing, "source_urls": source_urls, + "reference_page_limit": reference_page_limit, + "reference_max_features": reference_max_features, "attribution": { "orthophoto": "Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen", "reference": "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen", @@ -470,6 +580,8 @@ def main() -> int: ), output_dir, force=args.force, + reference_page_limit=args.reference_page_limit, + reference_max_features=args.reference_max_features, ) for sample in selected_samples(args.samples) ] @@ -482,6 +594,8 @@ def main() -> int: "sample_width": args.width, "sample_height": args.height, "half_size_scale": args.half_size_scale, + "reference_page_limit": args.reference_page_limit, + "reference_max_features": args.reference_max_features, "samples": samples, } manifest_path = output_dir / args.manifest_name