diff --git a/CHANGELOG.md b/CHANGELOG.md index dc63552f..23b5afb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1414,3 +1414,4 @@ Added: - Added `scripts/audit_operator_yolo_dataset_quality.py`, an operator-only YOLO tile dataset quality audit that produces JSON and Markdown reports for sample coverage, split coverage, repeated hard-negative pressure and label-size integrity before further training runs. - Added pytest coverage and readiness syntax checking for the new operator YOLO dataset audit script. - Recorded live Tower audit results showing `yolo-building-tile-expanded160` as the clean current baseline and r4/r8 hard-negative datasets as repeat-heavy evidence sets that need more unique background AOIs before further hard-negative training. +- Expanded the explicit operator background-candidate AOI registry from 3 to 9 unique hard-negative locations and added tests for diversity/spread before further YOLO training. diff --git a/backend/tests/test_sprint131_operator_sample_expansion.py b/backend/tests/test_sprint131_operator_sample_expansion.py index dc699fa9..9295a5c9 100644 --- a/backend/tests/test_sprint131_operator_sample_expansion.py +++ b/backend/tests/test_sprint131_operator_sample_expansion.py @@ -25,7 +25,16 @@ def test_operator_sample_registry_includes_kempen_reference_and_background_candi module = load_sample_preparer() expected_reference_slugs = {"geel", "mol", "turnhout", "herentals", "balen", "retie", "westerlo"} - expected_background_slugs = {"postel_bos", "lommel_heide", "kasterlee_bos"} + expected_background_slugs = { + "postel_bos", + "lommel_heide", + "kasterlee_bos", + "dessel_heide", + "ravels_bos", + "meerhout_bos", + "geel_bel", + "arendonk_heide", + } assert expected_reference_slugs.issubset(module.SAMPLES) assert expected_background_slugs.issubset(module.SAMPLES) @@ -34,6 +43,26 @@ def test_operator_sample_registry_includes_kempen_reference_and_background_candi assert all(module.SAMPLES[slug].sample_role == "background_candidate" for slug in expected_background_slugs) +def test_operator_background_candidates_are_unique_enough_for_hard_negative_training() -> None: + module = load_sample_preparer() + + background_samples = [ + sample + for sample in module.SAMPLES.values() + if sample.sample_role == "background_candidate" + ] + centers = {(round(sample.center_lon, 4), round(sample.center_lat, 4)) for sample in background_samples} + half_sizes = {sample.half_size_m for sample in background_samples} + + assert len(background_samples) >= 8 + assert len(centers) == len(background_samples) + assert min(sample.center_lon for sample in background_samples) < 4.85 + assert max(sample.center_lon for sample in background_samples) > 5.25 + assert min(sample.center_lat for sample in background_samples) < 51.18 + assert max(sample.center_lat for sample in background_samples) > 51.33 + assert half_sizes == {260.0} + + def test_background_candidate_can_write_empty_reference_geojson(tmp_path: Path, monkeypatch) -> None: module = load_sample_preparer() diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 48175065..955e0784 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -5670,3 +5670,25 @@ Open: ## Next recommended pass - Add more unique hard-negative/background AOIs before another hard-negative training run. The current label files are clean, so the bottleneck is dataset diversity and balance rather than label-file corruption. + +# Sprint 147 - Unique hard-negative AOI expansion + +## What changed + +- Expanded `scripts/prepare_operator_real_data_samples.py` with six additional explicit background-candidate AOIs: Dessel-heide, Ravels-bos, Meerhout-bos, Geel-Bel, Arendonk-heide and Herenthout-bos. +- Background candidates remain operator/runtime samples only. They are not product providers, not fixtures and not automatic app fetches. +- Added sample-registry test coverage for minimum unique background count, unique centers and regional spread. +- Updated operator documentation with the expanded default corpus and the next required Tower regeneration step. + +## What was tested + +- `python -m pytest backend\tests\test_sprint131_operator_sample_expansion.py -q` + +## Known limitations + +- The new AOIs still need to be prepared on Tower before they affect the live operator YOLO tile exports. +- GRB may return sparse buildings in some background candidates; they remain valid hard-negative candidates only after the manifest and tile audit confirm their actual labels. + +## Next recommended pass + +- Pull this commit on Tower, rerun `prepare_operator_real_data_samples.py`, export a new hard-negative tile dataset without excessive repeat pressure and audit it before training. diff --git a/docs/TODO.md b/docs/TODO.md index 5ff894a3..76c01f32 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -442,5 +442,13 @@ This file now starts with the current implementation status. Older preparation/b - [x] Report sample coverage, validation coverage, repeated hard-negative pressure and YOLO label area integrity. - [x] Wire the audit script into the readiness syntax gate. - [x] Use live audit output to decide whether the next model pass needs more positive AOIs, label cleanup or unique hard negatives. -- [ ] Add more unique background/hard-negative AOIs before repeating hard-negative-balanced YOLO training. +- [x] Add more unique background/hard-negative AOIs before repeating hard-negative-balanced YOLO training. - [ ] Keep `yolo-building-tile-expanded160` as the clean current training baseline; avoid promoting r4/r8 repeat-heavy datasets as defaults. +- [ ] Regenerate Tower operator samples, export a new unique-hard-negative tile dataset and rerun the dataset audit before training. + +# Sprint 147 - Unique hard-negative AOI expansion + +- [x] Expand the documented operator background candidates from 3 to 9 unique AOIs. +- [x] Keep every new background AOI explicit, `allow_empty_reference=True`, and `sample_role='background_candidate'`. +- [x] Add test coverage for minimum background candidate count, unique centers and regional spread. +- [ ] Prepare the new samples on Tower and build a fresh hard-negative tile dataset. diff --git a/scripts/README.md b/scripts/README.md index 575d1c4a..0f9241bb 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -182,13 +182,14 @@ This writes GeoTIFF/GeoJSON pairs and `operator_samples_manifest.json` under `/app/storage/operator-data` inside the container, which maps to `storage/operator-data` in the Tower appdata checkout. The default corpus contains reference AOIs for Geel, Mol, Turnhout, Herentals, Balen, Retie and -Westerlo plus background candidates for Postel-bos, Lommel-heide and -Kasterlee-bos. Normal reference AOIs still fail when GRB returns no buildings; -background candidates are explicitly marked with `sample_role` and may write an -empty reference FeatureCollection for negative-tile training. The helper fetches -only the explicit documented AOIs, records Digitaal Vlaanderen attribution and -reuses existing files by default. Use `--force` only when the local runtime -artifacts should be regenerated. +Westerlo plus background candidates for Postel-bos, Lommel-heide, +Kasterlee-bos, Dessel-heide, Ravels-bos, Meerhout-bos, Geel-Bel, +Arendonk-heide and Herenthout-bos. Normal reference AOIs still fail when GRB +returns no buildings; background candidates are explicitly marked with +`sample_role` and may write an empty reference FeatureCollection for +negative-tile training. The helper fetches only the explicit documented AOIs, +records Digitaal Vlaanderen attribution and reuses existing files by default. +Use `--force` only when the local runtime artifacts should be regenerated. The real-data smoke is intentionally mutating and refuses to run without operator-supplied files. Current V1 upload support expects a georeferenced @@ -351,6 +352,8 @@ Current Tower audit status: - `yolo-building-tile-hardneg160r4` and `yolo-building-tile-hardneg160r8`: repeat-heavy hard-negative variants; useful evidence, but add more unique background AOIs before training another hard-negative-balanced candidate. +- Regenerate `operator_samples_manifest.json` after pulling Sprint 147+ so the + expanded unique background AOI set is available for the next tile export. For hard-negative-balanced experiments, repeat only train-split negative tiles from samples marked `sample_role=background_candidate`: @@ -415,7 +418,7 @@ before changing model defaults: ```bash OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \ -OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos" \ +OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos dessel_heide ravels_bos meerhout_bos geel_bel arendonk_heide herenthout_bos" \ QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-building-yolov8n-tile30-pt yolov8s-building-segmentation-pt" \ QUALITY_TILE_SIZES="640" \ QUALITY_TILE_OVERLAPS="64" \ diff --git a/scripts/prepare_operator_real_data_samples.py b/scripts/prepare_operator_real_data_samples.py index e3efb84e..277ce166 100644 --- a/scripts/prepare_operator_real_data_samples.py +++ b/scripts/prepare_operator_real_data_samples.py @@ -114,6 +114,60 @@ SAMPLES: dict[str, OperatorSample] = { sample_role="background_candidate", allow_empty_reference=True, ), + "dessel_heide": OperatorSample( + slug="dessel_heide", + display_name="Dessel heath background candidate", + center_lon=5.092, + center_lat=51.235, + half_size_m=260.0, + sample_role="background_candidate", + allow_empty_reference=True, + ), + "ravels_bos": OperatorSample( + slug="ravels_bos", + display_name="Ravels forest background candidate", + center_lon=4.977, + center_lat=51.384, + half_size_m=260.0, + sample_role="background_candidate", + allow_empty_reference=True, + ), + "meerhout_bos": OperatorSample( + slug="meerhout_bos", + display_name="Meerhout forest background candidate", + center_lon=5.069, + center_lat=51.115, + half_size_m=260.0, + sample_role="background_candidate", + allow_empty_reference=True, + ), + "geel_bel": OperatorSample( + slug="geel_bel", + display_name="Geel-Bel rural background candidate", + center_lon=5.046, + center_lat=51.137, + half_size_m=260.0, + sample_role="background_candidate", + allow_empty_reference=True, + ), + "arendonk_heide": OperatorSample( + slug="arendonk_heide", + display_name="Arendonk heath background candidate", + center_lon=5.238, + center_lat=51.334, + half_size_m=260.0, + sample_role="background_candidate", + allow_empty_reference=True, + ), + "herenthout_bos": OperatorSample( + slug="herenthout_bos", + display_name="Herenthout forest background candidate", + center_lon=4.781, + center_lat=51.143, + half_size_m=260.0, + sample_role="background_candidate", + allow_empty_reference=True, + ), }