diff --git a/CHANGELOG.md b/CHANGELOG.md index 57da0c10..2bfe197f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,11 @@ - Pinned the opt-in CPU runtime to PyTorch 2.13.0 and torchvision 0.28.0 from the official CPU wheel index, avoiding unused CUDA runtime packages while preserving the currently validated framework versions. - Kept AI dependencies opt-in and model weights local-only; no API, migration, model activation or inference contract changed. +## Sprint 171.1 Validation coverage provenance (2026-07-12) + +- Added explicit retained and empty validation sample lists to generated YOLO tile summaries. +- Quality-filtered holdouts such as Arendonk-heide remain visible in provenance without being counted as actual retained validation coverage. + ## Sprint 171 Positive AOI expansion and split safety (2026-07-12) - Added four explicit, real-reference Kempen training AOIs: Olen, Lille, Oud-Turnhout and Kasterlee center. diff --git a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py index 76924031..4651b293 100644 --- a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py +++ b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py @@ -100,6 +100,23 @@ def test_default_validation_split_is_explicit_and_rejects_holdout_leakage() -> N module.validate_validation_split(samples, {"turnhout", "missing"}) +def test_validation_coverage_reports_holdouts_without_retained_tiles() -> None: + module = load_tile_exporter() + coverage = module.validation_sample_coverage( + [ + {"sample_slug": "turnhout", "split": "val", "kept": True}, + {"sample_slug": "retie", "split": "val", "kept": True}, + {"sample_slug": "geel", "split": "train", "kept": True}, + ], + {"turnhout", "retie", "arendonk_heide"}, + ) + + assert coverage == { + "retained_validation_sample_slugs": ["retie", "turnhout"], + "empty_validation_sample_slugs": ["arendonk_heide"], + } + + def test_iter_tile_windows_covers_edges_without_duplicates() -> None: module = load_tile_exporter() diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 22ec06b2..cafe2307 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -7033,3 +7033,9 @@ Open: ## Remaining validation - Build the AI-enabled all-in-one image on Tower after the current inactive model training run finishes, verify Torch reports a CPU build and rerun live migration/browser smokes before replacing the runtime. + +# Sprint 171.1 - Validation coverage provenance + +- The expanded live export exposed that Arendonk-heide remained configured as a holdout while all of its low-variance tiles were correctly filtered out. +- Added `retained_validation_sample_slugs` and `empty_validation_sample_slugs` to tile dataset summaries so configured and actual validation coverage cannot be confused. +- Added a focused regression test and kept filtering behavior unchanged; no blank tile was reintroduced. diff --git a/scripts/README.md b/scripts/README.md index efc9e797..c9457056 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -366,6 +366,9 @@ The tile exporter clips GRB building bounding boxes into each tile, writes YOLO labels beside each tile image, keeps a deterministic ratio of empty negative tiles, and records `yolo_tile_dataset_summary.json` with `positive_tile_count`, `negative_tile_count` and skipped negative tile counts. +It separately records configured, retained and empty validation sample slugs; +this keeps a holdout that lost every tile to quality filtering visible without +pretending it contributed evaluation data. `--min-label-visible-ratio` drops labels where only a small clipped fragment of the original building bbox is visible inside the tile; this reduces noisy tile-edge labels in overlapping-tile datasets. Use `0` for legacy behavior. diff --git a/scripts/export_operator_yolo_tile_dataset.py b/scripts/export_operator_yolo_tile_dataset.py index aa4e8bce..19ad6362 100644 --- a/scripts/export_operator_yolo_tile_dataset.py +++ b/scripts/export_operator_yolo_tile_dataset.py @@ -181,6 +181,21 @@ def validate_validation_split(samples: list[dict[str, Any]], val_slugs: set[str] return val_slugs +def validation_sample_coverage( + tiles: list[dict[str, Any]], + val_slugs: set[str], +) -> dict[str, list[str]]: + retained = { + str(tile.get("sample_slug") or "").strip().lower() + for tile in tiles + if tile.get("kept", True) and str(tile.get("split") or "") == "val" + } + return { + "retained_validation_sample_slugs": sorted(retained), + "empty_validation_sample_slugs": sorted(val_slugs - retained), + } + + def edge_starts(length: int, tile_size: int, stride: int) -> list[int]: if tile_size <= 0: raise ValueError("tile_size must be positive") @@ -555,6 +570,7 @@ def main() -> int: for tile in skipped_negative_tiles if tile.get("skip_reason") == LOW_VARIANCE_NEGATIVE_SKIP_REASON ] + validation_coverage = validation_sample_coverage(kept_tiles, val_slugs) summary = { "status": "ok", "dataset_yaml": str(dataset_yaml), @@ -570,6 +586,7 @@ def main() -> int: "blank_range_threshold": args.blank_range_threshold, "source_sample_count": len(samples), "validation_sample_slugs": sorted(val_slugs), + **validation_coverage, "tile_count": len(kept_tiles), "positive_tile_count": len(positive_tiles), "negative_tile_count": len(negative_tiles),