From 8c9a58948ba7c05552448ee29994757a62e11217 Mon Sep 17 00:00:00 2001 From: Jens Date: Wed, 29 Jul 2026 17:21:18 +0200 Subject: [PATCH] Support governed train-only corpus shards --- ...st_sprint130_operator_yolo_tile_dataset.py | 9 +++++++ scripts/export_operator_yolo_tile_dataset.py | 26 ++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py index a813c6a6..ce4432bf 100644 --- a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py +++ b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py @@ -142,6 +142,15 @@ def test_manifest_sample_selection_keeps_external_holdouts_out_of_targeted_datas module.select_manifest_samples(samples, {"geel", "missing"}) +def test_train_only_shard_requires_explicit_opt_in() -> None: + module = load_tile_exporter() + samples = [{"sample_slug": "new-train-aoi", "recommended_split": "train"}] + + with pytest.raises(SystemExit, match="must include at least one"): + module.validate_validation_split(samples, set()) + assert module.validate_validation_split(samples, set(), allow_empty=True) == set() + + def test_validation_coverage_reports_holdouts_without_retained_tiles() -> None: module = load_tile_exporter() coverage = module.validation_sample_coverage( diff --git a/scripts/export_operator_yolo_tile_dataset.py b/scripts/export_operator_yolo_tile_dataset.py index 1dbccef6..bc772f7d 100644 --- a/scripts/export_operator_yolo_tile_dataset.py +++ b/scripts/export_operator_yolo_tile_dataset.py @@ -110,6 +110,14 @@ def parse_args() -> argparse.Namespace: "Defaults to all documented validation samples." ), ) + parser.add_argument( + "--allow-empty-validation", + action="store_true", + help=( + "Allow a train-only export shard with no validation AOI. This is only for later " + "composition into a dataset whose independent validation split is supplied separately." + ), + ) parser.add_argument( "--negative-keep-ratio", type=float, @@ -210,13 +218,20 @@ def select_manifest_samples( return selected, excluded -def validate_validation_split(samples: list[dict[str, Any]], val_slugs: set[str]) -> set[str]: +def validate_validation_split( + samples: list[dict[str, Any]], + val_slugs: set[str], + *, + allow_empty: bool = False, +) -> set[str]: sample_slugs = { str(sample.get("sample_slug") or "").strip().lower() for sample in samples if str(sample.get("sample_slug") or "").strip() } if not val_slugs: + if allow_empty: + return set() raise SystemExit("YOLO validation split must include at least one sample") unknown = val_slugs - sample_slugs if unknown: @@ -625,7 +640,11 @@ def main() -> int: manifest_samples, split_slugs(args.samples), ) - val_slugs = validate_validation_split(samples, split_slugs(args.val_samples)) + val_slugs = validate_validation_split( + samples, + split_slugs(args.val_samples), + allow_empty=args.allow_empty_validation, + ) exported_tiles: list[dict[str, Any]] = [] for sample in samples: exported_tiles.extend( @@ -650,7 +669,7 @@ def main() -> int: kept_tiles = [tile for tile in exported_tiles if tile["kept"]] if not any(tile["split"] == "train" for tile in kept_tiles): raise SystemExit("YOLO tile dataset export produced no training tiles") - if not any(tile["split"] == "val" for tile in kept_tiles): + if not args.allow_empty_validation and not any(tile["split"] == "val" for tile in kept_tiles): raise SystemExit("YOLO tile dataset export produced no validation tiles") dataset_yaml = write_dataset_yaml(args.output_dir, class_name) positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]] @@ -684,6 +703,7 @@ def main() -> int: ), "excluded_sample_slugs": excluded_sample_slugs, "validation_sample_slugs": sorted(val_slugs), + "train_only_shard": args.allow_empty_validation, **validation_coverage, "tile_count": len(kept_tiles), "positive_tile_count": len(positive_tiles),