Expand YOLO training AOIs safely

scripts/README.md

@@ -405,6 +405,43 @@ This refreshed cleanpx dataset is the minimum pre-training baseline after the
 visual contact-sheet pass found six blank-looking `arendonk_heide` validation
 negatives in the older export.
 
+The persisted false-negative audit subsequently showed that the cleanpx12
+candidate still misses about 79-92% of the comparable reference population and
+misses every reference building below 25 m2 in the seven-AOI review. Do not
+train another candidate from the same four positive training AOIs. Refresh the
+existing AOI1024 sample directory after pulling Sprint 171; existing files are
+reused and only the four new explicit positive AOIs need to be fetched:
+
+```bash
+docker exec -it geointel python3 /app/scripts/prepare_operator_real_data_samples.py \
+  --output-dir /app/storage/operator-data/operator-samples-1024 \
+  --width 1024 \
+  --height 1024 \
+  --half-size-scale 2
+```
+
+The expansion adds Olen, Lille, Oud-Turnhout and Kasterlee center as training
+samples. Turnhout, Retie, Westerlo and Arendonk-heide remain explicit validation
+holdouts in generated manifest provenance. The tile exporter defaults to those
+four holdouts and rejects a manifest-aware split that leaks one into training.
+Use the lower `min-label-px=4` profile first to measure small-building retention;
+it remains subject to dataset audit and visual contact-sheet review before any
+training:
+
+```bash
+docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \
+  --manifest-path /app/storage/operator-data/operator-samples-1024/operator_samples_manifest.json \
+  --output-dir /app/storage/operator-data/yolo-building-aoi1024-expanded-minpx4vis035 \
+  --tile-size 512 \
+  --stride 256 \
+  --negative-keep-ratio 1.0 \
+  --min-label-px 4 \
+  --min-label-visible-ratio 0.35 \
+  --drop-low-variance-negatives \
+  --blank-range-threshold 3 \
+  --force
+```
+
 Then audit with stricter small-box gates:
 
 ```bash

scripts/audit_detection_false_negative_evidence.py

@@ -116,6 +116,7 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) ->
         raise SystemExit("Evidence GeoJSON must be a FeatureCollection")
     features = payload.get("features") or []
     false_negative_ids: set[str] = set()
+    reference_ids: set[str] = set()
     false_negative_areas: list[float] = []
     matched_reference_areas: list[float] = []
     bucket_counts = {
@@ -141,11 +142,13 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) ->
             )
         area_m2 = abs(float(geod.geometry_area_perimeter(geometry)[0]))
         bucket = area_bucket(area_m2)
+        reference_id = stable_reference_id(feature, geometry)
+        reference_ids.add(reference_id)
         bucket_role = "false_negative" if role == "false_negative" else "matched_reference"
         bucket_counts[bucket][bucket_role] += 1
         bucket_counts[bucket]["total_reference"] += 1
         if role == "false_negative":
-            false_negative_ids.add(stable_reference_id(feature, geometry))
+            false_negative_ids.add(reference_id)
             false_negative_areas.append(area_m2)
         else:
             matched_reference_areas.append(area_m2)
@@ -157,6 +160,7 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) ->
     total_reference = len(false_negative_areas) + len(matched_reference_areas)
     return {
         "false_negative_ids": false_negative_ids,
+        "reference_ids": reference_ids,
         "false_negative_count": len(false_negative_areas),
         "matched_reference_count": len(matched_reference_areas),
         "total_reference_count": total_reference,
@@ -284,16 +288,31 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]:
     for sample_slug in sorted(expected_slugs):
         portfolio_rows = []
         false_negative_sets = []
+        reference_sets = []
         for label, _ in parsed_portfolios:
             raw = portfolio_samples[label][sample_slug]
             false_negative_sets.append(raw["false_negative_ids"])
+            reference_sets.append(raw["reference_ids"])
             portfolio_rows.append(
-                {key: value for key, value in raw.items() if key != "false_negative_ids"}
+                {
+                    key: value
+                    for key, value in raw.items()
+                    if key not in {"false_negative_ids", "reference_ids"}
+                }
+            )
+        if any(reference_ids != reference_sets[0] for reference_ids in reference_sets[1:]):
+            counts = ", ".join(
+                f"{label}={len(reference_ids)}"
+                for (label, _), reference_ids in zip(parsed_portfolios, reference_sets, strict=True)
+            )
+            raise SystemExit(
+                f"Sample {sample_slug} has different reference populations across portfolios ({counts})"
             )
         persistent_ids = sorted(set.intersection(*false_negative_sets))
         sample_reports.append(
             {
                 "sample_slug": sample_slug,
+                "reference_population_count": len(reference_sets[0]),
                 "persistent_false_negative_count": len(persistent_ids),
                 "persistent_reference_ids": persistent_ids,
                 "portfolios": portfolio_rows,

scripts/export_operator_yolo_tile_dataset.py

@@ -25,6 +25,10 @@ REFERENCE_AOI_CATEGORY = "reference_aoi"
 PURE_EMPTY_BACKGROUND_CATEGORY = "pure_empty_negative"
 SPARSE_BACKGROUND_CATEGORY = "sparse_building_context"
 LOW_VARIANCE_NEGATIVE_SKIP_REASON = "low_visual_variance_negative"
+DEFAULT_VALIDATION_SAMPLE_SLUGS = frozenset(
+    {"turnhout", "retie", "westerlo", "arendonk_heide"}
+)
+DEFAULT_VALIDATION_SAMPLES = ",".join(sorted(DEFAULT_VALIDATION_SAMPLE_SLUGS))
 rasterio: Any = None
 Window: Any = None
 Transformer: Any = None
@@ -67,8 +71,11 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--stride", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_STRIDE", "128")))
     parser.add_argument(
         "--val-samples",
-        default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"),
-        help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.",
+        default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", DEFAULT_VALIDATION_SAMPLES),
+        help=(
+            "Comma/space separated sample slugs assigned to validation. "
+            "Defaults to the documented Turnhout, Retie, Westerlo and Arendonk-heide holdouts."
+        ),
     )
     parser.add_argument(
         "--negative-keep-ratio",
@@ -145,6 +152,35 @@ def split_slugs(raw: str) -> set[str]:
     return {value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()}
 
 
+def validate_validation_split(samples: list[dict[str, Any]], val_slugs: set[str]) -> set[str]:
+    sample_slugs = {
+        str(sample.get("sample_slug") or "").strip().lower()
+        for sample in samples
+        if str(sample.get("sample_slug") or "").strip()
+    }
+    if not val_slugs:
+        raise SystemExit("YOLO validation split must include at least one sample")
+    unknown = val_slugs - sample_slugs
+    if unknown:
+        raise SystemExit(
+            "YOLO validation split references unknown samples: " + ", ".join(sorted(unknown))
+        )
+    recommended_holdouts = {
+        str(sample.get("sample_slug") or "").strip().lower()
+        for sample in samples
+        if str(sample.get("recommended_split") or "").strip().lower() == "val"
+    }
+    missing_holdouts = recommended_holdouts - val_slugs
+    if missing_holdouts:
+        raise SystemExit(
+            "YOLO validation split omits recommended validation holdouts: "
+            + ", ".join(sorted(missing_holdouts))
+        )
+    if not sample_slugs - val_slugs:
+        raise SystemExit("YOLO validation split leaves no training samples")
+    return val_slugs
+
+
 def edge_starts(length: int, tile_size: int, stride: int) -> list[int]:
     if tile_size <= 0:
         raise ValueError("tile_size must be positive")
@@ -367,6 +403,7 @@ def export_sample_tiles(
     sample_slug = str(sample["sample_slug"])
     sample_role = str(sample.get("sample_role") or "reference")
     background_category = background_category_for_sample(sample)
+    recommended_split = str(sample.get("recommended_split") or "")
     split = "val" if sample_slug.lower() in val_slugs else "train"
     raster_path = resolve_manifest_path(str(sample["raster_path"]), manifest_path)
     reference_path = resolve_manifest_path(str(sample["reference_path"]), manifest_path)
@@ -390,6 +427,9 @@ def export_sample_tiles(
                 exported.append(
                     {
                         "sample_slug": sample_slug,
+                        "sample_role": sample_role,
+                        "background_category": background_category,
+                        "recommended_split": recommended_split,
                         "split": split,
                         "tile_index": tile_index,
                         "kept": False,
@@ -410,6 +450,7 @@ def export_sample_tiles(
                         "sample_slug": sample_slug,
                         "sample_role": sample_role,
                         "background_category": background_category,
+                        "recommended_split": recommended_split,
                         "split": split,
                         "tile_index": tile_index,
                         "kept": False,
@@ -446,6 +487,7 @@ def export_sample_tiles(
                         "sample_slug": sample_slug,
                         "sample_role": sample_role,
                         "background_category": background_category,
+                        "recommended_split": recommended_split,
                         "split": split,
                         "tile_index": tile_index,
                         "repeat_index": repeat_index,
@@ -479,7 +521,7 @@ def main() -> int:
     samples = manifest.get("samples") or []
     if not samples:
         raise SystemExit("Operator sample manifest contains no samples")
-    val_slugs = split_slugs(args.val_samples)
+    val_slugs = validate_validation_split(samples, split_slugs(args.val_samples))
     exported_tiles: list[dict[str, Any]] = []
     for sample in samples:
         exported_tiles.extend(
@@ -527,6 +569,7 @@ def main() -> int:
         "drop_low_variance_negatives": args.drop_low_variance_negatives,
         "blank_range_threshold": args.blank_range_threshold,
         "source_sample_count": len(samples),
+        "validation_sample_slugs": sorted(val_slugs),
         "tile_count": len(kept_tiles),
         "positive_tile_count": len(positive_tiles),
         "negative_tile_count": len(negative_tiles),

scripts/prepare_operator_real_data_samples.py

@@ -25,6 +25,12 @@ DEFAULT_GRB_MAX_FEATURES = 100000
 REFERENCE_AOI_CATEGORY = "reference_aoi"
 PURE_EMPTY_BACKGROUND_CATEGORY = "pure_empty_negative"
 SPARSE_BACKGROUND_CATEGORY = "sparse_building_context"
+TRAINING_EXPANSION_SAMPLE_SLUGS = frozenset(
+    {"olen_center", "lille_center", "oud_turnhout_center", "kasterlee_center"}
+)
+DEFAULT_VALIDATION_SAMPLE_SLUGS = frozenset(
+    {"turnhout", "retie", "westerlo", "arendonk_heide"}
+)
 requests: Any = None
 rasterio: Any = None
 Transformer: Any = None
@@ -92,6 +98,30 @@ SAMPLES: dict[str, OperatorSample] = {
         center_lat=51.0909,
         half_size_m=220.0,
     ),
+    "olen_center": OperatorSample(
+        slug="olen_center",
+        display_name="Olen center training expansion",
+        center_lon=4.8597257,
+        center_lat=51.1438611,
+    ),
+    "lille_center": OperatorSample(
+        slug="lille_center",
+        display_name="Lille center training expansion",
+        center_lon=4.8242404,
+        center_lat=51.2382180,
+    ),
+    "oud_turnhout_center": OperatorSample(
+        slug="oud_turnhout_center",
+        display_name="Oud-Turnhout center training expansion",
+        center_lon=4.9817086,
+        center_lat=51.3178319,
+    ),
+    "kasterlee_center": OperatorSample(
+        slug="kasterlee_center",
+        display_name="Kasterlee center training expansion",
+        center_lon=4.9678120,
+        center_lat=51.2407915,
+    ),
     "postel_bos": OperatorSample(
         slug="postel_bos",
         display_name="Postel forest background candidate",
@@ -270,6 +300,10 @@ def background_category_for_sample(sample: OperatorSample, reference_feature_cou
     return PURE_EMPTY_BACKGROUND_CATEGORY if reference_feature_count <= 0 else SPARSE_BACKGROUND_CATEGORY
 
 
+def recommended_split_for_sample(sample: OperatorSample) -> str:
+    return "val" if sample.slug in DEFAULT_VALIDATION_SAMPLE_SLUGS else "train"
+
+
 def apply_sample_overrides(
     sample: OperatorSample,
     *,
@@ -481,6 +515,7 @@ def fetch_reference(
     reference["sample_role"] = sample.sample_role
     reference["allow_empty_reference"] = sample.allow_empty_reference
     reference["background_category"] = background_category_for_sample(sample, len(features))
+    reference["recommended_split"] = recommended_split_for_sample(sample)
     reference["reference_page_limit"] = page_limit
     reference["reference_max_features"] = max_features
     reference["reference_pages_fetched"] = len(pages)
@@ -495,6 +530,7 @@ def fetch_reference(
         props.setdefault("sample_slug", sample.slug)
         props.setdefault("sample_role", sample.sample_role)
         props.setdefault("background_category", background_category_for_sample(sample, len(features)))
+        props.setdefault("recommended_split", recommended_split_for_sample(sample))
 
     reference_path.write_text(json.dumps(reference, ensure_ascii=False), encoding="utf-8")
     return prepared_url(GRB_GBG_URL, ogc_params), len(features)
@@ -537,6 +573,7 @@ def prepare_sample(
         "sample_role": sample.sample_role,
         "allow_empty_reference": sample.allow_empty_reference,
         "background_category": background_category,
+        "recommended_split": recommended_split_for_sample(sample),
         "raster_path": str(ortho_path),
         "reference_path": str(reference_path),
         "reference_feature_count": reference_feature_count,
@@ -572,7 +609,8 @@ def write_readme(output_dir: Path, samples: list[dict[str, Any]]) -> None:
             f"- `{sample['sample_slug']}`: `{Path(sample['raster_path']).name}` and "
             f"`{Path(sample['reference_path']).name}`, "
             f"{sample['reference_feature_count']} reference features, role `{sample['sample_role']}`, "
-            f"background category `{sample['background_category']}`."
+            f"background category `{sample['background_category']}`, "
+            f"recommended split `{sample['recommended_split']}`."
         )
     lines.append("")
     lines.append("Purpose: configured-YOLO detection + persisted QA/QC validation with operator-provided files.")
@@ -602,7 +640,7 @@ def main() -> int:
     write_readme(output_dir, samples)
 
     manifest = {
-        "schema_version": 1,
+        "schema_version": 2,
         "description": "GeoIntel operator real-data samples for configured-YOLO QA validation.",
         "output_dir": str(output_dir),
         "sample_width": args.width,
@@ -610,6 +648,8 @@ def main() -> int:
         "half_size_scale": args.half_size_scale,
         "reference_page_limit": args.reference_page_limit,
         "reference_max_features": args.reference_max_features,
+        "default_validation_sample_slugs": sorted(DEFAULT_VALIDATION_SAMPLE_SLUGS),
+        "training_expansion_sample_slugs": sorted(TRAINING_EXPANSION_SAMPLE_SLUGS),
         "samples": samples,
     }
     manifest_path = output_dir / args.manifest_name