Filter low variance YOLO negative tiles

scripts/README.md

@@ -367,6 +367,12 @@ negative tiles, and records `yolo_tile_dataset_summary.json` with
 `--min-label-visible-ratio` drops labels where only a small clipped fragment of
 the original building bbox is visible inside the tile; this reduces noisy
 tile-edge labels in overlapping-tile datasets. Use `0` for legacy behavior.
+Use `--drop-low-variance-negatives` to skip negative tiles whose rendered image
+has a max-min pixel range at or below `--blank-range-threshold`. This gate is
+intended for blank/no-data pure-empty negatives only; positive/labeled tiles are
+not removed by this filter. The summary records
+`skipped_low_variance_negative_tile_count` and skipped tile records with
+`skip_reason=low_visual_variance_negative`.
 For legacy operator manifests that predate explicit `background_category`, the
 exporter derives the same categories as the split-background evaluator:
 background samples with `reference_feature_count == 0` become
@@ -387,10 +393,16 @@ docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.
   --negative-keep-ratio 1.0 \
   --min-label-px 12 \
   --min-label-visible-ratio 0.35 \
+  --drop-low-variance-negatives \
+  --blank-range-threshold 3 \
   --val-samples turnhout,retie,westerlo,arendonk_heide \
   --force
 ```
 
+This refreshed cleanpx dataset is the minimum pre-training baseline after the
+visual contact-sheet pass found six blank-looking `arendonk_heide` validation
+negatives in the older export.
+
 Then audit with stricter small-box gates:
 
 ```bash

scripts/export_operator_yolo_tile_dataset.py

@@ -24,6 +24,7 @@ DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset
 REFERENCE_AOI_CATEGORY = "reference_aoi"
 PURE_EMPTY_BACKGROUND_CATEGORY = "pure_empty_negative"
 SPARSE_BACKGROUND_CATEGORY = "sparse_building_context"
+LOW_VARIANCE_NEGATIVE_SKIP_REASON = "low_visual_variance_negative"
 rasterio: Any = None
 Window: Any = None
 Transformer: Any = None
@@ -96,10 +97,32 @@ def parse_args() -> argparse.Namespace:
         default=int(os.environ.get("OPERATOR_YOLO_BACKGROUND_NEGATIVE_REPEAT", "1")),
         help="Repeat kept train/background negative tiles this many times for hard-negative balancing.",
     )
+    parser.add_argument(
+        "--drop-low-variance-negatives",
+        action=argparse.BooleanOptionalAction,
+        default=env_flag("OPERATOR_YOLO_DROP_LOW_VARIANCE_NEGATIVES", default=False),
+        help=(
+            "Skip negative tiles whose rendered image has very low pixel variance. "
+            "This is intended for no-data/blank pure-empty negatives only."
+        ),
+    )
+    parser.add_argument(
+        "--blank-range-threshold",
+        type=int,
+        default=int(os.environ.get("OPERATOR_YOLO_BLANK_RANGE_THRESHOLD", "3")),
+        help="Max pixel value range used to classify a negative tile as visually blank/low-variance.",
+    )
     parser.add_argument("--force", action="store_true", help="Remove and recreate output-dir before exporting.")
     return parser.parse_args()
 
 
+def env_flag(name: str, *, default: bool) -> bool:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
 def ensure_dependencies() -> None:
     global Image, Transformer, Window, rasterio
     try:
@@ -293,6 +316,17 @@ def image_array_from_raster_window(dataset: Any, tile_window: TileWindow) -> Any
     return rgb
 
 
+def image_array_has_low_visual_variance(image_array: Any, blank_range_threshold: int) -> bool:
+    import numpy as np
+
+    array = np.asarray(image_array)
+    if array.size == 0:
+        return True
+    max_value = float(np.nanmax(array))
+    min_value = float(np.nanmin(array))
+    return (max_value - min_value) <= blank_range_threshold
+
+
 def ensure_yolo_directories(output_dir: Path) -> None:
     for relative_path in ("images/train", "labels/train", "images/val", "labels/val"):
         (output_dir / relative_path).mkdir(parents=True, exist_ok=True)
@@ -327,6 +361,8 @@ def export_sample_tiles(
     min_label_px: float,
     min_label_visible_ratio: float,
     background_negative_repeat: int,
+    drop_low_variance_negatives: bool,
+    blank_range_threshold: int,
 ) -> list[dict[str, Any]]:
     sample_slug = str(sample["sample_slug"])
     sample_role = str(sample.get("sample_role") or "reference")
@@ -359,6 +395,34 @@ def export_sample_tiles(
                         "kept": False,
                         "label_count": 0,
                         "is_negative": True,
+                        "skip_reason": "negative_keep_ratio",
+                    }
+                )
+                continue
+            image_array = image_array_from_raster_window(dataset, tile_window)
+            low_visual_variance = image_array_has_low_visual_variance(
+                image_array,
+                blank_range_threshold=blank_range_threshold,
+            )
+            if is_negative and drop_low_variance_negatives and low_visual_variance:
+                exported.append(
+                    {
+                        "sample_slug": sample_slug,
+                        "sample_role": sample_role,
+                        "background_category": background_category,
+                        "split": split,
+                        "tile_index": tile_index,
+                        "kept": False,
+                        "label_count": 0,
+                        "is_negative": True,
+                        "low_visual_variance": True,
+                        "skip_reason": LOW_VARIANCE_NEGATIVE_SKIP_REASON,
+                        "window": {
+                            "row_off": tile_window.row_off,
+                            "col_off": tile_window.col_off,
+                            "height": tile_window.height,
+                            "width": tile_window.width,
+                        },
                     }
                 )
                 continue
@@ -368,7 +432,6 @@ def export_sample_tiles(
                 split=split,
                 background_negative_repeat=background_negative_repeat,
             )
-            image_array = image_array_from_raster_window(dataset, tile_window)
             for repeat_index in range(repeats):
                 repeat_suffix = f"_hn{repeat_index + 1:02d}" if repeats > 1 else ""
                 tile_name = f"{sample_slug}_{tile_index:04d}_r{tile_window.row_off}_c{tile_window.col_off}{repeat_suffix}"
@@ -391,6 +454,7 @@ def export_sample_tiles(
                         "label_path": str(label_path),
                         "label_count": len(labels),
                         "is_negative": is_negative,
+                        "low_visual_variance": low_visual_variance,
                         "is_repeated_background_negative": repeats > 1,
                         "window": {
                             "row_off": tile_window.row_off,
@@ -430,6 +494,8 @@ def main() -> int:
                 min_label_px=args.min_label_px,
                 min_label_visible_ratio=args.min_label_visible_ratio,
                 background_negative_repeat=args.background_negative_repeat,
+                drop_low_variance_negatives=args.drop_low_variance_negatives,
+                blank_range_threshold=args.blank_range_threshold,
             )
         )
 
@@ -442,6 +508,11 @@ def main() -> int:
     positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
     negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
     skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]]
+    skipped_low_variance_negative_tiles = [
+        tile
+        for tile in skipped_negative_tiles
+        if tile.get("skip_reason") == LOW_VARIANCE_NEGATIVE_SKIP_REASON
+    ]
     summary = {
         "status": "ok",
         "dataset_yaml": str(dataset_yaml),
@@ -453,11 +524,14 @@ def main() -> int:
         "background_negative_repeat": args.background_negative_repeat,
         "min_label_px": args.min_label_px,
         "min_label_visible_ratio": args.min_label_visible_ratio,
+        "drop_low_variance_negatives": args.drop_low_variance_negatives,
+        "blank_range_threshold": args.blank_range_threshold,
         "source_sample_count": len(samples),
         "tile_count": len(kept_tiles),
         "positive_tile_count": len(positive_tiles),
         "negative_tile_count": len(negative_tiles),
         "skipped_negative_tile_count": len(skipped_negative_tiles),
+        "skipped_low_variance_negative_tile_count": len(skipped_low_variance_negative_tiles),
         "label_count": sum(tile["label_count"] for tile in kept_tiles),
         "train_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "train"),
         "val_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "val"),