Filter low variance YOLO negative tiles

backend/tests/test_sprint130_operator_yolo_tile_dataset.py

@@ -5,6 +5,8 @@ from pathlib import Path
 import subprocess
 import sys
 
+import numpy as np
+
 
 ROOT = Path(__file__).resolve().parents[2]
 
@@ -39,6 +41,8 @@ def test_operator_yolo_tile_dataset_export_script_contract() -> None:
     assert "stride" in script
     assert "negative_keep_ratio" in script
     assert "min_label_visible_ratio" in script
+    assert "drop_low_variance_negatives" in script
+    assert "skipped_low_variance_negative_tile_count" in script
     assert "positive_tile_count" in script
     assert "negative_tile_count" in script
     assert "skipped_negative_tile_count" in script
@@ -67,6 +71,8 @@ def test_operator_yolo_tile_dataset_export_help_does_not_require_gis_dependencie
     assert "--negative-keep-ratio" in result.stdout
     assert "--min-label-visible-ratio" in result.stdout
     assert "--background-negative-repeat" in result.stdout
+    assert "--drop-low-variance-negatives" in result.stdout
+    assert "--blank-range-threshold" in result.stdout
 
 
 def test_iter_tile_windows_covers_edges_without_duplicates() -> None:
@@ -171,3 +177,86 @@ def test_background_category_is_derived_for_legacy_operator_manifests() -> None:
     assert module.background_category_for_sample(pure_empty_sample) == "pure_empty_negative"
     assert module.background_category_for_sample(sparse_context_sample) == "sparse_building_context"
     assert module.background_category_for_sample(reference_sample) == "reference_aoi"
+
+
+def test_export_can_skip_low_variance_negative_tiles(tmp_path: Path, monkeypatch) -> None:
+    module = load_tile_exporter()
+    raster_path = tmp_path / "sample.tif"
+    reference_path = tmp_path / "reference.geojson"
+    raster_path.write_bytes(b"fake-raster")
+    reference_path.write_text('{"type": "FeatureCollection", "features": []}', encoding="utf-8")
+
+    class FakeDataset:
+        width = 256
+        height = 128
+        crs = "EPSG:31370"
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc, traceback):
+            return False
+
+    class FakeRasterio:
+        @staticmethod
+        def open(path):
+            assert Path(path) == raster_path
+            return FakeDataset()
+
+    class FakeImageObject:
+        def __init__(self, array):
+            self.array = array
+
+        def save(self, path):
+            Path(path).write_bytes(b"png")
+
+    class FakeImage:
+        @staticmethod
+        def fromarray(array):
+            return FakeImageObject(array)
+
+    def fake_image_array_from_raster_window(dataset, tile_window):
+        if tile_window.col_off == 0:
+            return np.full((128, 128, 3), 255, dtype=np.uint8)
+        image = np.zeros((128, 128, 3), dtype=np.uint8)
+        image[:, 64:, :] = 80
+        return image
+
+    monkeypatch.setattr(module, "rasterio", FakeRasterio)
+    monkeypatch.setattr(module, "Image", FakeImage)
+    monkeypatch.setattr(module, "load_reference_pixel_boxes", lambda reference_path, dataset, min_label_px: [])
+    monkeypatch.setattr(module, "image_array_from_raster_window", fake_image_array_from_raster_window)
+
+    records = module.export_sample_tiles(
+        sample={
+            "sample_slug": "blank_negative",
+            "sample_role": "background_candidate",
+            "background_category": "pure_empty_negative",
+            "raster_path": str(raster_path),
+            "reference_path": str(reference_path),
+        },
+        manifest_path=tmp_path / "operator_samples_manifest.json",
+        output_dir=tmp_path / "dataset",
+        val_slugs=set(),
+        tile_size=128,
+        stride=128,
+        negative_keep_ratio=1.0,
+        min_label_px=4,
+        min_label_visible_ratio=0.0,
+        background_negative_repeat=1,
+        drop_low_variance_negatives=True,
+        blank_range_threshold=3,
+    )
+
+    skipped = [record for record in records if not record["kept"]]
+    kept = [record for record in records if record["kept"]]
+
+    assert len(skipped) == 1
+    assert skipped[0]["skip_reason"] == "low_visual_variance_negative"
+    assert skipped[0]["low_visual_variance"] is True
+    assert skipped[0]["is_negative"] is True
+    assert skipped[0]["tile_index"] == 0
+    assert len(kept) == 1
+    assert kept[0]["tile_index"] == 1
+    assert kept[0]["low_visual_variance"] is False
+    assert Path(kept[0]["image_path"]).exists()

docs/CODEX_EXECUTION_LOG.md

@@ -6866,3 +6866,41 @@ Open:
 ## Next recommended pass
 
 - Add no-data/low-variance filtering to the operator YOLO tile export path, regenerate the clean AOI1024 dataset, rerun the contact-sheet QA, and only then consider another training attempt.
+
+# Sprint 168 - Operator YOLO low-variance negative filtering
+
+## What changed
+
+- Added opt-in low-variance negative filtering to `scripts/export_operator_yolo_tile_dataset.py`.
+- Added CLI/env controls:
+  - `--drop-low-variance-negatives` / `OPERATOR_YOLO_DROP_LOW_VARIANCE_NEGATIVES`;
+  - `--blank-range-threshold` / `OPERATOR_YOLO_BLANK_RANGE_THRESHOLD`.
+- The filter evaluates the rendered raster tile image and skips only negative tiles when enabled.
+- Positive/labeled tiles are never removed by this variance gate.
+- Kept tile records now include `low_visual_variance`.
+- Skipped blank/no-data negative records use `skip_reason="low_visual_variance_negative"`.
+- Dataset summaries now include:
+  - `drop_low_variance_negatives`;
+  - `blank_range_threshold`;
+  - `skipped_low_variance_negative_tile_count`.
+- Updated operator documentation with the refreshed AOI1024 cleanpx export command.
+- Added design and execution plan docs under `docs/superpowers/`.
+
+## Local validation
+
+- RED: `python -m pytest backend/tests/test_sprint130_operator_yolo_tile_dataset.py::test_export_can_skip_low_variance_negative_tiles -q` failed because `export_sample_tiles()` did not accept `drop_low_variance_negatives`.
+- GREEN: same targeted test passed after adding the filter.
+- Ran `python -m pytest backend/tests/test_sprint130_operator_yolo_tile_dataset.py -q`: 8 passed.
+- Ran `python -m pytest backend/tests/test_sprint130_operator_yolo_tile_dataset.py backend/tests/test_sprint167_operator_yolo_label_qa_contact_sheets.py backend/tests/test_docker_runtime_config.py::test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use backend/tests/test_docker_runtime_config.py::test_all_in_one_dockerfile_copies_operator_scripts_after_dependency_install -q`: 11 passed.
+- Ran `python scripts/export_operator_yolo_tile_dataset.py --help`: the CLI exposes `--drop-low-variance-negatives`, `--no-drop-low-variance-negatives` and `--blank-range-threshold` without loading GIS dependencies.
+- Ran `bash scripts/run_readiness_check.sh`: 460 backend tests passed, frontend typecheck passed, frontend build passed, readiness passed.
+
+## Known limitations
+
+- The low-variance gate is deliberately simple and only identifies visually blank/no-data-looking negative tiles.
+- It is opt-in to avoid silently changing historical dataset exports.
+- Operator visual contact-sheet review remains required before any new training run.
+
+## Next recommended pass
+
+- Redeploy Tower, regenerate the AOI1024 cleanpx dataset with `--drop-low-variance-negatives`, rerun dataset audit and contact-sheet QA, then decide whether the filtered dataset is suitable for another training run.

docs/TODO.md

@@ -131,7 +131,8 @@ This file now starts with the current implementation status. Older preparation/b
 - [x] Add guarded promoted-candidate activation helper requiring a promotion report path and exact candidate key before `.env` can be changed.
 - [x] Add per-sample YOLO dataset audit diagnostics for parsed labels, median box area, small-box share and AOI-specific warning codes.
 - [x] Add deterministic visual YOLO label QA contact sheets before spending more CPU on another training run.
-- [ ] Filter no-data/low-variance pure-empty negative tiles from operator YOLO exports before the next training run.
+- [x] Filter no-data/low-variance pure-empty negative tiles from operator YOLO exports before the next training run.
+- [ ] Regenerate the AOI1024 cleanpx YOLO dataset with low-variance negative filtering and rerun visual contact-sheet QA before training.
 - [ ] Apply promoted V1 default building detector only after explicit operator review of the emitted `.env` updates, followed by rebuild/restart and browser/runtime smoke.
 
 ## Sprint 8 status

docs/superpowers/plans/2026-07-11-yolo-low-variance-negative-filter.md

@@ -0,0 +1,56 @@
+# YOLO Low-Variance Negative Filter Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Add opt-in filtering for blank/low-variance negative tiles in the operator YOLO tile exporter.
+
+**Architecture:** Keep the behavior inside `scripts/export_operator_yolo_tile_dataset.py` because this is operator-only dataset construction, not application inference. Compute low-variance from the raster window image array, skip only negative tiles when explicitly enabled, and expose all decisions in `yolo_tile_dataset_summary.json`.
+
+**Tech Stack:** Python, pytest, rasterio/Pillow runtime helpers already used by the exporter.
+
+---
+
+### Task 1: Regression Test
+
+**Files:**
+- Modify: `backend/tests/test_sprint130_operator_yolo_tile_dataset.py`
+
+- [x] Add a test that monkeypatches `image_array_from_raster_window` and verifies `export_sample_tiles` skips only low-variance negative tiles when `drop_low_variance_negatives=True`.
+- [x] Run the targeted test and confirm it fails because the exporter does not yet accept/report the new filter fields.
+
+### Task 2: Exporter Implementation
+
+**Files:**
+- Modify: `scripts/export_operator_yolo_tile_dataset.py`
+
+- [x] Add CLI/env options:
+  - `--drop-low-variance-negatives`
+  - `--blank-range-threshold`
+- [x] Add a helper to detect low visual variance from an image array.
+- [x] Thread the options into `export_sample_tiles`.
+- [x] Skip only negative low-variance tiles when the option is enabled.
+- [x] Add `low_visual_variance` to kept tile records.
+- [x] Add skipped records with `skip_reason="low_visual_variance_negative"`.
+- [x] Add summary fields for the option, threshold and skipped count.
+
+### Task 3: Documentation
+
+**Files:**
+- Modify: `scripts/README.md`
+- Modify: `docs/CODEX_EXECUTION_LOG.md`
+- Modify: `docs/TODO.md`
+
+- [x] Document the new operator export flags and intended Tower command.
+- [x] Record local validation and known limitation.
+- [x] Mark the low-variance export filter item as implemented after validation.
+
+### Task 4: Validation and Handoff
+
+**Files:**
+- No additional file changes expected.
+
+- [x] Run targeted pytest for the exporter/contact-sheet tests.
+- [x] Run `bash scripts/run_readiness_check.sh`.
+- [ ] Commit and push.
+- [ ] Rebuild/deploy Tower if code changed.
+- [ ] Regenerate the AOI1024 dataset with the new filter enabled, then render contact sheets and record the result.

docs/superpowers/specs/2026-07-11-yolo-low-variance-negative-filter-design.md

@@ -0,0 +1,34 @@
+# YOLO Low-Variance Negative Filter Design
+
+## Goal
+
+Prevent blank/no-data pure-empty negative tiles from entering the next operator YOLO training dataset while keeping the exporter conservative and auditable.
+
+## Scope
+
+This pass changes only operator-side YOLO tile export tooling. It does not change backend APIs, database schema, frontend behavior, model activation, training behavior, provider fetching or inference.
+
+## Approach
+
+Add an opt-in filter to `scripts/export_operator_yolo_tile_dataset.py` that evaluates raster tile image variance before writing negative tiles. The filter applies only after labels are computed and only when a tile is negative. Positive tiles are never dropped by this gate, even if visually low-variance, because dropping labeled data silently would be a worse failure mode.
+
+The filter will use the same simple max-min grayscale range heuristic as the contact-sheet QA script. A tile with range at or below `OPERATOR_YOLO_BLANK_RANGE_THRESHOLD` is treated as low-variance. When `--drop-low-variance-negatives` is enabled, that negative tile is skipped and recorded in summary fields instead of being written into `images/` and `labels/`.
+
+## Reporting
+
+The dataset summary must include:
+
+- `drop_low_variance_negatives`
+- `blank_range_threshold`
+- `skipped_low_variance_negative_tile_count`
+- skipped tile records with `skip_reason="low_visual_variance_negative"`
+
+Kept tile records should include `low_visual_variance` so downstream contact-sheet and audit tooling can expose the signal.
+
+## Validation
+
+Add a regression test that constructs a blank negative raster window and a patterned negative raster window, enables the filter, and verifies that only the blank negative is skipped for `low_visual_variance_negative`.
+
+## Known Limitation
+
+The heuristic is intentionally simple. It identifies blank/no-data tiles, not semantic quality. Operator visual QA remains required before another training run.

scripts/README.md

@@ -367,6 +367,12 @@ negative tiles, and records `yolo_tile_dataset_summary.json` with
 `--min-label-visible-ratio` drops labels where only a small clipped fragment of
 the original building bbox is visible inside the tile; this reduces noisy
 tile-edge labels in overlapping-tile datasets. Use `0` for legacy behavior.
+Use `--drop-low-variance-negatives` to skip negative tiles whose rendered image
+has a max-min pixel range at or below `--blank-range-threshold`. This gate is
+intended for blank/no-data pure-empty negatives only; positive/labeled tiles are
+not removed by this filter. The summary records
+`skipped_low_variance_negative_tile_count` and skipped tile records with
+`skip_reason=low_visual_variance_negative`.
 For legacy operator manifests that predate explicit `background_category`, the
 exporter derives the same categories as the split-background evaluator:
 background samples with `reference_feature_count == 0` become
@@ -387,10 +393,16 @@ docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.
   --negative-keep-ratio 1.0 \
   --min-label-px 12 \
   --min-label-visible-ratio 0.35 \
+  --drop-low-variance-negatives \
+  --blank-range-threshold 3 \
   --val-samples turnhout,retie,westerlo,arendonk_heide \
   --force
 ```
 
+This refreshed cleanpx dataset is the minimum pre-training baseline after the
+visual contact-sheet pass found six blank-looking `arendonk_heide` validation
+negatives in the older export.
+
 Then audit with stricter small-box gates:
 
 ```bash

scripts/export_operator_yolo_tile_dataset.py

@@ -24,6 +24,7 @@ DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset
 REFERENCE_AOI_CATEGORY = "reference_aoi"
 PURE_EMPTY_BACKGROUND_CATEGORY = "pure_empty_negative"
 SPARSE_BACKGROUND_CATEGORY = "sparse_building_context"
+LOW_VARIANCE_NEGATIVE_SKIP_REASON = "low_visual_variance_negative"
 rasterio: Any = None
 Window: Any = None
 Transformer: Any = None
@@ -96,10 +97,32 @@ def parse_args() -> argparse.Namespace:
         default=int(os.environ.get("OPERATOR_YOLO_BACKGROUND_NEGATIVE_REPEAT", "1")),
         help="Repeat kept train/background negative tiles this many times for hard-negative balancing.",
     )
+    parser.add_argument(
+        "--drop-low-variance-negatives",
+        action=argparse.BooleanOptionalAction,
+        default=env_flag("OPERATOR_YOLO_DROP_LOW_VARIANCE_NEGATIVES", default=False),
+        help=(
+            "Skip negative tiles whose rendered image has very low pixel variance. "
+            "This is intended for no-data/blank pure-empty negatives only."
+        ),
+    )
+    parser.add_argument(
+        "--blank-range-threshold",
+        type=int,
+        default=int(os.environ.get("OPERATOR_YOLO_BLANK_RANGE_THRESHOLD", "3")),
+        help="Max pixel value range used to classify a negative tile as visually blank/low-variance.",
+    )
     parser.add_argument("--force", action="store_true", help="Remove and recreate output-dir before exporting.")
     return parser.parse_args()
 
 
+def env_flag(name: str, *, default: bool) -> bool:
+    raw = os.environ.get(name)
+    if raw is None:
+        return default
+    return raw.strip().lower() in {"1", "true", "yes", "on"}
+
+
 def ensure_dependencies() -> None:
     global Image, Transformer, Window, rasterio
     try:
@@ -293,6 +316,17 @@ def image_array_from_raster_window(dataset: Any, tile_window: TileWindow) -> Any
     return rgb
 
 
+def image_array_has_low_visual_variance(image_array: Any, blank_range_threshold: int) -> bool:
+    import numpy as np
+
+    array = np.asarray(image_array)
+    if array.size == 0:
+        return True
+    max_value = float(np.nanmax(array))
+    min_value = float(np.nanmin(array))
+    return (max_value - min_value) <= blank_range_threshold
+
+
 def ensure_yolo_directories(output_dir: Path) -> None:
     for relative_path in ("images/train", "labels/train", "images/val", "labels/val"):
         (output_dir / relative_path).mkdir(parents=True, exist_ok=True)
@@ -327,6 +361,8 @@ def export_sample_tiles(
     min_label_px: float,
     min_label_visible_ratio: float,
     background_negative_repeat: int,
+    drop_low_variance_negatives: bool,
+    blank_range_threshold: int,
 ) -> list[dict[str, Any]]:
     sample_slug = str(sample["sample_slug"])
     sample_role = str(sample.get("sample_role") or "reference")
@@ -359,6 +395,34 @@ def export_sample_tiles(
                         "kept": False,
                         "label_count": 0,
                         "is_negative": True,
+                        "skip_reason": "negative_keep_ratio",
+                    }
+                )
+                continue
+            image_array = image_array_from_raster_window(dataset, tile_window)
+            low_visual_variance = image_array_has_low_visual_variance(
+                image_array,
+                blank_range_threshold=blank_range_threshold,
+            )
+            if is_negative and drop_low_variance_negatives and low_visual_variance:
+                exported.append(
+                    {
+                        "sample_slug": sample_slug,
+                        "sample_role": sample_role,
+                        "background_category": background_category,
+                        "split": split,
+                        "tile_index": tile_index,
+                        "kept": False,
+                        "label_count": 0,
+                        "is_negative": True,
+                        "low_visual_variance": True,
+                        "skip_reason": LOW_VARIANCE_NEGATIVE_SKIP_REASON,
+                        "window": {
+                            "row_off": tile_window.row_off,
+                            "col_off": tile_window.col_off,
+                            "height": tile_window.height,
+                            "width": tile_window.width,
+                        },
                     }
                 )
                 continue
@@ -368,7 +432,6 @@ def export_sample_tiles(
                 split=split,
                 background_negative_repeat=background_negative_repeat,
             )
-            image_array = image_array_from_raster_window(dataset, tile_window)
             for repeat_index in range(repeats):
                 repeat_suffix = f"_hn{repeat_index + 1:02d}" if repeats > 1 else ""
                 tile_name = f"{sample_slug}_{tile_index:04d}_r{tile_window.row_off}_c{tile_window.col_off}{repeat_suffix}"
@@ -391,6 +454,7 @@ def export_sample_tiles(
                         "label_path": str(label_path),
                         "label_count": len(labels),
                         "is_negative": is_negative,
+                        "low_visual_variance": low_visual_variance,
                         "is_repeated_background_negative": repeats > 1,
                         "window": {
                             "row_off": tile_window.row_off,
@@ -430,6 +494,8 @@ def main() -> int:
                 min_label_px=args.min_label_px,
                 min_label_visible_ratio=args.min_label_visible_ratio,
                 background_negative_repeat=args.background_negative_repeat,
+                drop_low_variance_negatives=args.drop_low_variance_negatives,
+                blank_range_threshold=args.blank_range_threshold,
             )
         )
 
@@ -442,6 +508,11 @@ def main() -> int:
     positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
     negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
     skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]]
+    skipped_low_variance_negative_tiles = [
+        tile
+        for tile in skipped_negative_tiles
+        if tile.get("skip_reason") == LOW_VARIANCE_NEGATIVE_SKIP_REASON
+    ]
     summary = {
         "status": "ok",
         "dataset_yaml": str(dataset_yaml),
@@ -453,11 +524,14 @@ def main() -> int:
         "background_negative_repeat": args.background_negative_repeat,
         "min_label_px": args.min_label_px,
         "min_label_visible_ratio": args.min_label_visible_ratio,
+        "drop_low_variance_negatives": args.drop_low_variance_negatives,
+        "blank_range_threshold": args.blank_range_threshold,
         "source_sample_count": len(samples),
         "tile_count": len(kept_tiles),
         "positive_tile_count": len(positive_tiles),
         "negative_tile_count": len(negative_tiles),
         "skipped_negative_tile_count": len(skipped_negative_tiles),
+        "skipped_low_variance_negative_tile_count": len(skipped_low_variance_negative_tiles),
         "label_count": sum(tile["label_count"] for tile in kept_tiles),
         "train_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "train"),
         "val_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "val"),