From a159370a1112b0f41756435680dd51cb87edf850 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 11 Jul 2026 22:57:59 +0200 Subject: [PATCH] Filter low variance YOLO negative tiles --- ...st_sprint130_operator_yolo_tile_dataset.py | 89 +++++++++++++++++++ docs/CODEX_EXECUTION_LOG.md | 38 ++++++++ docs/TODO.md | 3 +- ...07-11-yolo-low-variance-negative-filter.md | 56 ++++++++++++ ...olo-low-variance-negative-filter-design.md | 34 +++++++ scripts/README.md | 12 +++ scripts/export_operator_yolo_tile_dataset.py | 76 +++++++++++++++- 7 files changed, 306 insertions(+), 2 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-11-yolo-low-variance-negative-filter.md create mode 100644 docs/superpowers/specs/2026-07-11-yolo-low-variance-negative-filter-design.md diff --git a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py index d630f356..16bcb8d4 100644 --- a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py +++ b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py @@ -5,6 +5,8 @@ from pathlib import Path import subprocess import sys +import numpy as np + ROOT = Path(__file__).resolve().parents[2] @@ -39,6 +41,8 @@ def test_operator_yolo_tile_dataset_export_script_contract() -> None: assert "stride" in script assert "negative_keep_ratio" in script assert "min_label_visible_ratio" in script + assert "drop_low_variance_negatives" in script + assert "skipped_low_variance_negative_tile_count" in script assert "positive_tile_count" in script assert "negative_tile_count" in script assert "skipped_negative_tile_count" in script @@ -67,6 +71,8 @@ def test_operator_yolo_tile_dataset_export_help_does_not_require_gis_dependencie assert "--negative-keep-ratio" in result.stdout assert "--min-label-visible-ratio" in result.stdout assert "--background-negative-repeat" in result.stdout + assert "--drop-low-variance-negatives" in result.stdout + assert "--blank-range-threshold" in result.stdout def test_iter_tile_windows_covers_edges_without_duplicates() -> None: @@ -171,3 +177,86 @@ def test_background_category_is_derived_for_legacy_operator_manifests() -> None: assert module.background_category_for_sample(pure_empty_sample) == "pure_empty_negative" assert module.background_category_for_sample(sparse_context_sample) == "sparse_building_context" assert module.background_category_for_sample(reference_sample) == "reference_aoi" + + +def test_export_can_skip_low_variance_negative_tiles(tmp_path: Path, monkeypatch) -> None: + module = load_tile_exporter() + raster_path = tmp_path / "sample.tif" + reference_path = tmp_path / "reference.geojson" + raster_path.write_bytes(b"fake-raster") + reference_path.write_text('{"type": "FeatureCollection", "features": []}', encoding="utf-8") + + class FakeDataset: + width = 256 + height = 128 + crs = "EPSG:31370" + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + class FakeRasterio: + @staticmethod + def open(path): + assert Path(path) == raster_path + return FakeDataset() + + class FakeImageObject: + def __init__(self, array): + self.array = array + + def save(self, path): + Path(path).write_bytes(b"png") + + class FakeImage: + @staticmethod + def fromarray(array): + return FakeImageObject(array) + + def fake_image_array_from_raster_window(dataset, tile_window): + if tile_window.col_off == 0: + return np.full((128, 128, 3), 255, dtype=np.uint8) + image = np.zeros((128, 128, 3), dtype=np.uint8) + image[:, 64:, :] = 80 + return image + + monkeypatch.setattr(module, "rasterio", FakeRasterio) + monkeypatch.setattr(module, "Image", FakeImage) + monkeypatch.setattr(module, "load_reference_pixel_boxes", lambda reference_path, dataset, min_label_px: []) + monkeypatch.setattr(module, "image_array_from_raster_window", fake_image_array_from_raster_window) + + records = module.export_sample_tiles( + sample={ + "sample_slug": "blank_negative", + "sample_role": "background_candidate", + "background_category": "pure_empty_negative", + "raster_path": str(raster_path), + "reference_path": str(reference_path), + }, + manifest_path=tmp_path / "operator_samples_manifest.json", + output_dir=tmp_path / "dataset", + val_slugs=set(), + tile_size=128, + stride=128, + negative_keep_ratio=1.0, + min_label_px=4, + min_label_visible_ratio=0.0, + background_negative_repeat=1, + drop_low_variance_negatives=True, + blank_range_threshold=3, + ) + + skipped = [record for record in records if not record["kept"]] + kept = [record for record in records if record["kept"]] + + assert len(skipped) == 1 + assert skipped[0]["skip_reason"] == "low_visual_variance_negative" + assert skipped[0]["low_visual_variance"] is True + assert skipped[0]["is_negative"] is True + assert skipped[0]["tile_index"] == 0 + assert len(kept) == 1 + assert kept[0]["tile_index"] == 1 + assert kept[0]["low_visual_variance"] is False + assert Path(kept[0]["image_path"]).exists() diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 977b52be..b42a57fe 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -6866,3 +6866,41 @@ Open: ## Next recommended pass - Add no-data/low-variance filtering to the operator YOLO tile export path, regenerate the clean AOI1024 dataset, rerun the contact-sheet QA, and only then consider another training attempt. + +# Sprint 168 - Operator YOLO low-variance negative filtering + +## What changed + +- Added opt-in low-variance negative filtering to `scripts/export_operator_yolo_tile_dataset.py`. +- Added CLI/env controls: + - `--drop-low-variance-negatives` / `OPERATOR_YOLO_DROP_LOW_VARIANCE_NEGATIVES`; + - `--blank-range-threshold` / `OPERATOR_YOLO_BLANK_RANGE_THRESHOLD`. +- The filter evaluates the rendered raster tile image and skips only negative tiles when enabled. +- Positive/labeled tiles are never removed by this variance gate. +- Kept tile records now include `low_visual_variance`. +- Skipped blank/no-data negative records use `skip_reason="low_visual_variance_negative"`. +- Dataset summaries now include: + - `drop_low_variance_negatives`; + - `blank_range_threshold`; + - `skipped_low_variance_negative_tile_count`. +- Updated operator documentation with the refreshed AOI1024 cleanpx export command. +- Added design and execution plan docs under `docs/superpowers/`. + +## Local validation + +- RED: `python -m pytest backend/tests/test_sprint130_operator_yolo_tile_dataset.py::test_export_can_skip_low_variance_negative_tiles -q` failed because `export_sample_tiles()` did not accept `drop_low_variance_negatives`. +- GREEN: same targeted test passed after adding the filter. +- Ran `python -m pytest backend/tests/test_sprint130_operator_yolo_tile_dataset.py -q`: 8 passed. +- Ran `python -m pytest backend/tests/test_sprint130_operator_yolo_tile_dataset.py backend/tests/test_sprint167_operator_yolo_label_qa_contact_sheets.py backend/tests/test_docker_runtime_config.py::test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use backend/tests/test_docker_runtime_config.py::test_all_in_one_dockerfile_copies_operator_scripts_after_dependency_install -q`: 11 passed. +- Ran `python scripts/export_operator_yolo_tile_dataset.py --help`: the CLI exposes `--drop-low-variance-negatives`, `--no-drop-low-variance-negatives` and `--blank-range-threshold` without loading GIS dependencies. +- Ran `bash scripts/run_readiness_check.sh`: 460 backend tests passed, frontend typecheck passed, frontend build passed, readiness passed. + +## Known limitations + +- The low-variance gate is deliberately simple and only identifies visually blank/no-data-looking negative tiles. +- It is opt-in to avoid silently changing historical dataset exports. +- Operator visual contact-sheet review remains required before any new training run. + +## Next recommended pass + +- Redeploy Tower, regenerate the AOI1024 cleanpx dataset with `--drop-low-variance-negatives`, rerun dataset audit and contact-sheet QA, then decide whether the filtered dataset is suitable for another training run. diff --git a/docs/TODO.md b/docs/TODO.md index 693382a2..9e83b6d5 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -131,7 +131,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add guarded promoted-candidate activation helper requiring a promotion report path and exact candidate key before `.env` can be changed. - [x] Add per-sample YOLO dataset audit diagnostics for parsed labels, median box area, small-box share and AOI-specific warning codes. - [x] Add deterministic visual YOLO label QA contact sheets before spending more CPU on another training run. -- [ ] Filter no-data/low-variance pure-empty negative tiles from operator YOLO exports before the next training run. +- [x] Filter no-data/low-variance pure-empty negative tiles from operator YOLO exports before the next training run. +- [ ] Regenerate the AOI1024 cleanpx YOLO dataset with low-variance negative filtering and rerun visual contact-sheet QA before training. - [ ] Apply promoted V1 default building detector only after explicit operator review of the emitted `.env` updates, followed by rebuild/restart and browser/runtime smoke. ## Sprint 8 status diff --git a/docs/superpowers/plans/2026-07-11-yolo-low-variance-negative-filter.md b/docs/superpowers/plans/2026-07-11-yolo-low-variance-negative-filter.md new file mode 100644 index 00000000..e2b3e15e --- /dev/null +++ b/docs/superpowers/plans/2026-07-11-yolo-low-variance-negative-filter.md @@ -0,0 +1,56 @@ +# YOLO Low-Variance Negative Filter Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add opt-in filtering for blank/low-variance negative tiles in the operator YOLO tile exporter. + +**Architecture:** Keep the behavior inside `scripts/export_operator_yolo_tile_dataset.py` because this is operator-only dataset construction, not application inference. Compute low-variance from the raster window image array, skip only negative tiles when explicitly enabled, and expose all decisions in `yolo_tile_dataset_summary.json`. + +**Tech Stack:** Python, pytest, rasterio/Pillow runtime helpers already used by the exporter. + +--- + +### Task 1: Regression Test + +**Files:** +- Modify: `backend/tests/test_sprint130_operator_yolo_tile_dataset.py` + +- [x] Add a test that monkeypatches `image_array_from_raster_window` and verifies `export_sample_tiles` skips only low-variance negative tiles when `drop_low_variance_negatives=True`. +- [x] Run the targeted test and confirm it fails because the exporter does not yet accept/report the new filter fields. + +### Task 2: Exporter Implementation + +**Files:** +- Modify: `scripts/export_operator_yolo_tile_dataset.py` + +- [x] Add CLI/env options: + - `--drop-low-variance-negatives` + - `--blank-range-threshold` +- [x] Add a helper to detect low visual variance from an image array. +- [x] Thread the options into `export_sample_tiles`. +- [x] Skip only negative low-variance tiles when the option is enabled. +- [x] Add `low_visual_variance` to kept tile records. +- [x] Add skipped records with `skip_reason="low_visual_variance_negative"`. +- [x] Add summary fields for the option, threshold and skipped count. + +### Task 3: Documentation + +**Files:** +- Modify: `scripts/README.md` +- Modify: `docs/CODEX_EXECUTION_LOG.md` +- Modify: `docs/TODO.md` + +- [x] Document the new operator export flags and intended Tower command. +- [x] Record local validation and known limitation. +- [x] Mark the low-variance export filter item as implemented after validation. + +### Task 4: Validation and Handoff + +**Files:** +- No additional file changes expected. + +- [x] Run targeted pytest for the exporter/contact-sheet tests. +- [x] Run `bash scripts/run_readiness_check.sh`. +- [ ] Commit and push. +- [ ] Rebuild/deploy Tower if code changed. +- [ ] Regenerate the AOI1024 dataset with the new filter enabled, then render contact sheets and record the result. diff --git a/docs/superpowers/specs/2026-07-11-yolo-low-variance-negative-filter-design.md b/docs/superpowers/specs/2026-07-11-yolo-low-variance-negative-filter-design.md new file mode 100644 index 00000000..440a189d --- /dev/null +++ b/docs/superpowers/specs/2026-07-11-yolo-low-variance-negative-filter-design.md @@ -0,0 +1,34 @@ +# YOLO Low-Variance Negative Filter Design + +## Goal + +Prevent blank/no-data pure-empty negative tiles from entering the next operator YOLO training dataset while keeping the exporter conservative and auditable. + +## Scope + +This pass changes only operator-side YOLO tile export tooling. It does not change backend APIs, database schema, frontend behavior, model activation, training behavior, provider fetching or inference. + +## Approach + +Add an opt-in filter to `scripts/export_operator_yolo_tile_dataset.py` that evaluates raster tile image variance before writing negative tiles. The filter applies only after labels are computed and only when a tile is negative. Positive tiles are never dropped by this gate, even if visually low-variance, because dropping labeled data silently would be a worse failure mode. + +The filter will use the same simple max-min grayscale range heuristic as the contact-sheet QA script. A tile with range at or below `OPERATOR_YOLO_BLANK_RANGE_THRESHOLD` is treated as low-variance. When `--drop-low-variance-negatives` is enabled, that negative tile is skipped and recorded in summary fields instead of being written into `images/` and `labels/`. + +## Reporting + +The dataset summary must include: + +- `drop_low_variance_negatives` +- `blank_range_threshold` +- `skipped_low_variance_negative_tile_count` +- skipped tile records with `skip_reason="low_visual_variance_negative"` + +Kept tile records should include `low_visual_variance` so downstream contact-sheet and audit tooling can expose the signal. + +## Validation + +Add a regression test that constructs a blank negative raster window and a patterned negative raster window, enables the filter, and verifies that only the blank negative is skipped for `low_visual_variance_negative`. + +## Known Limitation + +The heuristic is intentionally simple. It identifies blank/no-data tiles, not semantic quality. Operator visual QA remains required before another training run. diff --git a/scripts/README.md b/scripts/README.md index c773dc17..40f09224 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -367,6 +367,12 @@ negative tiles, and records `yolo_tile_dataset_summary.json` with `--min-label-visible-ratio` drops labels where only a small clipped fragment of the original building bbox is visible inside the tile; this reduces noisy tile-edge labels in overlapping-tile datasets. Use `0` for legacy behavior. +Use `--drop-low-variance-negatives` to skip negative tiles whose rendered image +has a max-min pixel range at or below `--blank-range-threshold`. This gate is +intended for blank/no-data pure-empty negatives only; positive/labeled tiles are +not removed by this filter. The summary records +`skipped_low_variance_negative_tile_count` and skipped tile records with +`skip_reason=low_visual_variance_negative`. For legacy operator manifests that predate explicit `background_category`, the exporter derives the same categories as the split-background evaluator: background samples with `reference_feature_count == 0` become @@ -387,10 +393,16 @@ docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset. --negative-keep-ratio 1.0 \ --min-label-px 12 \ --min-label-visible-ratio 0.35 \ + --drop-low-variance-negatives \ + --blank-range-threshold 3 \ --val-samples turnhout,retie,westerlo,arendonk_heide \ --force ``` +This refreshed cleanpx dataset is the minimum pre-training baseline after the +visual contact-sheet pass found six blank-looking `arendonk_heide` validation +negatives in the older export. + Then audit with stricter small-box gates: ```bash diff --git a/scripts/export_operator_yolo_tile_dataset.py b/scripts/export_operator_yolo_tile_dataset.py index 8f6d3a76..e591a68d 100644 --- a/scripts/export_operator_yolo_tile_dataset.py +++ b/scripts/export_operator_yolo_tile_dataset.py @@ -24,6 +24,7 @@ DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset REFERENCE_AOI_CATEGORY = "reference_aoi" PURE_EMPTY_BACKGROUND_CATEGORY = "pure_empty_negative" SPARSE_BACKGROUND_CATEGORY = "sparse_building_context" +LOW_VARIANCE_NEGATIVE_SKIP_REASON = "low_visual_variance_negative" rasterio: Any = None Window: Any = None Transformer: Any = None @@ -96,10 +97,32 @@ def parse_args() -> argparse.Namespace: default=int(os.environ.get("OPERATOR_YOLO_BACKGROUND_NEGATIVE_REPEAT", "1")), help="Repeat kept train/background negative tiles this many times for hard-negative balancing.", ) + parser.add_argument( + "--drop-low-variance-negatives", + action=argparse.BooleanOptionalAction, + default=env_flag("OPERATOR_YOLO_DROP_LOW_VARIANCE_NEGATIVES", default=False), + help=( + "Skip negative tiles whose rendered image has very low pixel variance. " + "This is intended for no-data/blank pure-empty negatives only." + ), + ) + parser.add_argument( + "--blank-range-threshold", + type=int, + default=int(os.environ.get("OPERATOR_YOLO_BLANK_RANGE_THRESHOLD", "3")), + help="Max pixel value range used to classify a negative tile as visually blank/low-variance.", + ) parser.add_argument("--force", action="store_true", help="Remove and recreate output-dir before exporting.") return parser.parse_args() +def env_flag(name: str, *, default: bool) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip().lower() in {"1", "true", "yes", "on"} + + def ensure_dependencies() -> None: global Image, Transformer, Window, rasterio try: @@ -293,6 +316,17 @@ def image_array_from_raster_window(dataset: Any, tile_window: TileWindow) -> Any return rgb +def image_array_has_low_visual_variance(image_array: Any, blank_range_threshold: int) -> bool: + import numpy as np + + array = np.asarray(image_array) + if array.size == 0: + return True + max_value = float(np.nanmax(array)) + min_value = float(np.nanmin(array)) + return (max_value - min_value) <= blank_range_threshold + + def ensure_yolo_directories(output_dir: Path) -> None: for relative_path in ("images/train", "labels/train", "images/val", "labels/val"): (output_dir / relative_path).mkdir(parents=True, exist_ok=True) @@ -327,6 +361,8 @@ def export_sample_tiles( min_label_px: float, min_label_visible_ratio: float, background_negative_repeat: int, + drop_low_variance_negatives: bool, + blank_range_threshold: int, ) -> list[dict[str, Any]]: sample_slug = str(sample["sample_slug"]) sample_role = str(sample.get("sample_role") or "reference") @@ -359,6 +395,34 @@ def export_sample_tiles( "kept": False, "label_count": 0, "is_negative": True, + "skip_reason": "negative_keep_ratio", + } + ) + continue + image_array = image_array_from_raster_window(dataset, tile_window) + low_visual_variance = image_array_has_low_visual_variance( + image_array, + blank_range_threshold=blank_range_threshold, + ) + if is_negative and drop_low_variance_negatives and low_visual_variance: + exported.append( + { + "sample_slug": sample_slug, + "sample_role": sample_role, + "background_category": background_category, + "split": split, + "tile_index": tile_index, + "kept": False, + "label_count": 0, + "is_negative": True, + "low_visual_variance": True, + "skip_reason": LOW_VARIANCE_NEGATIVE_SKIP_REASON, + "window": { + "row_off": tile_window.row_off, + "col_off": tile_window.col_off, + "height": tile_window.height, + "width": tile_window.width, + }, } ) continue @@ -368,7 +432,6 @@ def export_sample_tiles( split=split, background_negative_repeat=background_negative_repeat, ) - image_array = image_array_from_raster_window(dataset, tile_window) for repeat_index in range(repeats): repeat_suffix = f"_hn{repeat_index + 1:02d}" if repeats > 1 else "" tile_name = f"{sample_slug}_{tile_index:04d}_r{tile_window.row_off}_c{tile_window.col_off}{repeat_suffix}" @@ -391,6 +454,7 @@ def export_sample_tiles( "label_path": str(label_path), "label_count": len(labels), "is_negative": is_negative, + "low_visual_variance": low_visual_variance, "is_repeated_background_negative": repeats > 1, "window": { "row_off": tile_window.row_off, @@ -430,6 +494,8 @@ def main() -> int: min_label_px=args.min_label_px, min_label_visible_ratio=args.min_label_visible_ratio, background_negative_repeat=args.background_negative_repeat, + drop_low_variance_negatives=args.drop_low_variance_negatives, + blank_range_threshold=args.blank_range_threshold, ) ) @@ -442,6 +508,11 @@ def main() -> int: positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]] negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]] skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]] + skipped_low_variance_negative_tiles = [ + tile + for tile in skipped_negative_tiles + if tile.get("skip_reason") == LOW_VARIANCE_NEGATIVE_SKIP_REASON + ] summary = { "status": "ok", "dataset_yaml": str(dataset_yaml), @@ -453,11 +524,14 @@ def main() -> int: "background_negative_repeat": args.background_negative_repeat, "min_label_px": args.min_label_px, "min_label_visible_ratio": args.min_label_visible_ratio, + "drop_low_variance_negatives": args.drop_low_variance_negatives, + "blank_range_threshold": args.blank_range_threshold, "source_sample_count": len(samples), "tile_count": len(kept_tiles), "positive_tile_count": len(positive_tiles), "negative_tile_count": len(negative_tiles), "skipped_negative_tile_count": len(skipped_negative_tiles), + "skipped_low_variance_negative_tile_count": len(skipped_low_variance_negative_tiles), "label_count": sum(tile["label_count"] for tile in kept_tiles), "train_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "train"), "val_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "val"),