diff --git a/CHANGELOG.md b/CHANGELOG.md index 8df97f37..62d0241d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ # Changelog +## Sprint 150 YOLO label visible-ratio gate (2026-07-09) + +- Added `--min-label-visible-ratio` / `OPERATOR_YOLO_MIN_LABEL_VISIBLE_RATIO` to the operator YOLO tile dataset exporter. +- The exporter can now drop clipped building labels where only a small share of the original object bbox is visible in an overlapping tile. +- Tile dataset summaries and audit reports now retain/report `min_label_visible_ratio`. +- Updated operator documentation for the recommended next dataset pass. +- No model was activated, no detections were faked, no provider fetching was introduced and no migration changed. + ## Sprint 149 YOLO duplicate suppression evidence (2026-07-09) - Added configured-YOLO cross-tile duplicate suppression before `Detection` rows are persisted. diff --git a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py index 9677b35c..d94edcd9 100644 --- a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py +++ b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py @@ -38,6 +38,7 @@ def test_operator_yolo_tile_dataset_export_script_contract() -> None: assert "tile_size" in script assert "stride" in script assert "negative_keep_ratio" in script + assert "min_label_visible_ratio" in script assert "positive_tile_count" in script assert "negative_tile_count" in script assert "skipped_negative_tile_count" in script @@ -64,6 +65,7 @@ def test_operator_yolo_tile_dataset_export_help_does_not_require_gis_dependencie assert "--tile-size" in result.stdout assert "--stride" in result.stdout assert "--negative-keep-ratio" in result.stdout + assert "--min-label-visible-ratio" in result.stdout assert "--background-negative-repeat" in result.stdout @@ -96,6 +98,28 @@ def test_negative_tile_keep_is_deterministic_and_ratio_bound() -> None: assert not any(none_kept) +def test_labels_for_tile_can_drop_tiny_visible_box_fragments() -> None: + module = load_tile_exporter() + tile = module.TileWindow(row_off=0, col_off=0, height=100, width=100) + mostly_outside_box = module.PixelBox(min_col=90, min_row=10, max_col=190, max_row=90) + + labels_without_gate = module.labels_for_tile( + tile, + [mostly_outside_box], + min_label_px=4, + min_visible_ratio=0.0, + ) + labels_with_gate = module.labels_for_tile( + tile, + [mostly_outside_box], + min_label_px=4, + min_visible_ratio=0.25, + ) + + assert labels_without_gate == ["0 0.95000000 0.50000000 0.10000000 0.80000000"] + assert labels_with_gate == [] + + def test_background_negative_repeat_only_applies_to_training_background_tiles() -> None: module = load_tile_exporter() diff --git a/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py b/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py index b86cf5de..568253f7 100644 --- a/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py +++ b/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py @@ -44,6 +44,7 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat "negative_keep_ratio": 1.0, "background_negative_repeat": 2, "min_label_px": 2, + "min_label_visible_ratio": 0.25, "source_sample_count": 3, "tile_count": 4, "positive_tile_count": 2, @@ -146,6 +147,7 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat assert report["repeated_background_negative_tile_count"] == 1 assert report["label_stats"]["parsed_label_count"] == 3 assert report["label_stats"]["invalid_label_count"] == 0 + assert report["min_label_visible_ratio"] == 0.25 warning_codes = {warning["code"] for warning in report["warnings"]} assert "positive_sample_count_below_gate" in warning_codes @@ -157,4 +159,5 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat markdown = (output_dir / "operator_yolo_dataset_quality_audit.md").read_text(encoding="utf-8") assert "Operator YOLO Dataset Quality Audit" in markdown assert "Label Quality" in markdown + assert "Minimum visible label ratio" in markdown assert "positive_sample_count_below_gate" in markdown diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 242c4d48..51ffe56d 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,27 @@ +## Sprint 150 YOLO label visible-ratio gate (2026-07-09) + +Changed: +- Added `--min-label-visible-ratio` / `OPERATOR_YOLO_MIN_LABEL_VISIBLE_RATIO` to `scripts/export_operator_yolo_tile_dataset.py`. +- The tile exporter now computes the visible share of each original building bbox inside a tile and can drop labels below the configured ratio. +- Default remains `0` for legacy behavior; use `0.25` for the next overlap-heavy operator dataset experiment. +- Tile dataset summaries include `min_label_visible_ratio`. +- `scripts/audit_operator_yolo_dataset_quality.py` now reports `min_label_visible_ratio` in JSON and Markdown. +- Updated operator script documentation. + +Why: +- The current rejected AOI512 candidate still shows low precision/recall after max-det and duplicate suppression hardening. +- A likely label-quality issue is that overlapping tile export can create many small clipped edge labels for buildings mostly outside a tile. +- This pass improves the next training dataset gate without activating a model, faking detections, fetching providers or changing persistence. + +Tested: +- Red step: `python -m pytest backend\tests\test_sprint130_operator_yolo_tile_dataset.py -q` failed because the exporter lacked `min_label_visible_ratio`, CLI help and visible-fragment filtering. +- `python -m pytest backend\tests\test_sprint130_operator_yolo_tile_dataset.py -q` (`6 passed`) +- Red step: `python -m pytest backend\tests\test_sprint146_operator_yolo_dataset_quality_audit.py -q` failed because the audit report did not expose `min_label_visible_ratio`. +- `python -m pytest backend\tests\test_sprint130_operator_yolo_tile_dataset.py backend\tests\test_sprint146_operator_yolo_dataset_quality_audit.py -q` (`7 passed`) + +Next: +- Run full readiness, deploy Tower, export a new visible-ratio-gated operator tile dataset, audit it, then decide whether it is good enough for another CPU training candidate. + ## Sprint 149 YOLO duplicate suppression evidence (2026-07-09) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index c04c6a6f..c214da08 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -114,9 +114,10 @@ This file now starts with the current implementation status. Older preparation/b - [x] Rerun live dense-AOI calibration after redeploy with `YOLO_MAX_DETECTIONS=1000`; Westerlo reached 523/1000 detections at lower thresholds and Turnhout reached 822/1000, confirming the old 300 cap is removed. - [x] Add configured-YOLO cross-tile duplicate suppression and raw/suppressed calibration evidence fields. - [x] Rerun live dense-AOI calibration after redeploy with `YOLO_DUPLICATE_IOU_THRESHOLD=0.5`; Westerlo 0.25 improved to F1 `0.2537313432835821` and Turnhout 0.25 improved to F1 `0.14114114114114112`, but the candidate remains rejected. +- [x] Add `OPERATOR_YOLO_MIN_LABEL_VISIBLE_RATIO` so the next overlapping-tile dataset can drop tiny clipped edge-fragment labels. - [ ] Find or train a materially stronger aerial/Kempen building model candidate; `geointel-building-yolov8n-expanded160e50-pt` is the best current dense-AOI candidate but still too weak and too noisy for a V1 default. - [ ] Train a higher-capacity local aerial-building detector with stronger positive recall while preserving the hard-negative false-positive gate. -- [ ] Add more diverse positive AOIs and revisit geometry-to-box label strategy before the next default-model training attempt. +- [ ] Export and audit a visible-ratio-gated tile dataset on Tower before the next default-model training attempt. - [ ] Build the next candidate gate around better positive AOI coverage, label strategy and hard-negative retention. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index 30f4b979..b145504a 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -321,6 +321,7 @@ docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset. --tile-size 160 \ --stride 80 \ --negative-keep-ratio 1.0 \ + --min-label-visible-ratio 0.25 \ --val-samples turnhout,retie,kasterlee_bos \ --force ``` @@ -329,6 +330,9 @@ The tile exporter clips GRB building bounding boxes into each tile, writes YOLO labels beside each tile image, keeps a deterministic ratio of empty negative tiles, and records `yolo_tile_dataset_summary.json` with `positive_tile_count`, `negative_tile_count` and skipped negative tile counts. +`--min-label-visible-ratio` drops labels where only a small clipped fragment of +the original building bbox is visible inside the tile; this reduces noisy +tile-edge labels in overlapping-tile datasets. Use `0` for legacy behavior. It remains operator tooling only: no provider fetch, no API mutation and no automatic model training. @@ -344,7 +348,8 @@ The audit reads the tile summary and YOLO label files, then writes `operator_yolo_dataset_quality_audit.json` and `operator_yolo_dataset_quality_audit.md`. It reports positive/background sample coverage, train/validation split coverage, repeated hard-negative pressure, -missing or invalid label rows and normalized box-area signals. Treat +minimum visible label ratio, missing or invalid label rows and normalized +box-area signals. Treat `needs_attention` as a dataset-design warning, not as a runtime failure: the next action is usually more positive AOIs, better validation coverage or more unique hard negatives rather than simply extending epochs. diff --git a/scripts/audit_operator_yolo_dataset_quality.py b/scripts/audit_operator_yolo_dataset_quality.py index 079f3d90..7433ec53 100644 --- a/scripts/audit_operator_yolo_dataset_quality.py +++ b/scripts/audit_operator_yolo_dataset_quality.py @@ -291,6 +291,7 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name "negative_keep_ratio": summary.get("negative_keep_ratio"), "background_negative_repeat": summary.get("background_negative_repeat"), "min_label_px": summary.get("min_label_px"), + "min_label_visible_ratio": summary.get("min_label_visible_ratio"), "tile_count": len(tiles), "positive_tile_count": len(positive_tiles), "negative_tile_count": len(negative_tiles), @@ -348,6 +349,7 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: f"- Tiles: {report['tile_count']} ({report['positive_tile_count']} positive, {report['negative_tile_count']} negative)", f"- Samples: {report['sample_count']} ({report['positive_sample_count']} positive, {report['background_sample_count']} background)", f"- Repeated background negative share: {report['repeated_background_negative_share_of_negatives']:.3f}", + f"- Minimum visible label ratio: {format_optional_float(report.get('min_label_visible_ratio'))}", "", "## Label Quality", "", diff --git a/scripts/export_operator_yolo_tile_dataset.py b/scripts/export_operator_yolo_tile_dataset.py index 5157bc44..fa93fd53 100644 --- a/scripts/export_operator_yolo_tile_dataset.py +++ b/scripts/export_operator_yolo_tile_dataset.py @@ -78,6 +78,15 @@ def parse_args() -> argparse.Namespace: default=float(os.environ.get("OPERATOR_YOLO_MIN_LABEL_PX", "4")), help="Minimum clipped box width/height in pixels before a tile label is kept.", ) + parser.add_argument( + "--min-label-visible-ratio", + type=float, + default=float(os.environ.get("OPERATOR_YOLO_MIN_LABEL_VISIBLE_RATIO", "0")), + help=( + "Minimum visible share of the original object bbox required before a clipped tile label is kept. " + "Use 0 to keep legacy edge-fragment labels." + ), + ) parser.add_argument( "--background-negative-repeat", type=int, @@ -217,7 +226,7 @@ def load_reference_pixel_boxes(reference_path: Path, dataset: Any, min_label_px: return boxes -def labels_for_tile(tile_window: TileWindow, boxes: list[PixelBox], min_label_px: float) -> list[str]: +def labels_for_tile(tile_window: TileWindow, boxes: list[PixelBox], min_label_px: float, min_visible_ratio: float = 0.0) -> list[str]: labels: list[str] = [] tile_min_col = tile_window.col_off tile_min_row = tile_window.row_off @@ -233,6 +242,11 @@ def labels_for_tile(tile_window: TileWindow, boxes: list[PixelBox], min_label_px box_height = max_row - min_row if box_width < min_label_px or box_height < min_label_px: continue + original_area = max((box.max_col - box.min_col) * (box.max_row - box.min_row), 0.0) + visible_area = box_width * box_height + visible_ratio = visible_area / original_area if original_area > 0 else 0.0 + if min_visible_ratio > 0 and visible_ratio < min_visible_ratio: + continue local_min_col = min_col - tile_min_col local_max_col = max_col - tile_min_col local_min_row = min_row - tile_min_row @@ -296,6 +310,7 @@ def export_sample_tiles( stride: int, negative_keep_ratio: float, min_label_px: float, + min_label_visible_ratio: float, background_negative_repeat: int, ) -> list[dict[str, Any]]: sample_slug = str(sample["sample_slug"]) @@ -312,7 +327,12 @@ def export_sample_tiles( with rasterio.open(raster_path) as dataset: boxes = load_reference_pixel_boxes(reference_path, dataset, min_label_px=min_label_px) for tile_index, tile_window in enumerate(iter_tile_windows(dataset.width, dataset.height, tile_size, stride)): - labels = labels_for_tile(tile_window, boxes, min_label_px=min_label_px) + labels = labels_for_tile( + tile_window, + boxes, + min_label_px=min_label_px, + min_visible_ratio=min_label_visible_ratio, + ) is_negative = not labels if is_negative and not keep_negative_tile(sample_slug, tile_index, negative_keep_ratio): exported.append( @@ -391,6 +411,7 @@ def main() -> int: stride=args.stride, negative_keep_ratio=args.negative_keep_ratio, min_label_px=args.min_label_px, + min_label_visible_ratio=args.min_label_visible_ratio, background_negative_repeat=args.background_negative_repeat, ) ) @@ -414,6 +435,7 @@ def main() -> int: "negative_keep_ratio": args.negative_keep_ratio, "background_negative_repeat": args.background_negative_repeat, "min_label_px": args.min_label_px, + "min_label_visible_ratio": args.min_label_visible_ratio, "source_sample_count": len(samples), "tile_count": len(kept_tiles), "positive_tile_count": len(positive_tiles),