From 9bd6752128394b63f93c256382ca457047041b85 Mon Sep 17 00:00:00 2001 From: Codex Date: Tue, 7 Jul 2026 23:01:51 +0200 Subject: [PATCH] Add hard-negative balanced YOLO tile export --- CHANGELOG.md | 12 +++ ...st_sprint130_operator_yolo_tile_dataset.py | 30 +++++++ docs/CODEX_EXECUTION_LOG.md | 54 ++++++++++++ docs/TODO.md | 2 +- scripts/README.md | 28 ++++++ scripts/export_operator_yolo_tile_dataset.py | 86 +++++++++++++------ 6 files changed, 187 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea0204fe..1056d51b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ # Changelog +## Sprint 133 Hard-negative-balanced YOLO candidate (2026-07-07) + +- Added `--background-negative-repeat` / `OPERATOR_YOLO_BACKGROUND_NEGATIVE_REPEAT` support to `scripts/export_operator_yolo_tile_dataset.py` so train-split background-candidate negative tiles can be repeated deterministically without duplicating validation tiles. +- Added exported tile provenance fields `sample_role`, `repeat_index` and `is_repeated_background_negative` plus regression coverage in `backend/tests/test_sprint130_operator_yolo_tile_dataset.py`. +- Live Tower export produced `/app/storage/operator-data/yolo-building-tile-hardneg160r8` with tile size `160`, stride `80`, background repeat `8`, 864 tiles, 260 positive tiles, 604 negative tiles, 11213 labels, 756 train tiles and 108 validation tiles. +- Live Tower 40-epoch CPU training produced `/app/models/geointel-building-yolov8n-hardneg160r8e40.pt`; the model catalog exposes it as `geointel-building-yolov8n-hardneg160r8e40-pt` with SHA256 `7a77bd9f68e4c3927ffc8a8cd978a81067b02f42cffe77ada5334b5f8dbb6b50`. +- Live YOLO preflight loaded the model successfully with `status=ready`, `model_load_ok=true`, `manifest_valid=true`, `tile_paths_exist=true`, `will_download_models=false` and `will_run_inference=false`. +- Live 60-run dense QA matrix showed `geointel-building-yolov8n-expanded160e50-pt` remains the better dense-AOI candidate; hardneg160r8e40 underperformed it on Geel, Mol, Turnhout and Retie. +- Live 36-run background matrix showed hardneg160r8e40 materially reduced false-positive pressure: Kasterlee-bos dropped from expanded160e50's 38/46/76 detections to 5/9/25 at thresholds `0.25`/`0.15`/`0.05`, and Postel-bos/Lommel-heide stayed at 0 detections across all thresholds. +- Decision: hardneg160r8e40 is useful evidence for a low-false-positive training direction, but it should not become the V1 default because dense-AOI recall/F1 regressed. The next model pass should combine stronger positive coverage with hard-negative balancing or test a stronger aerial-building architecture. +- No Training Studio UI, API contract change, provider fetching, model auto-provisioning, fake detections or app-side model training behavior was introduced. + ## Sprint 132 Operator hard-negative detection matrix (2026-07-07) - Added `scripts/run_operator_hard_negative_detection_matrix.sh` to score configured-YOLO false-positive pressure on documented background-candidate operator AOIs without uploading reference vectors or running QA/QC. diff --git a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py index ee58a955..9677b35c 100644 --- a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py +++ b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py @@ -64,6 +64,7 @@ def test_operator_yolo_tile_dataset_export_help_does_not_require_gis_dependencie assert "--tile-size" in result.stdout assert "--stride" in result.stdout assert "--negative-keep-ratio" in result.stdout + assert "--background-negative-repeat" in result.stdout def test_iter_tile_windows_covers_edges_without_duplicates() -> None: @@ -93,3 +94,32 @@ def test_negative_tile_keep_is_deterministic_and_ratio_bound() -> None: assert 1 <= sum(first) <= 25 assert all(all_kept) assert not any(none_kept) + + +def test_background_negative_repeat_only_applies_to_training_background_tiles() -> None: + module = load_tile_exporter() + + assert module.background_negative_repeat_count( + is_negative=True, + sample_role="background_candidate", + split="train", + background_negative_repeat=4, + ) == 4 + assert module.background_negative_repeat_count( + is_negative=True, + sample_role="background_candidate", + split="val", + background_negative_repeat=4, + ) == 1 + assert module.background_negative_repeat_count( + is_negative=False, + sample_role="background_candidate", + split="train", + background_negative_repeat=4, + ) == 1 + assert module.background_negative_repeat_count( + is_negative=True, + sample_role="reference", + split="train", + background_negative_repeat=4, + ) == 1 diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index ea047240..01ef9fb7 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,57 @@ +## Sprint 133 Hard-negative-balanced YOLO candidate (2026-07-07) + +Changed: +- Hardened `scripts/export_operator_yolo_tile_dataset.py` with deterministic train-only background-negative repetition through `--background-negative-repeat` and `OPERATOR_YOLO_BACKGROUND_NEGATIVE_REPEAT`. +- Background-negative repetition applies only when `is_negative=true`, `sample_role=background_candidate` and `split=train`; validation tiles, positive tiles and normal reference samples are not duplicated. +- Added tile-level provenance fields `sample_role`, `repeat_index` and `is_repeated_background_negative`. +- Added regression coverage in `backend/tests/test_sprint130_operator_yolo_tile_dataset.py`. +- Updated `scripts/README.md`, `docs/TODO.md`, `docs/CODEX_EXECUTION_LOG.md` and `CHANGELOG.md`. + +Tested: +- RED: `python -m pytest backend\tests\test_sprint130_operator_yolo_tile_dataset.py -q` failed before `--background-negative-repeat` and `background_negative_repeat_count` existed. +- `python -m pytest backend\tests\test_sprint130_operator_yolo_tile_dataset.py -q` passed. +- `python -m py_compile scripts\export_operator_yolo_tile_dataset.py` passed. +- `python scripts\export_operator_yolo_tile_dataset.py --help` passed. +- Live Tower hard-negative-balanced tile export passed: + - dataset: `/app/storage/operator-data/yolo-building-tile-hardneg160r8` + - source samples: 10 + - tile size: `160` + - stride: `80` + - negative keep ratio: `1.0` + - background negative repeat: `8` + - exported tiles: `864` + - positive tiles: `260` + - negative tiles: `604` + - labels: `11213` + - train tiles: `756` + - validation tiles: `108` +- Live Tower 40-epoch CPU training passed: + - output model: `/app/models/geointel-building-yolov8n-hardneg160r8e40.pt` + - catalog asset: `geointel-building-yolov8n-hardneg160r8e40-pt` + - SHA256: `7a77bd9f68e4c3927ffc8a8cd978a81067b02f42cffe77ada5334b5f8dbb6b50` + - final validation: precision `0.403`, recall `0.378`, mAP50 `0.301`, mAP50-95 `0.0944` +- Live API preflight passed for `geointel-building-yolov8n-hardneg160r8e40-pt` with `status=ready`, `model_load_ok=true`, `manifest_valid=true`, `tile_paths_exist=true`, `will_download_models=false` and `will_run_inference=false`. +- Live 60-run multi-sample QA matrix completed: + - output: `/mnt/user/appdata/geointel/artifacts/detection-quality-matrix/multi-sample/hardneg160r8e40-live/multi_sample_quality_summary.json` + - command compared `geointel-building-yolov8n-hardneg160r8e40-pt`, `geointel-building-yolov8n-expanded160e50-pt`, `geointel-building-yolov8n-tile30-pt` and `yolov8s-building-segmentation-pt` over Geel, Mol, Turnhout, Retie and Kasterlee-bos with tile `640`, overlap `64`, thresholds `0.25`/`0.15`/`0.05`. + - best overall score and recall remained Geel with `geointel-building-yolov8n-expanded160e50-pt`, precision `0.30333333333333334`, recall `0.14748784440842788`, F1 `0.1984732824427481`. + - hardneg160r8e40 dense F1 lagged expanded160e50 on Geel (`0.14394765539803708` vs `0.1984732824427481`), Mol (`0.11572700296735906` vs `0.1651651651651652`), Turnhout (`0.14911463187325258` vs `0.1938490214352283`) and Retie (`0.10538116591928251` vs `0.1569506726457399`). +- Live 36-run hard-negative matrix completed: + - output: `/mnt/user/appdata/geointel/artifacts/detection-hard-negatives/hardneg160r8e40-live/hard_negative_matrix_summary.json` + - Postel-bos: hardneg160r8e40 produced 0/0/0 detections at thresholds `0.25`/`0.15`/`0.05`; expanded160e50 produced 0/0/1. + - Lommel-heide: hardneg160r8e40 produced 0/0/0 detections; expanded160e50 produced 0/0/10. + - Kasterlee-bos: hardneg160r8e40 produced 5/9/25 detections; expanded160e50 produced 38/46/76. + +Open: +- None for the hard-negative-balanced tile export contract itself. + +Limitations: +- `geointel-building-yolov8n-hardneg160r8e40-pt` reduced false-positive pressure but regressed dense-AOI recall/F1. It should not become the V1 default. +- This remains operator tooling only. It does not add Training Studio, browser training controls, provider fetching, fake detections, model auto-provisioning or API contract changes. + +Next recommended pass: +- Train or import a materially stronger aerial/Kempen building model candidate, then benchmark it against the same dense QA and hard-negative matrices before changing default model selection. + ## Sprint 132 Operator hard-negative detection matrix (2026-07-07) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index 592a92ad..bbc63f9d 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -106,7 +106,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Calibrate confidence, IoU and model selection against additional local orthophoto/reference samples beyond Geel/Mol/Turnhout. - [x] Add negative/background AOIs to the operator sample corpus and train an expanded local tile-level YOLO candidate. - [x] Add a hard-negative model-quality pass with sparse/background AOIs and explicit false-positive scoring. -- [ ] Train a hard-negative-balanced YOLO candidate and rerun dense QA plus background false-positive matrices. +- [x] Train a hard-negative-balanced YOLO candidate and rerun dense QA plus background false-positive matrices. - [ ] Find or train a materially stronger aerial/Kempen building model candidate; `geointel-building-yolov8n-expanded160e50-pt` is the best current dense-AOI candidate but still too weak and too noisy for a V1 default. ## Sprint 8 status diff --git a/scripts/README.md b/scripts/README.md index 575e8997..f3182d19 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -328,6 +328,27 @@ negative tiles, and records `yolo_tile_dataset_summary.json` with It remains operator tooling only: no provider fetch, no API mutation and no automatic model training. +For hard-negative-balanced experiments, repeat only train-split negative tiles +from samples marked `sample_role=background_candidate`: + +```bash +docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \ + --manifest-path /app/storage/operator-data/operator_samples_manifest.json \ + --output-dir /app/storage/operator-data/yolo-building-tile-hardneg160r8 \ + --tile-size 160 \ + --stride 80 \ + --negative-keep-ratio 1.0 \ + --background-negative-repeat 8 \ + --val-samples turnhout,retie,kasterlee_bos \ + --force +``` + +The repeat option can also be set with +`OPERATOR_YOLO_BACKGROUND_NEGATIVE_REPEAT`. It does not duplicate validation +tiles, positive tiles or normal reference-sample negatives. Repeated background +tiles receive deterministic `_hnXX` filenames and tile metadata records +`sample_role`, `repeat_index` and `is_repeated_background_negative`. + Train against the tile dataset by pointing the existing wrapper at the tile output directory: @@ -388,6 +409,13 @@ was clean on Postel-bos and Lommel-heide at thresholds `0.25` and `0.15`, but produced 38 detections on Kasterlee-bos even at `0.25`. That blocks it from becoming a V1 default until a hard-negative-balanced candidate improves. +The hard-negative-balanced `geointel-building-yolov8n-hardneg160r8e40-pt` +candidate reduced Kasterlee-bos false-positive pressure to 5/9/25 detections +at thresholds `0.25`/`0.15`/`0.05` and stayed at 0 detections on Postel-bos and +Lommel-heide across all tested thresholds. It also regressed dense-AOI F1 +against `geointel-building-yolov8n-expanded160e50-pt`, so it is useful model +quality evidence but not a V1 default. + Export calibration QA evidence for visual review: ```bash diff --git a/scripts/export_operator_yolo_tile_dataset.py b/scripts/export_operator_yolo_tile_dataset.py index 5b7b28b4..5157bc44 100644 --- a/scripts/export_operator_yolo_tile_dataset.py +++ b/scripts/export_operator_yolo_tile_dataset.py @@ -78,6 +78,12 @@ def parse_args() -> argparse.Namespace: default=float(os.environ.get("OPERATOR_YOLO_MIN_LABEL_PX", "4")), help="Minimum clipped box width/height in pixels before a tile label is kept.", ) + parser.add_argument( + "--background-negative-repeat", + type=int, + default=int(os.environ.get("OPERATOR_YOLO_BACKGROUND_NEGATIVE_REPEAT", "1")), + help="Repeat kept train/background negative tiles this many times for hard-negative balancing.", + ) parser.add_argument("--force", action="store_true", help="Remove and recreate output-dir before exporting.") return parser.parse_args() @@ -134,6 +140,22 @@ def keep_negative_tile(sample_slug: str, tile_index: int, negative_keep_ratio: f return bucket < negative_keep_ratio +def background_negative_repeat_count( + *, + is_negative: bool, + sample_role: str, + split: str, + background_negative_repeat: int, +) -> int: + if not is_negative: + return 1 + if split != "train": + return 1 + if sample_role != "background_candidate": + return 1 + return max(1, background_negative_repeat) + + def resolve_manifest_path(raw: str, manifest_path: Path) -> Path: path = Path(raw) if path.exists(): @@ -274,8 +296,10 @@ def export_sample_tiles( stride: int, negative_keep_ratio: float, min_label_px: float, + background_negative_repeat: int, ) -> list[dict[str, Any]]: sample_slug = str(sample["sample_slug"]) + sample_role = str(sample.get("sample_role") or "reference") split = "val" if sample_slug.lower() in val_slugs else "train" raster_path = resolve_manifest_path(str(sample["raster_path"]), manifest_path) reference_path = resolve_manifest_path(str(sample["reference_path"]), manifest_path) @@ -302,31 +326,43 @@ def export_sample_tiles( } ) continue - tile_name = f"{sample_slug}_{tile_index:04d}_r{tile_window.row_off}_c{tile_window.col_off}" - image_path = output_dir / "images" / split / f"{tile_name}.png" - label_path = output_dir / "labels" / split / f"{tile_name}.txt" - image_path.parent.mkdir(parents=True, exist_ok=True) - label_path.parent.mkdir(parents=True, exist_ok=True) - Image.fromarray(image_array_from_raster_window(dataset, tile_window)).save(image_path) - label_path.write_text("\n".join(labels) + ("\n" if labels else ""), encoding="utf-8") - exported.append( - { - "sample_slug": sample_slug, - "split": split, - "tile_index": tile_index, - "kept": True, - "image_path": str(image_path), - "label_path": str(label_path), - "label_count": len(labels), - "is_negative": is_negative, - "window": { - "row_off": tile_window.row_off, - "col_off": tile_window.col_off, - "height": tile_window.height, - "width": tile_window.width, - }, - } + repeats = background_negative_repeat_count( + is_negative=is_negative, + sample_role=sample_role, + split=split, + background_negative_repeat=background_negative_repeat, ) + image_array = image_array_from_raster_window(dataset, tile_window) + for repeat_index in range(repeats): + repeat_suffix = f"_hn{repeat_index + 1:02d}" if repeats > 1 else "" + tile_name = f"{sample_slug}_{tile_index:04d}_r{tile_window.row_off}_c{tile_window.col_off}{repeat_suffix}" + image_path = output_dir / "images" / split / f"{tile_name}.png" + label_path = output_dir / "labels" / split / f"{tile_name}.txt" + image_path.parent.mkdir(parents=True, exist_ok=True) + label_path.parent.mkdir(parents=True, exist_ok=True) + Image.fromarray(image_array).save(image_path) + label_path.write_text("\n".join(labels) + ("\n" if labels else ""), encoding="utf-8") + exported.append( + { + "sample_slug": sample_slug, + "sample_role": sample_role, + "split": split, + "tile_index": tile_index, + "repeat_index": repeat_index, + "kept": True, + "image_path": str(image_path), + "label_path": str(label_path), + "label_count": len(labels), + "is_negative": is_negative, + "is_repeated_background_negative": repeats > 1, + "window": { + "row_off": tile_window.row_off, + "col_off": tile_window.col_off, + "height": tile_window.height, + "width": tile_window.width, + }, + } + ) return exported @@ -355,6 +391,7 @@ def main() -> int: stride=args.stride, negative_keep_ratio=args.negative_keep_ratio, min_label_px=args.min_label_px, + background_negative_repeat=args.background_negative_repeat, ) ) @@ -375,6 +412,7 @@ def main() -> int: "tile_size": args.tile_size, "stride": args.stride, "negative_keep_ratio": args.negative_keep_ratio, + "background_negative_repeat": args.background_negative_repeat, "min_label_px": args.min_label_px, "source_sample_count": len(samples), "tile_count": len(kept_tiles),