diff --git a/backend/tests/test_failure_driven_yolo_sampling.py b/backend/tests/test_failure_driven_yolo_sampling.py index e613be79..5b99c298 100644 --- a/backend/tests/test_failure_driven_yolo_sampling.py +++ b/backend/tests/test_failure_driven_yolo_sampling.py @@ -209,6 +209,48 @@ def test_region_cap_drops_only_repeats_and_preserves_every_unique_tile() -> None assert metadata["sampled_entries_by_region"]["flanders"] / len(paths) <= .65 +def test_region_cap_rotates_repeats_between_sampling_rounds() -> None: + manifest = {"samples": [ + {"sample_slug": "fl", "split": "train", "region": "flanders", "context": "industrial"}, + {"sample_slug": "wa", "split": "train", "region": "wallonia", "context": "rural-town"}, + {"sample_slug": "br", "split": "train", "region": "brussels", "context": "dense-urban"}, + ]} + summary = {"tiles": [ + {"sample_slug": "fl", "split": "train", "label_count": 2, "image_path": f"/tmp/fl-{index}.png"} + for index in range(4) + ] + [ + {"sample_slug": "wa", "split": "train", "label_count": 2, "image_path": f"/tmp/wa-{index}.png"} + for index in range(2) + ] + [ + {"sample_slug": "br", "split": "train", "label_count": 2, "image_path": f"/tmp/br-{index}.png"} + for index in range(2) + ]} + assessment = { + "status": "continue_training_loop", + "gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4, + "max_pure_empty_false_positives": 0}, + "calibration": {"regions": { + "flanders": {"f1": .2, "precision": .3, "recall": .2}, + "wallonia": {"f1": .6, "precision": .6, "recall": .6}, + "brussels": {"f1": .6, "precision": .6, "recall": .6}, + }}, + } + + first, first_metadata = MODULE.build_sampling( + summary=summary, manifest=manifest, assessment=assessment, + positive_repeat=5, max_region_share=.65, sampling_round=1, + ) + second, second_metadata = MODULE.build_sampling( + summary=summary, manifest=manifest, assessment=assessment, + positive_repeat=5, max_region_share=.65, sampling_round=2, + ) + + assert first != second + assert set(first) == set(second) + assert first_metadata["sampling_round"] == 1 + assert second_metadata["sampling_round"] == 2 + + def test_coastal_precision_failure_targets_port_and_dunes_negatives() -> None: manifest = {"samples": [ {"sample_slug": "coastal-train", "split": "train", "region": "flanders", "context": "coastal-urban"}, diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 47d2e651..2f4b3666 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11933,6 +11933,17 @@ Verified: AI-assisted inspection found labels aligned with visible roof footprints and retained the deliberately sparse hard-negative contexts. - NVIDIA preflight: RTX 4080 SUPER idle and available before the v43 handoff. +- Iteration 2 selected its epoch-5 checkpoint, reached aggregate calibration F1 + `0.584` and zero pure-empty detections, but remained closed because Flanders + F1/precision/recall and Wallonia precision failed. Iteration 3 started from + the exact rejected checkpoint without opening test/background. +- Confirmed Ultralytics preserves duplicate paths from the 4,077-entry + failure-driven list. Added deterministic per-round rotation of the repeats + retained by the 65% regional cap, preventing persistent identical failures + from producing an identical capped list forever while preserving every + unique train tile and all protected-split exclusions. +- `py -3 -m pytest -q backend/tests/test_failure_driven_yolo_sampling.py backend/tests/test_belgium_training_loop.py` + (`20 passed`). Open: diff --git a/docs/TODO.md b/docs/TODO.md index b01dcbd4..3db2f29a 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1019,5 +1019,6 @@ This file now starts with the current implementation status. Older preparation/b - [x] Pass structural corpus and 56,474-label tile-quality audits with zero invalid or missing labels and zero spatial leakage failures. - [x] Render and inspect a 56-tile contact sheet covering all 14 new AOIs before retraining. - [ ] Run the v43 calibration-first, failure-driven CUDA loop against the frozen regional release gates. +- [x] Rotate region-capped repeat windows deterministically per loop round so persistent failures cannot reuse an identical training list indefinitely. - [ ] Open protected test and pure-background evidence only after every calibration gate passes. - [ ] Queue final representative human sign-off only after all automated gates pass, then promote and redeploy the exact checksummed model. diff --git a/scripts/build_failure_driven_yolo_sampling.py b/scripts/build_failure_driven_yolo_sampling.py index c2e6a247..14fb1aa9 100644 --- a/scripts/build_failure_driven_yolo_sampling.py +++ b/scripts/build_failure_driven_yolo_sampling.py @@ -7,6 +7,7 @@ import argparse import hashlib import json import math +import re from collections import Counter from pathlib import Path from typing import Any @@ -49,6 +50,7 @@ def build_sampling( context_positive_repeat: int = 5, context_negative_repeat: int = 6, max_region_share: float = 0.65, + sampling_round: int = 0, ) -> tuple[list[str], dict[str, Any]]: if assessment.get("status") != "continue_training_loop": raise ValueError("Failure-driven sampling requires a failed assessment") @@ -62,6 +64,8 @@ def build_sampling( raise ValueError("Repeat factors must be positive") if not 0 < max_region_share <= 1: raise ValueError("max_region_share must be in (0, 1]") + if sampling_round < 0: + raise ValueError("sampling_round must be non-negative") samples = {item["sample_slug"]: item for item in manifest["samples"]} gates = assessment["gates"] @@ -172,7 +176,13 @@ def build_sampling( base_paths = base_paths_by_region[region] extra_limit = capped_counts[region] - len(base_paths) image_paths.extend(base_paths) - image_paths.extend(extra_paths_by_region[region][:extra_limit]) + extras = extra_paths_by_region[region] + if extras and extra_limit < len(extras): + # Rotate the capped repeat window between loop rounds so persistent + # failures cannot yield the exact same training list indefinitely. + offset = sampling_round % len(extras) + extras = extras[offset:] + extras[:offset] + image_paths.extend(extras[:extra_limit]) repeat_counts = Counter({region: capped_counts[region] for region in capped_counts}) if not image_paths: raise ValueError("No train-only tiles selected") @@ -195,6 +205,7 @@ def build_sampling( "context_positive_repeat": context_positive_repeat, "context_negative_repeat": context_negative_repeat, "max_region_share": max_region_share, + "sampling_round": sampling_round, "source_train_tile_count": sum( 1 for tile in summary["tiles"] @@ -223,11 +234,16 @@ def main() -> int: parser.add_argument("--context-positive-repeat", type=int, default=5) parser.add_argument("--context-negative-repeat", type=int, default=6) parser.add_argument("--max-region-share", type=float, default=0.65) + parser.add_argument("--sampling-round", type=int) args = parser.parse_args() summary = json.loads(args.summary.read_text(encoding="utf-8")) manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8")) assessment = json.loads(args.assessment.read_text(encoding="utf-8")) + sampling_round = args.sampling_round + if sampling_round is None: + match = re.fullmatch(r"iteration-(\d+)", args.output_dir.parent.name) + sampling_round = int(match.group(1)) if match else 0 paths, metadata = build_sampling( summary=summary, manifest=manifest, @@ -238,6 +254,7 @@ def main() -> int: context_positive_repeat=args.context_positive_repeat, context_negative_repeat=args.context_negative_repeat, max_region_share=args.max_region_share, + sampling_round=sampling_round, ) args.output_dir.mkdir(parents=True, exist_ok=True) train_list = args.output_dir / "train-failure-driven.txt" diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index f1eef4ef..e2dcf1eb 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -164,6 +164,7 @@ def failure_sampling_command( corpus_manifest: Path, assessment: Path, output_dir: Path, + sampling_round: int = 0, ) -> list[str]: return [ sys.executable, @@ -172,6 +173,7 @@ def failure_sampling_command( "--corpus-manifest", str(corpus_manifest), "--assessment", str(assessment), "--output-dir", str(output_dir), + "--sampling-round", str(sampling_round), ] @@ -443,6 +445,7 @@ def main() -> int: corpus_manifest=args.corpus_manifest, assessment=assessment, output_dir=sampling_dir, + sampling_round=index, ), iteration_dir / "failure-driven-sampling.log", )