diff --git a/backend/tests/test_failure_driven_yolo_sampling.py b/backend/tests/test_failure_driven_yolo_sampling.py index 5b99c298..71f395f6 100644 --- a/backend/tests/test_failure_driven_yolo_sampling.py +++ b/backend/tests/test_failure_driven_yolo_sampling.py @@ -251,6 +251,33 @@ def test_region_cap_rotates_repeats_between_sampling_rounds() -> None: assert second_metadata["sampling_round"] == 2 +def test_precision_guard_band_keeps_near_gate_region_stabilized() -> None: + manifest = {"samples": [ + {"sample_slug": "wa-positive", "split": "train", "region": "wallonia", "context": "rural-town"}, + {"sample_slug": "wa-negative", "split": "train", "region": "wallonia", "context": "farmland-hard-negative"}, + ]} + summary = {"tiles": [ + {"sample_slug": "wa-positive", "split": "train", "label_count": 1, "image_path": "/tmp/wa-positive.png"}, + {"sample_slug": "wa-negative", "split": "train", "label_count": 0, "image_path": "/tmp/wa-negative.png"}, + ]} + assessment = { + "status": "continue_training_loop", + "gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4, + "max_pure_empty_false_positives": 0}, + "calibration": {"regions": { + "wallonia": {"f1": .6, "precision": .52, "recall": .7}, + }}, + } + + paths, metadata = MODULE.build_sampling( + summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0, + ) + + assert "wallonia" in metadata["weak_precision_regions"] + assert paths.count(str(Path("/tmp/wa-negative.png").resolve())) == 4 + assert metadata["precision_guard_band"] == .03 + + def test_coastal_precision_failure_targets_port_and_dunes_negatives() -> None: manifest = {"samples": [ {"sample_slug": "coastal-train", "split": "train", "region": "flanders", "context": "coastal-urban"}, diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 2f4b3666..52fa8459 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11944,6 +11944,12 @@ Verified: unique train tile and all protected-split exclusions. - `py -3 -m pytest -q backend/tests/test_failure_driven_yolo_sampling.py backend/tests/test_belgium_training_loop.py` (`20 passed`). +- Iteration 3 raised aggregate calibration F1 to `0.605`, passed Brussels and + every Wallonia gate, and isolated the remaining blocker to Flanders. When + iteration 4 removed Wallonia stabilization, Wallonia precision regressed + narrowly from `0.525` to `0.496`. Added a `0.03` precision/recall sampling + guard-band so a just-passing region retains stabilizing evidence while the + hard-failing region remains the primary target (`21 passed`). Open: diff --git a/docs/TODO.md b/docs/TODO.md index 3db2f29a..91a45aa4 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -1020,5 +1020,6 @@ This file now starts with the current implementation status. Older preparation/b - [x] Render and inspect a 56-tile contact sheet covering all 14 new AOIs before retraining. - [ ] Run the v43 calibration-first, failure-driven CUDA loop against the frozen regional release gates. - [x] Rotate region-capped repeat windows deterministically per loop round so persistent failures cannot reuse an identical training list indefinitely. +- [x] Keep near-gate regional precision/recall stabilization inside a 0.03 sampling guard-band to prevent cross-region seesaw regressions. - [ ] Open protected test and pure-background evidence only after every calibration gate passes. - [ ] Queue final representative human sign-off only after all automated gates pass, then promote and redeploy the exact checksummed model. diff --git a/scripts/build_failure_driven_yolo_sampling.py b/scripts/build_failure_driven_yolo_sampling.py index 14fb1aa9..343d762d 100644 --- a/scripts/build_failure_driven_yolo_sampling.py +++ b/scripts/build_failure_driven_yolo_sampling.py @@ -51,6 +51,8 @@ def build_sampling( context_negative_repeat: int = 6, max_region_share: float = 0.65, sampling_round: int = 0, + precision_guard_band: float = 0.03, + recall_guard_band: float = 0.03, ) -> tuple[list[str], dict[str, Any]]: if assessment.get("status") != "continue_training_loop": raise ValueError("Failure-driven sampling requires a failed assessment") @@ -66,6 +68,8 @@ def build_sampling( raise ValueError("max_region_share must be in (0, 1]") if sampling_round < 0: raise ValueError("sampling_round must be non-negative") + if precision_guard_band < 0 or recall_guard_band < 0: + raise ValueError("Guard bands must be non-negative") samples = {item["sample_slug"]: item for item in manifest["samples"]} gates = assessment["gates"] @@ -77,12 +81,12 @@ def build_sampling( region for region, metrics in regions.items() if metrics["f1"] < gates["min_region_f1"] - or metrics["recall"] < gates["min_region_recall"] + or metrics["recall"] < gates["min_region_recall"] + recall_guard_band } weak_precision_regions = { region for region, metrics in regions.items() - if metrics["precision"] < gates["min_region_precision"] + if metrics["precision"] < gates["min_region_precision"] + precision_guard_band } background = assessment.get("background") background_failed = bool( @@ -206,6 +210,8 @@ def build_sampling( "context_negative_repeat": context_negative_repeat, "max_region_share": max_region_share, "sampling_round": sampling_round, + "precision_guard_band": precision_guard_band, + "recall_guard_band": recall_guard_band, "source_train_tile_count": sum( 1 for tile in summary["tiles"] @@ -235,6 +241,8 @@ def main() -> int: parser.add_argument("--context-negative-repeat", type=int, default=6) parser.add_argument("--max-region-share", type=float, default=0.65) parser.add_argument("--sampling-round", type=int) + parser.add_argument("--precision-guard-band", type=float, default=0.03) + parser.add_argument("--recall-guard-band", type=float, default=0.03) args = parser.parse_args() summary = json.loads(args.summary.read_text(encoding="utf-8")) @@ -255,6 +263,8 @@ def main() -> int: context_negative_repeat=args.context_negative_repeat, max_region_share=args.max_region_share, sampling_round=sampling_round, + precision_guard_band=args.precision_guard_band, + recall_guard_band=args.recall_guard_band, ) args.output_dir.mkdir(parents=True, exist_ok=True) train_list = args.output_dir / "train-failure-driven.txt"