From b8e40ba7a81ce59ecfc7b471a79b117e32c60f1b Mon Sep 17 00:00:00 2001 From: Jens Date: Wed, 29 Jul 2026 23:10:19 +0200 Subject: [PATCH] Prioritize recall in failure-driven sampling --- .../test_failure_driven_yolo_sampling.py | 30 ++++++++++++++++++- scripts/build_failure_driven_yolo_sampling.py | 16 +++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_failure_driven_yolo_sampling.py b/backend/tests/test_failure_driven_yolo_sampling.py index 71f395f6..19bdb0f8 100644 --- a/backend/tests/test_failure_driven_yolo_sampling.py +++ b/backend/tests/test_failure_driven_yolo_sampling.py @@ -163,11 +163,39 @@ def test_sampling_targets_failed_calibration_contexts_without_using_protected_ti ) assert paths.count(str(Path("/tmp/industry-pos.png").resolve())) == 5 - assert paths.count(str(Path("/tmp/industry-neg.png").resolve())) == 6 + assert paths.count(str(Path("/tmp/industry-neg.png").resolve())) == 1 assert paths.count(str(Path("/tmp/suburban-pos.png").resolve())) == 3 assert not any("protected" in path for path in paths) assert metadata["weak_recall_contexts"] == ["flanders:industrial"] assert metadata["weak_precision_contexts"] == ["flanders:industrial"] + assert metadata["recall_dominant_regions"] == ["flanders"] + + +def test_recall_dominance_does_not_suppress_negatives_when_background_gate_failed() -> None: + manifest = {"samples": [ + {"sample_slug": "positive", "split": "train", "region": "flanders", "context": "industrial"}, + {"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"}, + ]} + summary = {"tiles": [ + {"sample_slug": "positive", "split": "train", "label_count": 1, "image_path": "/tmp/positive.png"}, + {"sample_slug": "negative", "split": "train", "label_count": 0, "image_path": "/tmp/negative.png"}, + ]} + assessment = { + "status": "continue_training_loop", + "gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4, + "max_pure_empty_false_positives": 0}, + "calibration": {"regions": { + "flanders": {"f1": .25, "precision": .4, "recall": .2}, + }}, + "background": {"pure_empty_false_positives": 1}, + } + + paths, metadata = MODULE.build_sampling( + summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0, + ) + + assert paths.count(str(Path("/tmp/negative.png").resolve())) == 4 + assert metadata["recall_dominant_regions"] == [] def test_region_cap_drops_only_repeats_and_preserves_every_unique_tile() -> None: diff --git a/scripts/build_failure_driven_yolo_sampling.py b/scripts/build_failure_driven_yolo_sampling.py index 343d762d..89e701ea 100644 --- a/scripts/build_failure_driven_yolo_sampling.py +++ b/scripts/build_failure_driven_yolo_sampling.py @@ -94,6 +94,15 @@ def build_sampling( and background["pure_empty_false_positives"] > gates["max_pure_empty_false_positives"] ) + recall_dominant_regions = { + region + for region in weak_recall_regions & weak_precision_regions + if not background_failed + and ( + regions[region]["recall"] / gates["min_region_recall"] + < regions[region]["precision"] / gates["min_region_precision"] + ) + } weak_recall_contexts: set[tuple[str, str]] = set() weak_precision_contexts: set[tuple[str, str]] = set() for sample_slug, metrics in evaluation.get("samples", {}).items(): @@ -140,7 +149,11 @@ def build_sampling( # Precision-only correction still needs positive examples to avoid # shifting the classifier toward background and sacrificing recall. repeat = precision_positive_repeat - if tile["label_count"] == 0 and (background_failed or region in weak_precision_regions): + if ( + tile["label_count"] == 0 + and (background_failed or region in weak_precision_regions) + and region not in recall_dominant_regions + ): repeat = ( context_negative_repeat if context_key in targeted_negative_contexts @@ -197,6 +210,7 @@ def build_sampling( "failure_evidence_source": "test" if assessment.get("test") else "calibration", "weak_recall_regions": sorted(weak_recall_regions), "weak_precision_regions": sorted(weak_precision_regions), + "recall_dominant_regions": sorted(recall_dominant_regions), "weak_recall_contexts": [f"{region}:{context}" for region, context in sorted(weak_recall_contexts)], "weak_precision_contexts": [f"{region}:{context}" for region, context in sorted(weak_precision_contexts)], "targeted_negative_contexts": [