Prioritize recall in failure-driven sampling
This commit is contained in:
@@ -163,11 +163,39 @@ def test_sampling_targets_failed_calibration_contexts_without_using_protected_ti
|
||||
)
|
||||
|
||||
assert paths.count(str(Path("/tmp/industry-pos.png").resolve())) == 5
|
||||
assert paths.count(str(Path("/tmp/industry-neg.png").resolve())) == 6
|
||||
assert paths.count(str(Path("/tmp/industry-neg.png").resolve())) == 1
|
||||
assert paths.count(str(Path("/tmp/suburban-pos.png").resolve())) == 3
|
||||
assert not any("protected" in path for path in paths)
|
||||
assert metadata["weak_recall_contexts"] == ["flanders:industrial"]
|
||||
assert metadata["weak_precision_contexts"] == ["flanders:industrial"]
|
||||
assert metadata["recall_dominant_regions"] == ["flanders"]
|
||||
|
||||
|
||||
def test_recall_dominance_does_not_suppress_negatives_when_background_gate_failed() -> None:
|
||||
manifest = {"samples": [
|
||||
{"sample_slug": "positive", "split": "train", "region": "flanders", "context": "industrial"},
|
||||
{"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"},
|
||||
]}
|
||||
summary = {"tiles": [
|
||||
{"sample_slug": "positive", "split": "train", "label_count": 1, "image_path": "/tmp/positive.png"},
|
||||
{"sample_slug": "negative", "split": "train", "label_count": 0, "image_path": "/tmp/negative.png"},
|
||||
]}
|
||||
assessment = {
|
||||
"status": "continue_training_loop",
|
||||
"gates": {"min_region_f1": .45, "min_region_precision": .5, "min_region_recall": .4,
|
||||
"max_pure_empty_false_positives": 0},
|
||||
"calibration": {"regions": {
|
||||
"flanders": {"f1": .25, "precision": .4, "recall": .2},
|
||||
}},
|
||||
"background": {"pure_empty_false_positives": 1},
|
||||
}
|
||||
|
||||
paths, metadata = MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0,
|
||||
)
|
||||
|
||||
assert paths.count(str(Path("/tmp/negative.png").resolve())) == 4
|
||||
assert metadata["recall_dominant_regions"] == []
|
||||
|
||||
|
||||
def test_region_cap_drops_only_repeats_and_preserves_every_unique_tile() -> None:
|
||||
|
||||
@@ -94,6 +94,15 @@ def build_sampling(
|
||||
and background["pure_empty_false_positives"]
|
||||
> gates["max_pure_empty_false_positives"]
|
||||
)
|
||||
recall_dominant_regions = {
|
||||
region
|
||||
for region in weak_recall_regions & weak_precision_regions
|
||||
if not background_failed
|
||||
and (
|
||||
regions[region]["recall"] / gates["min_region_recall"]
|
||||
< regions[region]["precision"] / gates["min_region_precision"]
|
||||
)
|
||||
}
|
||||
weak_recall_contexts: set[tuple[str, str]] = set()
|
||||
weak_precision_contexts: set[tuple[str, str]] = set()
|
||||
for sample_slug, metrics in evaluation.get("samples", {}).items():
|
||||
@@ -140,7 +149,11 @@ def build_sampling(
|
||||
# Precision-only correction still needs positive examples to avoid
|
||||
# shifting the classifier toward background and sacrificing recall.
|
||||
repeat = precision_positive_repeat
|
||||
if tile["label_count"] == 0 and (background_failed or region in weak_precision_regions):
|
||||
if (
|
||||
tile["label_count"] == 0
|
||||
and (background_failed or region in weak_precision_regions)
|
||||
and region not in recall_dominant_regions
|
||||
):
|
||||
repeat = (
|
||||
context_negative_repeat
|
||||
if context_key in targeted_negative_contexts
|
||||
@@ -197,6 +210,7 @@ def build_sampling(
|
||||
"failure_evidence_source": "test" if assessment.get("test") else "calibration",
|
||||
"weak_recall_regions": sorted(weak_recall_regions),
|
||||
"weak_precision_regions": sorted(weak_precision_regions),
|
||||
"recall_dominant_regions": sorted(recall_dominant_regions),
|
||||
"weak_recall_contexts": [f"{region}:{context}" for region, context in sorted(weak_recall_contexts)],
|
||||
"weak_precision_contexts": [f"{region}:{context}" for region, context in sorted(weak_precision_contexts)],
|
||||
"targeted_negative_contexts": [
|
||||
|
||||
Reference in New Issue
Block a user