Drive corpus sampling from calibration failures

backend/tests/test_failure_driven_yolo_sampling.py

@@ -52,3 +52,31 @@ def test_sampling_repeats_only_failed_region_train_tiles() -> None:
     assert not any("protected" in path for path in paths)
     assert metadata["protected_samples_in_training"] == []
     assert metadata["weak_recall_regions"] == ["flanders"]
+
+
+def test_sampling_can_use_calibration_before_test_is_opened() -> None:
+    manifest = {"samples": [{"sample_slug": "train-fl", "split": "train", "region": "flanders"}]}
+    summary = {
+        "tiles": [
+            {"sample_slug": "train-fl", "split": "train", "label_count": 1, "image_path": "/tmp/fl.png"}
+        ]
+    }
+    assessment = {
+        "status": "continue_training_loop",
+        "gates": {
+            "min_region_f1": 0.45,
+            "min_region_precision": 0.5,
+            "min_region_recall": 0.4,
+            "max_pure_empty_false_positives": 0,
+        },
+        "calibration": {
+            "regions": {"flanders": {"f1": 0.4, "precision": 0.6, "recall": 0.35}}
+        },
+        "test": None,
+        "background": None,
+    }
+    paths, metadata = MODULE.build_sampling(
+        summary=summary, manifest=manifest, assessment=assessment
+    )
+    assert len(paths) == 3
+    assert metadata["failure_evidence_source"] == "calibration"

scripts/build_failure_driven_yolo_sampling.py

@@ -34,7 +34,10 @@ def build_sampling(
 
     samples = {item["sample_slug"]: item for item in manifest["samples"]}
     gates = assessment["gates"]
-    regions = assessment["test"]["regions"]
+    evaluation = assessment.get("test") or assessment.get("calibration")
+    if not evaluation or "regions" not in evaluation:
+        raise ValueError("Assessment has no regional calibration or test evidence")
+    regions = evaluation["regions"]
     weak_recall_regions = {
         region
         for region, metrics in regions.items()
@@ -46,8 +49,10 @@ def build_sampling(
         for region, metrics in regions.items()
         if metrics["precision"] < gates["min_region_precision"]
     }
-    background_failed = (
+    background = assessment.get("background")
-        assessment["background"]["pure_empty_false_positives"]
+    background_failed = bool(
+        background
+        and background["pure_empty_false_positives"]
         > gates["max_pure_empty_false_positives"]
     )
 
@@ -77,6 +82,7 @@ def build_sampling(
         "schema_version": 1,
         "status": "ok",
         "strategy": "failed-region-positive-and-hard-negative-repeat",
+        "failure_evidence_source": "test" if assessment.get("test") else "calibration",
         "weak_recall_regions": sorted(weak_recall_regions),
         "weak_precision_regions": sorted(weak_precision_regions),
         "background_gate_failed": background_failed,

scripts/run_belgium_building_training_loop.py

@@ -262,6 +262,13 @@ def main() -> int:
                     "phase": "calibration_rejected",
                     "threshold_selection_source": "calibration_only",
                     "selected_threshold": chosen["threshold"],
+                    "gates": {
+                        "min_aggregate_f1": args.min_aggregate_f1,
+                        "min_region_f1": args.min_region_f1,
+                        "min_region_precision": args.min_region_precision,
+                        "min_region_recall": args.min_region_recall,
+                        "max_pure_empty_false_positives": args.max_pure_empty_fp,
+                    },
                     "calibration": chosen,
                     "test": None,
                     "background": None,