Add fail-closed failure-driven YOLO sampling

2026-07-27 04:04:45 +02:00
parent 1e685431e3
commit 7744803461
4 changed files with 215 additions and 0 deletions
@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).parents[2] / "scripts" / "build_failure_driven_yolo_sampling.py"
+SPEC = importlib.util.spec_from_file_location("failure_sampling", SCRIPT)
+assert SPEC and SPEC.loader
+MODULE = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MODULE)
+
+
+def test_sampling_repeats_only_failed_region_train_tiles() -> None:
+    manifest = {
+        "samples": [
+            {"sample_slug": "train-fl", "split": "train", "region": "flanders"},
+            {"sample_slug": "train-wa", "split": "train", "region": "wallonia"},
+            {"sample_slug": "test-fl", "split": "test", "region": "flanders"},
+        ]
+    }
+    summary = {
+        "tiles": [
+            {"sample_slug": "train-fl", "split": "train", "label_count": 2, "image_path": "/tmp/fl-pos.png"},
+            {"sample_slug": "train-fl", "split": "train", "label_count": 0, "image_path": "/tmp/fl-neg.png"},
+            {"sample_slug": "train-wa", "split": "train", "label_count": 1, "image_path": "/tmp/wa-pos.png"},
+            {"sample_slug": "test-fl", "split": "val", "label_count": 1, "image_path": "/tmp/protected.png"},
+        ]
+    }
+    assessment = {
+        "status": "continue_training_loop",
+        "gates": {
+            "min_region_f1": 0.45,
+            "min_region_precision": 0.5,
+            "min_region_recall": 0.4,
+            "max_pure_empty_false_positives": 0,
+        },
+        "test": {
+            "regions": {
+                "flanders": {"f1": 0.2, "precision": 0.3, "recall": 0.2},
+                "wallonia": {"f1": 0.6, "precision": 0.6, "recall": 0.6},
+            }
+        },
+        "background": {"pure_empty_false_positives": 2},
+    }
+    paths, metadata = MODULE.build_sampling(
+        summary=summary, manifest=manifest, assessment=assessment
+    )
+    assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 3
+    assert paths.count(str(Path("/tmp/fl-neg.png").resolve())) == 4
+    assert paths.count(str(Path("/tmp/wa-pos.png").resolve())) == 1
+    assert not any("protected" in path for path in paths)
+    assert metadata["protected_samples_in_training"] == []
+    assert metadata["weak_recall_regions"] == ["flanders"]