Make building training loop calibration-gated

2026-07-27 09:29:07 +02:00
parent 8155acf969
commit 1c6cf6a6a8
3 changed files with 168 additions and 28 deletions
@@ -92,3 +92,49 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
    )
    assert result.returncode != 0
    assert "Dataset audit is not ok" in result.stderr
+
+
+def test_calibration_failure_blocks_protected_evaluation() -> None:
+    chosen = {
+        "threshold": 0.1,
+        "aggregate": {"f1": 0.54},
+        "regions": {
+            "flanders": {"f1": 0.44, "precision": 0.49, "recall": 0.39},
+            "wallonia": {"f1": 0.6, "precision": 0.6, "recall": 0.6},
+        },
+        "pure_empty_false_positives": 0,
+    }
+    failures = MODULE.calibration_failures(
+        chosen,
+        min_aggregate_f1=0.55,
+        min_region_f1=0.45,
+        min_region_precision=0.5,
+        min_region_recall=0.4,
+        max_pure_empty_fp=0,
+    )
+    assert failures == [
+        "calibration_aggregate_f1_below_gate",
+        "calibration_flanders_f1_below_gate",
+        "calibration_flanders_precision_below_gate",
+        "calibration_flanders_recall_below_gate",
+    ]
+
+
+def test_threshold_selection_uses_worst_region_then_aggregate() -> None:
+    report = {
+        "sweeps": [
+            {
+                "threshold": 0.1,
+                "aggregate": {"f1": 0.8},
+                "regions": {"a": {"f1": 0.4}, "b": {"f1": 0.7}},
+                "pure_empty_false_positives": 0,
+            },
+            {
+                "threshold": 0.2,
+                "aggregate": {"f1": 0.6},
+                "regions": {"a": {"f1": 0.5}, "b": {"f1": 0.5}},
+                "pure_empty_false_positives": 0,
+            },
+        ]
+    }
+    assert MODULE.select_calibration_threshold(report)["threshold"] == 0.2