Make building training loop calibration-gated

backend/tests/test_belgium_training_loop.py

@@ -92,3 +92,49 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
     )
     assert result.returncode != 0
     assert "Dataset audit is not ok" in result.stderr
+
+
+def test_calibration_failure_blocks_protected_evaluation() -> None:
+    chosen = {
+        "threshold": 0.1,
+        "aggregate": {"f1": 0.54},
+        "regions": {
+            "flanders": {"f1": 0.44, "precision": 0.49, "recall": 0.39},
+            "wallonia": {"f1": 0.6, "precision": 0.6, "recall": 0.6},
+        },
+        "pure_empty_false_positives": 0,
+    }
+    failures = MODULE.calibration_failures(
+        chosen,
+        min_aggregate_f1=0.55,
+        min_region_f1=0.45,
+        min_region_precision=0.5,
+        min_region_recall=0.4,
+        max_pure_empty_fp=0,
+    )
+    assert failures == [
+        "calibration_aggregate_f1_below_gate",
+        "calibration_flanders_f1_below_gate",
+        "calibration_flanders_precision_below_gate",
+        "calibration_flanders_recall_below_gate",
+    ]
+
+
+def test_threshold_selection_uses_worst_region_then_aggregate() -> None:
+    report = {
+        "sweeps": [
+            {
+                "threshold": 0.1,
+                "aggregate": {"f1": 0.8},
+                "regions": {"a": {"f1": 0.4}, "b": {"f1": 0.7}},
+                "pure_empty_false_positives": 0,
+            },
+            {
+                "threshold": 0.2,
+                "aggregate": {"f1": 0.6},
+                "regions": {"a": {"f1": 0.5}, "b": {"f1": 0.5}},
+                "pure_empty_false_positives": 0,
+            },
+        ]
+    }
+    assert MODULE.select_calibration_threshold(report)["threshold"] == 0.2

scripts/build_regional_yolo_expert_dataset.py

@@ -25,28 +25,38 @@ def main() -> int:
     parser.add_argument("--output-dir", type=Path, required=True)
     parser.add_argument("--positive-repeat", type=int, default=2)
     parser.add_argument("--negative-repeat", type=int, default=1)
+    parser.add_argument(
+        "--other-region-repeat",
+        type=int,
+        default=0,
+        help="Include each train tile outside the expert region this many times.",
+    )
     args = parser.parse_args()
-    if args.positive_repeat < 1 or args.negative_repeat < 1:
+    if args.positive_repeat < 1 or args.negative_repeat < 1 or args.other_region_repeat < 0:
-        raise SystemExit("repeat factors must be positive")
+        raise SystemExit("regional repeats must be positive and other-region repeat non-negative")
 
     summary = json.loads(args.summary.read_text(encoding="utf-8"))
     manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
     samples = {item["sample_slug"]: item for item in manifest["samples"]}
     paths: list[str] = []
     selected_samples: set[str] = set()
-    positive_tiles = negative_tiles = 0
+    positive_tiles = negative_tiles = other_region_tiles = 0
     for tile in summary["tiles"]:
         sample = samples[tile["sample_slug"]]
         if sample["split"] != "train" or tile["split"] != "train":
             continue
-        if sample["region"] != args.region:
-            continue
         positive = int(tile.get("label_count") or 0) > 0
+        if sample["region"] == args.region:
             repeat = args.positive_repeat if positive else args.negative_repeat
-        paths.extend([str(Path(tile["image_path"]).resolve())] * repeat)
-        selected_samples.add(tile["sample_slug"])
             positive_tiles += int(positive)
             negative_tiles += int(not positive)
+        else:
+            repeat = args.other_region_repeat
+            other_region_tiles += int(repeat > 0)
+        if repeat == 0:
+            continue
+        paths.extend([str(Path(tile["image_path"]).resolve())] * repeat)
+        selected_samples.add(tile["sample_slug"])
     if not paths or not positive_tiles:
         raise SystemExit(f"no positive train tiles found for region {args.region!r}")
 
@@ -69,8 +79,10 @@ def main() -> int:
         "corpus_manifest_sha256": sha256(args.corpus_manifest),
         "positive_repeat": args.positive_repeat,
         "negative_repeat": args.negative_repeat,
+        "other_region_repeat": args.other_region_repeat,
         "source_positive_tile_count": positive_tiles,
         "source_negative_tile_count": negative_tiles,
+        "source_other_region_tile_count": other_region_tiles,
         "sampled_train_entry_count": len(paths),
         "selected_train_samples": sorted(selected_samples),
         "protected_samples_in_training": [],

scripts/run_belgium_building_training_loop.py

@@ -29,6 +29,45 @@ def write_json(path: Path, value: dict[str, Any]) -> None:
     temporary.replace(path)
 
 
+def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
+    """Choose a threshold without consulting test or background evidence."""
+    eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
+    if not eligible:
+        eligible = report["sweeps"]
+    return max(
+        eligible,
+        key=lambda item: (
+            min(region["f1"] for region in item["regions"].values()),
+            item["aggregate"]["f1"],
+            -item["pure_empty_false_positives"],
+        ),
+    )
+
+
+def calibration_failures(
+    chosen: dict[str, Any],
+    *,
+    min_aggregate_f1: float,
+    min_region_f1: float,
+    min_region_precision: float,
+    min_region_recall: float,
+    max_pure_empty_fp: int,
+) -> list[str]:
+    failures: list[str] = []
+    if chosen["aggregate"]["f1"] < min_aggregate_f1:
+        failures.append("calibration_aggregate_f1_below_gate")
+    for region, values in chosen["regions"].items():
+        if values["f1"] < min_region_f1:
+            failures.append(f"calibration_{region}_f1_below_gate")
+        if values["precision"] < min_region_precision:
+            failures.append(f"calibration_{region}_precision_below_gate")
+        if values["recall"] < min_region_recall:
+            failures.append(f"calibration_{region}_recall_below_gate")
+    if chosen["pure_empty_false_positives"] > max_pure_empty_fp:
+        failures.append("calibration_pure_empty_false_positive_gate_failed")
+    return failures
+
+
 def training_command(
     yolo: str,
     *,
@@ -112,6 +151,11 @@ def main() -> int:
     parser.add_argument("--translate", type=float, default=0.1)
     parser.add_argument("--seed", type=int, default=20260731)
     parser.add_argument("--yolo", default="yolo")
+    parser.add_argument("--min-aggregate-f1", type=float, default=0.55)
+    parser.add_argument("--min-region-f1", type=float, default=0.45)
+    parser.add_argument("--min-region-precision", type=float, default=0.5)
+    parser.add_argument("--min-region-recall", type=float, default=0.4)
+    parser.add_argument("--max-pure-empty-fp", type=int, default=0)
     parser.add_argument("--dry-run", action="store_true")
     args = parser.parse_args()
     if args.iterations < 1:
@@ -174,11 +218,7 @@ def main() -> int:
         shutil.copy2(best, candidate)
 
         reports: dict[str, Path] = {}
-        for role, summary in (
+        for role, summary in (("calibration", args.calibration_summary),):
-            ("calibration", args.calibration_summary),
-            ("test", args.test_summary),
-            ("background", args.background_summary),
-        ):
             report = iteration_dir / f"{role}.json"
             reports[role] = report
             run(
@@ -203,18 +243,60 @@ def main() -> int:
                 iteration_dir / f"{role}.log",
             )
         assessment = iteration_dir / "assessment.json"
+        calibration = json.loads(reports["calibration"].read_text(encoding="utf-8"))
+        chosen = select_calibration_threshold(calibration)
+        failures = calibration_failures(
+            chosen,
+            min_aggregate_f1=args.min_aggregate_f1,
+            min_region_f1=args.min_region_f1,
+            min_region_precision=args.min_region_precision,
+            min_region_recall=args.min_region_recall,
+            max_pure_empty_fp=args.max_pure_empty_fp,
+        )
+        if failures:
+            write_json(
+                assessment,
+                {
+                    "schema_version": 1,
+                    "status": "continue_training_loop",
+                    "phase": "calibration_rejected",
+                    "threshold_selection_source": "calibration_only",
+                    "selected_threshold": chosen["threshold"],
+                    "calibration": chosen,
+                    "test": None,
+                    "background": None,
+                    "failures": failures,
+                },
+            )
+        else:
+            for role, summary in (
+                ("test", args.test_summary),
+                ("background", args.background_summary),
+            ):
+                report = iteration_dir / f"{role}.json"
+                reports[role] = report
+                run(
+                    [
+                        sys.executable,
+                        str(scripts_dir / "evaluate_belgium_building_candidate.py"),
+                        "--model", str(candidate),
+                        "--summary", str(summary),
+                        "--corpus-manifest", str(args.corpus_manifest),
+                        "--output", str(report),
+                        "--device", "cuda:0",
+                        "--max-det", str(args.max_det),
+                        "--imgsz", str(args.imgsz),
+                    ],
+                    iteration_dir / f"{role}.log",
+                )
             run(
                 [
                     sys.executable,
                     str(scripts_dir / "assess_belgium_building_training_iteration.py"),
-                "--calibration",
+                    "--calibration", str(reports["calibration"]),
-                str(reports["calibration"]),
+                    "--test", str(reports["test"]),
-                "--test",
+                    "--background", str(reports["background"]),
-                str(reports["test"]),
+                    "--output", str(assessment),
-                "--background",
-                str(reports["background"]),
-                "--output",
-                str(assessment),
                 ],
                 iteration_dir / "assessment.log",
                 allowed={0, 2},