diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py index 1bb73ddb..2e3a42f9 100644 --- a/backend/tests/test_belgium_training_loop.py +++ b/backend/tests/test_belgium_training_loop.py @@ -92,3 +92,49 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: ) assert result.returncode != 0 assert "Dataset audit is not ok" in result.stderr + + +def test_calibration_failure_blocks_protected_evaluation() -> None: + chosen = { + "threshold": 0.1, + "aggregate": {"f1": 0.54}, + "regions": { + "flanders": {"f1": 0.44, "precision": 0.49, "recall": 0.39}, + "wallonia": {"f1": 0.6, "precision": 0.6, "recall": 0.6}, + }, + "pure_empty_false_positives": 0, + } + failures = MODULE.calibration_failures( + chosen, + min_aggregate_f1=0.55, + min_region_f1=0.45, + min_region_precision=0.5, + min_region_recall=0.4, + max_pure_empty_fp=0, + ) + assert failures == [ + "calibration_aggregate_f1_below_gate", + "calibration_flanders_f1_below_gate", + "calibration_flanders_precision_below_gate", + "calibration_flanders_recall_below_gate", + ] + + +def test_threshold_selection_uses_worst_region_then_aggregate() -> None: + report = { + "sweeps": [ + { + "threshold": 0.1, + "aggregate": {"f1": 0.8}, + "regions": {"a": {"f1": 0.4}, "b": {"f1": 0.7}}, + "pure_empty_false_positives": 0, + }, + { + "threshold": 0.2, + "aggregate": {"f1": 0.6}, + "regions": {"a": {"f1": 0.5}, "b": {"f1": 0.5}}, + "pure_empty_false_positives": 0, + }, + ] + } + assert MODULE.select_calibration_threshold(report)["threshold"] == 0.2 diff --git a/scripts/build_regional_yolo_expert_dataset.py b/scripts/build_regional_yolo_expert_dataset.py index de62b73d..95fa0e9a 100644 --- a/scripts/build_regional_yolo_expert_dataset.py +++ b/scripts/build_regional_yolo_expert_dataset.py @@ -25,28 +25,38 @@ def main() -> int: parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--positive-repeat", type=int, default=2) parser.add_argument("--negative-repeat", type=int, default=1) + parser.add_argument( + "--other-region-repeat", + type=int, + default=0, + help="Include each train tile outside the expert region this many times.", + ) args = parser.parse_args() - if args.positive_repeat < 1 or args.negative_repeat < 1: - raise SystemExit("repeat factors must be positive") + if args.positive_repeat < 1 or args.negative_repeat < 1 or args.other_region_repeat < 0: + raise SystemExit("regional repeats must be positive and other-region repeat non-negative") summary = json.loads(args.summary.read_text(encoding="utf-8")) manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8")) samples = {item["sample_slug"]: item for item in manifest["samples"]} paths: list[str] = [] selected_samples: set[str] = set() - positive_tiles = negative_tiles = 0 + positive_tiles = negative_tiles = other_region_tiles = 0 for tile in summary["tiles"]: sample = samples[tile["sample_slug"]] if sample["split"] != "train" or tile["split"] != "train": continue - if sample["region"] != args.region: - continue positive = int(tile.get("label_count") or 0) > 0 - repeat = args.positive_repeat if positive else args.negative_repeat + if sample["region"] == args.region: + repeat = args.positive_repeat if positive else args.negative_repeat + positive_tiles += int(positive) + negative_tiles += int(not positive) + else: + repeat = args.other_region_repeat + other_region_tiles += int(repeat > 0) + if repeat == 0: + continue paths.extend([str(Path(tile["image_path"]).resolve())] * repeat) selected_samples.add(tile["sample_slug"]) - positive_tiles += int(positive) - negative_tiles += int(not positive) if not paths or not positive_tiles: raise SystemExit(f"no positive train tiles found for region {args.region!r}") @@ -69,8 +79,10 @@ def main() -> int: "corpus_manifest_sha256": sha256(args.corpus_manifest), "positive_repeat": args.positive_repeat, "negative_repeat": args.negative_repeat, + "other_region_repeat": args.other_region_repeat, "source_positive_tile_count": positive_tiles, "source_negative_tile_count": negative_tiles, + "source_other_region_tile_count": other_region_tiles, "sampled_train_entry_count": len(paths), "selected_train_samples": sorted(selected_samples), "protected_samples_in_training": [], diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index 0d4a6d94..433572da 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -29,6 +29,45 @@ def write_json(path: Path, value: dict[str, Any]) -> None: temporary.replace(path) +def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]: + """Choose a threshold without consulting test or background evidence.""" + eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0] + if not eligible: + eligible = report["sweeps"] + return max( + eligible, + key=lambda item: ( + min(region["f1"] for region in item["regions"].values()), + item["aggregate"]["f1"], + -item["pure_empty_false_positives"], + ), + ) + + +def calibration_failures( + chosen: dict[str, Any], + *, + min_aggregate_f1: float, + min_region_f1: float, + min_region_precision: float, + min_region_recall: float, + max_pure_empty_fp: int, +) -> list[str]: + failures: list[str] = [] + if chosen["aggregate"]["f1"] < min_aggregate_f1: + failures.append("calibration_aggregate_f1_below_gate") + for region, values in chosen["regions"].items(): + if values["f1"] < min_region_f1: + failures.append(f"calibration_{region}_f1_below_gate") + if values["precision"] < min_region_precision: + failures.append(f"calibration_{region}_precision_below_gate") + if values["recall"] < min_region_recall: + failures.append(f"calibration_{region}_recall_below_gate") + if chosen["pure_empty_false_positives"] > max_pure_empty_fp: + failures.append("calibration_pure_empty_false_positive_gate_failed") + return failures + + def training_command( yolo: str, *, @@ -112,6 +151,11 @@ def main() -> int: parser.add_argument("--translate", type=float, default=0.1) parser.add_argument("--seed", type=int, default=20260731) parser.add_argument("--yolo", default="yolo") + parser.add_argument("--min-aggregate-f1", type=float, default=0.55) + parser.add_argument("--min-region-f1", type=float, default=0.45) + parser.add_argument("--min-region-precision", type=float, default=0.5) + parser.add_argument("--min-region-recall", type=float, default=0.4) + parser.add_argument("--max-pure-empty-fp", type=int, default=0) parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() if args.iterations < 1: @@ -174,11 +218,7 @@ def main() -> int: shutil.copy2(best, candidate) reports: dict[str, Path] = {} - for role, summary in ( - ("calibration", args.calibration_summary), - ("test", args.test_summary), - ("background", args.background_summary), - ): + for role, summary in (("calibration", args.calibration_summary),): report = iteration_dir / f"{role}.json" reports[role] = report run( @@ -203,22 +243,64 @@ def main() -> int: iteration_dir / f"{role}.log", ) assessment = iteration_dir / "assessment.json" - run( - [ - sys.executable, - str(scripts_dir / "assess_belgium_building_training_iteration.py"), - "--calibration", - str(reports["calibration"]), - "--test", - str(reports["test"]), - "--background", - str(reports["background"]), - "--output", - str(assessment), - ], - iteration_dir / "assessment.log", - allowed={0, 2}, + calibration = json.loads(reports["calibration"].read_text(encoding="utf-8")) + chosen = select_calibration_threshold(calibration) + failures = calibration_failures( + chosen, + min_aggregate_f1=args.min_aggregate_f1, + min_region_f1=args.min_region_f1, + min_region_precision=args.min_region_precision, + min_region_recall=args.min_region_recall, + max_pure_empty_fp=args.max_pure_empty_fp, ) + if failures: + write_json( + assessment, + { + "schema_version": 1, + "status": "continue_training_loop", + "phase": "calibration_rejected", + "threshold_selection_source": "calibration_only", + "selected_threshold": chosen["threshold"], + "calibration": chosen, + "test": None, + "background": None, + "failures": failures, + }, + ) + else: + for role, summary in ( + ("test", args.test_summary), + ("background", args.background_summary), + ): + report = iteration_dir / f"{role}.json" + reports[role] = report + run( + [ + sys.executable, + str(scripts_dir / "evaluate_belgium_building_candidate.py"), + "--model", str(candidate), + "--summary", str(summary), + "--corpus-manifest", str(args.corpus_manifest), + "--output", str(report), + "--device", "cuda:0", + "--max-det", str(args.max_det), + "--imgsz", str(args.imgsz), + ], + iteration_dir / f"{role}.log", + ) + run( + [ + sys.executable, + str(scripts_dir / "assess_belgium_building_training_iteration.py"), + "--calibration", str(reports["calibration"]), + "--test", str(reports["test"]), + "--background", str(reports["background"]), + "--output", str(assessment), + ], + iteration_dir / "assessment.log", + allowed={0, 2}, + ) decision = json.loads(assessment.read_text(encoding="utf-8")) record = { "iteration": index,