Rotate release holdouts after failed fixed test

scripts/assess_belgium_building_training_iteration.py

@@ -34,6 +34,24 @@ def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
     )
 
 
+INFERENCE_CONFIG_FIELDS = (
+    "model", "match_iou", "test_time_augmentation", "inference_imgsz",
+    "max_detections_per_tile", "nms_iou", "containment_nms", "box_scale",
+    "box_offset_x", "box_offset_y", "additional_model", "ensemble_mode",
+    "ensemble_match_iou", "proposal_classifier", "proposal_classifier_threshold",
+    "proposal_crop_scale", "proposal_classifier_batch",
+)
+
+
+def assert_same_inference_config(calibration: dict[str, Any], report: dict[str, Any], role: str) -> None:
+    differences = [
+        field for field in INFERENCE_CONFIG_FIELDS
+        if calibration.get(field) != report.get(field)
+    ]
+    if differences:
+        raise ValueError(f"{role} inference configuration differs from calibration: {differences}")
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--calibration", type=Path, required=True)
@@ -45,13 +63,25 @@ def main() -> int:
     parser.add_argument("--min-region-precision", type=float, default=0.5)
     parser.add_argument("--min-region-recall", type=float, default=0.4)
     parser.add_argument("--max-pure-empty-fp", type=int, default=0)
+    parser.add_argument(
+        "--selected-threshold", type=float,
+        help="Previously frozen calibration threshold; omit to select by worst-region F1.",
+    )
     args = parser.parse_args()
 
     calibration = load(args.calibration)
-    chosen = select_calibration_threshold(calibration)
+    chosen = (
+        find_threshold(calibration, args.selected_threshold)
+        if args.selected_threshold is not None
+        else select_calibration_threshold(calibration)
+    )
     threshold = float(chosen["threshold"])
-    test = find_threshold(load(args.test), threshold)
-    background = find_threshold(load(args.background), threshold)
+    test_report = load(args.test)
+    background_report = load(args.background)
+    assert_same_inference_config(calibration, test_report, "test")
+    assert_same_inference_config(calibration, background_report, "background")
+    test = find_threshold(test_report, threshold)
+    background = find_threshold(background_report, threshold)
     failures: list[str] = []
     if test["aggregate"]["f1"] < args.min_aggregate_f1:
         failures.append("test_aggregate_f1_below_gate")