diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py index 18888342..9f5b43d8 100644 --- a/backend/tests/test_belgium_training_loop.py +++ b/backend/tests/test_belgium_training_loop.py @@ -266,3 +266,34 @@ def test_threshold_selection_uses_worst_region_then_aggregate() -> None: ] } assert MODULE.select_calibration_threshold(report)["threshold"] == 0.2 + + +def test_rejected_candidate_score_prioritizes_weakest_release_gate() -> None: + gates = { + "min_aggregate_f1": 0.55, + "min_region_f1": 0.45, + "min_region_precision": 0.5, + "min_region_recall": 0.4, + } + incumbent = { + "gates": gates, + "calibration": { + "aggregate": {"f1": 0.58}, + "regions": { + "flanders": {"f1": 0.34, "precision": 0.38, "recall": 0.31}, + "wallonia": {"f1": 0.60, "precision": 0.50, "recall": 0.75}, + }, + }, + } + regressed = { + "gates": gates, + "calibration": { + "aggregate": {"f1": 0.60}, + "regions": { + "flanders": {"f1": 0.31, "precision": 0.45, "recall": 0.24}, + "wallonia": {"f1": 0.62, "precision": 0.52, "recall": 0.77}, + }, + }, + } + + assert MODULE.rejected_candidate_score(incumbent) > MODULE.rejected_candidate_score(regressed) diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index e2dcf1eb..5d4619ff 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -92,6 +92,25 @@ def calibration_failures( return failures +def rejected_candidate_score(assessment: dict[str, Any]) -> tuple[float, ...]: + """Rank rejected candidates by the weakest normalized release gate first.""" + calibration = assessment["calibration"] + gates = assessment["gates"] + normalized: list[float] = [ + calibration["aggregate"]["f1"] / gates["min_aggregate_f1"] + ] + for metrics in calibration["regions"].values(): + normalized.extend( + ( + metrics["f1"] / gates["min_region_f1"], + metrics["precision"] / gates["min_region_precision"], + metrics["recall"] / gates["min_region_recall"], + ) + ) + normalized.sort() + return tuple(normalized) + + def training_command( yolo: str, *, @@ -430,7 +449,15 @@ def main() -> int: "failures": decision["failures"], } state["iterations"].append(record) - state["next_model"] = str(candidate) + score = rejected_candidate_score(decision) if decision["status"] != "training_complete" else () + incumbent_score = tuple(state.get("incumbent_rejected_score", ())) + if not incumbent_score or score > incumbent_score: + state["incumbent_rejected_model"] = str(candidate) + state["incumbent_rejected_score"] = list(score) + record["promoted_to_training_incumbent"] = True + else: + record["promoted_to_training_incumbent"] = False + state["next_model"] = state.get("incumbent_rejected_model", str(candidate)) if decision["status"] == "training_complete": state["status"] = "training_complete" state["completed_at"] = datetime.now(UTC).isoformat() @@ -457,7 +484,7 @@ def main() -> int: record["failure_driven_sampling_sha256"] = sha256(sampling_evidence) record["next_train_yaml"] = str(next_train_yaml) state["next_train_yaml"] = str(next_train_yaml) - model = candidate + model = Path(state["next_model"]) train_yaml = next_train_yaml write_json(state_path, state)