Make building training loop calibration-gated
This commit is contained in:
Jens
@@ -25,28 +25,38 @@ def main() -> int:
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--positive-repeat", type=int, default=2)
|
||||
parser.add_argument("--negative-repeat", type=int, default=1)
|
||||
parser.add_argument(
|
||||
"--other-region-repeat",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Include each train tile outside the expert region this many times.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if args.positive_repeat < 1 or args.negative_repeat < 1:
|
||||
raise SystemExit("repeat factors must be positive")
|
||||
if args.positive_repeat < 1 or args.negative_repeat < 1 or args.other_region_repeat < 0:
|
||||
raise SystemExit("regional repeats must be positive and other-region repeat non-negative")
|
||||
|
||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
||||
samples = {item["sample_slug"]: item for item in manifest["samples"]}
|
||||
paths: list[str] = []
|
||||
selected_samples: set[str] = set()
|
||||
positive_tiles = negative_tiles = 0
|
||||
positive_tiles = negative_tiles = other_region_tiles = 0
|
||||
for tile in summary["tiles"]:
|
||||
sample = samples[tile["sample_slug"]]
|
||||
if sample["split"] != "train" or tile["split"] != "train":
|
||||
continue
|
||||
if sample["region"] != args.region:
|
||||
continue
|
||||
positive = int(tile.get("label_count") or 0) > 0
|
||||
repeat = args.positive_repeat if positive else args.negative_repeat
|
||||
if sample["region"] == args.region:
|
||||
repeat = args.positive_repeat if positive else args.negative_repeat
|
||||
positive_tiles += int(positive)
|
||||
negative_tiles += int(not positive)
|
||||
else:
|
||||
repeat = args.other_region_repeat
|
||||
other_region_tiles += int(repeat > 0)
|
||||
if repeat == 0:
|
||||
continue
|
||||
paths.extend([str(Path(tile["image_path"]).resolve())] * repeat)
|
||||
selected_samples.add(tile["sample_slug"])
|
||||
positive_tiles += int(positive)
|
||||
negative_tiles += int(not positive)
|
||||
if not paths or not positive_tiles:
|
||||
raise SystemExit(f"no positive train tiles found for region {args.region!r}")
|
||||
|
||||
@@ -69,8 +79,10 @@ def main() -> int:
|
||||
"corpus_manifest_sha256": sha256(args.corpus_manifest),
|
||||
"positive_repeat": args.positive_repeat,
|
||||
"negative_repeat": args.negative_repeat,
|
||||
"other_region_repeat": args.other_region_repeat,
|
||||
"source_positive_tile_count": positive_tiles,
|
||||
"source_negative_tile_count": negative_tiles,
|
||||
"source_other_region_tile_count": other_region_tiles,
|
||||
"sampled_train_entry_count": len(paths),
|
||||
"selected_train_samples": sorted(selected_samples),
|
||||
"protected_samples_in_training": [],
|
||||
|
||||
@@ -29,6 +29,45 @@ def write_json(path: Path, value: dict[str, Any]) -> None:
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Choose a threshold without consulting test or background evidence."""
|
||||
eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
|
||||
if not eligible:
|
||||
eligible = report["sweeps"]
|
||||
return max(
|
||||
eligible,
|
||||
key=lambda item: (
|
||||
min(region["f1"] for region in item["regions"].values()),
|
||||
item["aggregate"]["f1"],
|
||||
-item["pure_empty_false_positives"],
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def calibration_failures(
|
||||
chosen: dict[str, Any],
|
||||
*,
|
||||
min_aggregate_f1: float,
|
||||
min_region_f1: float,
|
||||
min_region_precision: float,
|
||||
min_region_recall: float,
|
||||
max_pure_empty_fp: int,
|
||||
) -> list[str]:
|
||||
failures: list[str] = []
|
||||
if chosen["aggregate"]["f1"] < min_aggregate_f1:
|
||||
failures.append("calibration_aggregate_f1_below_gate")
|
||||
for region, values in chosen["regions"].items():
|
||||
if values["f1"] < min_region_f1:
|
||||
failures.append(f"calibration_{region}_f1_below_gate")
|
||||
if values["precision"] < min_region_precision:
|
||||
failures.append(f"calibration_{region}_precision_below_gate")
|
||||
if values["recall"] < min_region_recall:
|
||||
failures.append(f"calibration_{region}_recall_below_gate")
|
||||
if chosen["pure_empty_false_positives"] > max_pure_empty_fp:
|
||||
failures.append("calibration_pure_empty_false_positive_gate_failed")
|
||||
return failures
|
||||
|
||||
|
||||
def training_command(
|
||||
yolo: str,
|
||||
*,
|
||||
@@ -112,6 +151,11 @@ def main() -> int:
|
||||
parser.add_argument("--translate", type=float, default=0.1)
|
||||
parser.add_argument("--seed", type=int, default=20260731)
|
||||
parser.add_argument("--yolo", default="yolo")
|
||||
parser.add_argument("--min-aggregate-f1", type=float, default=0.55)
|
||||
parser.add_argument("--min-region-f1", type=float, default=0.45)
|
||||
parser.add_argument("--min-region-precision", type=float, default=0.5)
|
||||
parser.add_argument("--min-region-recall", type=float, default=0.4)
|
||||
parser.add_argument("--max-pure-empty-fp", type=int, default=0)
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.iterations < 1:
|
||||
@@ -174,11 +218,7 @@ def main() -> int:
|
||||
shutil.copy2(best, candidate)
|
||||
|
||||
reports: dict[str, Path] = {}
|
||||
for role, summary in (
|
||||
("calibration", args.calibration_summary),
|
||||
("test", args.test_summary),
|
||||
("background", args.background_summary),
|
||||
):
|
||||
for role, summary in (("calibration", args.calibration_summary),):
|
||||
report = iteration_dir / f"{role}.json"
|
||||
reports[role] = report
|
||||
run(
|
||||
@@ -203,22 +243,64 @@ def main() -> int:
|
||||
iteration_dir / f"{role}.log",
|
||||
)
|
||||
assessment = iteration_dir / "assessment.json"
|
||||
run(
|
||||
[
|
||||
sys.executable,
|
||||
str(scripts_dir / "assess_belgium_building_training_iteration.py"),
|
||||
"--calibration",
|
||||
str(reports["calibration"]),
|
||||
"--test",
|
||||
str(reports["test"]),
|
||||
"--background",
|
||||
str(reports["background"]),
|
||||
"--output",
|
||||
str(assessment),
|
||||
],
|
||||
iteration_dir / "assessment.log",
|
||||
allowed={0, 2},
|
||||
calibration = json.loads(reports["calibration"].read_text(encoding="utf-8"))
|
||||
chosen = select_calibration_threshold(calibration)
|
||||
failures = calibration_failures(
|
||||
chosen,
|
||||
min_aggregate_f1=args.min_aggregate_f1,
|
||||
min_region_f1=args.min_region_f1,
|
||||
min_region_precision=args.min_region_precision,
|
||||
min_region_recall=args.min_region_recall,
|
||||
max_pure_empty_fp=args.max_pure_empty_fp,
|
||||
)
|
||||
if failures:
|
||||
write_json(
|
||||
assessment,
|
||||
{
|
||||
"schema_version": 1,
|
||||
"status": "continue_training_loop",
|
||||
"phase": "calibration_rejected",
|
||||
"threshold_selection_source": "calibration_only",
|
||||
"selected_threshold": chosen["threshold"],
|
||||
"calibration": chosen,
|
||||
"test": None,
|
||||
"background": None,
|
||||
"failures": failures,
|
||||
},
|
||||
)
|
||||
else:
|
||||
for role, summary in (
|
||||
("test", args.test_summary),
|
||||
("background", args.background_summary),
|
||||
):
|
||||
report = iteration_dir / f"{role}.json"
|
||||
reports[role] = report
|
||||
run(
|
||||
[
|
||||
sys.executable,
|
||||
str(scripts_dir / "evaluate_belgium_building_candidate.py"),
|
||||
"--model", str(candidate),
|
||||
"--summary", str(summary),
|
||||
"--corpus-manifest", str(args.corpus_manifest),
|
||||
"--output", str(report),
|
||||
"--device", "cuda:0",
|
||||
"--max-det", str(args.max_det),
|
||||
"--imgsz", str(args.imgsz),
|
||||
],
|
||||
iteration_dir / f"{role}.log",
|
||||
)
|
||||
run(
|
||||
[
|
||||
sys.executable,
|
||||
str(scripts_dir / "assess_belgium_building_training_iteration.py"),
|
||||
"--calibration", str(reports["calibration"]),
|
||||
"--test", str(reports["test"]),
|
||||
"--background", str(reports["background"]),
|
||||
"--output", str(assessment),
|
||||
],
|
||||
iteration_dir / "assessment.log",
|
||||
allowed={0, 2},
|
||||
)
|
||||
decision = json.loads(assessment.read_text(encoding="utf-8"))
|
||||
record = {
|
||||
"iteration": index,
|
||||
|
||||
Reference in New Issue
Block a user