#!/usr/bin/env python3 """Select on calibration evidence and apply fixed, fail-closed release gates.""" from __future__ import annotations import argparse import json from pathlib import Path from typing import Any def load(path: Path) -> dict[str, Any]: return json.loads(path.read_text(encoding="utf-8")) def find_threshold(report: dict[str, Any], threshold: float) -> dict[str, Any]: for item in report["sweeps"]: if abs(float(item["threshold"]) - threshold) < 1e-9: return item raise ValueError(f"Threshold {threshold} is absent from {report.get('summary')}") def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]: eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0] if not eligible: eligible = report["sweeps"] return max( eligible, key=lambda item: ( min(region["f1"] for region in item["regions"].values()), item["aggregate"]["f1"], -item["pure_empty_false_positives"], ), ) def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--calibration", type=Path, required=True) parser.add_argument("--test", type=Path, required=True) parser.add_argument("--background", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--min-aggregate-f1", type=float, default=0.55) parser.add_argument("--min-region-f1", type=float, default=0.45) parser.add_argument("--min-region-precision", type=float, default=0.5) parser.add_argument("--min-region-recall", type=float, default=0.4) parser.add_argument("--max-pure-empty-fp", type=int, default=0) args = parser.parse_args() calibration = load(args.calibration) chosen = select_calibration_threshold(calibration) threshold = float(chosen["threshold"]) test = find_threshold(load(args.test), threshold) background = find_threshold(load(args.background), threshold) failures: list[str] = [] if test["aggregate"]["f1"] < args.min_aggregate_f1: failures.append("test_aggregate_f1_below_gate") for region, values in test["regions"].items(): if values["f1"] < args.min_region_f1: failures.append(f"test_{region}_f1_below_gate") if values["precision"] < args.min_region_precision: failures.append(f"test_{region}_precision_below_gate") if values["recall"] < args.min_region_recall: failures.append(f"test_{region}_recall_below_gate") if background["pure_empty_false_positives"] > args.max_pure_empty_fp: failures.append("pure_empty_false_positive_gate_failed") payload = { "schema_version": 1, "status": "training_complete" if not failures else "continue_training_loop", "threshold_selection_source": "calibration_only", "selected_threshold": threshold, "gates": { "min_aggregate_f1": args.min_aggregate_f1, "min_region_f1": args.min_region_f1, "min_region_precision": args.min_region_precision, "min_region_recall": args.min_region_recall, "max_pure_empty_false_positives": args.max_pure_empty_fp, }, "calibration": chosen, "test": test, "background": background, "failures": failures, } args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(payload, indent=2), encoding="utf-8") print(json.dumps(payload, indent=2)) return 0 if not failures else 2 if __name__ == "__main__": raise SystemExit(main())