Make dataset audit mandatory for training loop
This commit is contained in:
@@ -78,6 +78,7 @@ def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--initial-model", type=Path, required=True)
|
||||
parser.add_argument("--train-yaml", type=Path, required=True)
|
||||
parser.add_argument("--dataset-audit", type=Path, required=True)
|
||||
parser.add_argument("--calibration-summary", type=Path, required=True)
|
||||
parser.add_argument("--test-summary", type=Path, required=True)
|
||||
parser.add_argument("--background-summary", type=Path, required=True)
|
||||
@@ -93,6 +94,11 @@ def main() -> int:
|
||||
args = parser.parse_args()
|
||||
if args.iterations < 1:
|
||||
raise SystemExit("--iterations must be positive")
|
||||
dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8"))
|
||||
if dataset_audit.get("status") != "ok":
|
||||
raise SystemExit(f"Dataset audit is not ok: {args.dataset_audit}")
|
||||
if int(dataset_audit.get("low_variance_positive_tile_count") or 0) != 0:
|
||||
raise SystemExit("Dataset audit contains blank/low-variance positive tiles")
|
||||
|
||||
state_path = args.output_dir / "training-loop-state.json"
|
||||
state: dict[str, Any] = {
|
||||
@@ -101,6 +107,7 @@ def main() -> int:
|
||||
"started_at": datetime.now(UTC).isoformat(),
|
||||
"initial_model": str(args.initial_model),
|
||||
"train_yaml": str(args.train_yaml),
|
||||
"dataset_audit": str(args.dataset_audit),
|
||||
"corpus_manifest": str(args.corpus_manifest),
|
||||
"iterations": [],
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user