From 1ce291d6523b2859c51a2f712e984605b2861b80 Mon Sep 17 00:00:00 2001 From: Jens Date: Wed, 29 Jul 2026 15:37:08 +0200 Subject: [PATCH] Gate completed checkpoints through training loop --- backend/tests/test_belgium_training_loop.py | 25 +++++++++++++++++++ docs/BELGIUM_BUILDING_TRAINING_LOOP.md | 5 ++++ docs/CODEX_EXECUTION_LOG.md | 7 +++++- docs/TODO.md | 1 + scripts/run_belgium_building_training_loop.py | 25 ++++++++++++++----- 5 files changed, 56 insertions(+), 7 deletions(-) diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py index 54578540..f5e27dd5 100644 --- a/backend/tests/test_belgium_training_loop.py +++ b/backend/tests/test_belgium_training_loop.py @@ -76,6 +76,31 @@ def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_pat assert command[command.index("--output-dir") + 1].endswith("failure-driven-training") +def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -> None: + audit = tmp_path / "audit.json" + audit.write_text(json.dumps({ + "status": "needs_human_review", "failures": [], + "manifest_immutable": True, "spatial_leakage_status": "ok", + })) + result = subprocess.run( + [ + sys.executable, str(SCRIPT), + "--initial-model", str(tmp_path / "candidate.pt"), + "--train-yaml", str(tmp_path / "dataset.yaml"), + "--train-summary", str(tmp_path / "train-summary.json"), + "--dataset-audit", str(audit), + "--calibration-summary", str(tmp_path / "cal.json"), + "--test-summary", str(tmp_path / "test.json"), + "--background-summary", str(tmp_path / "background.json"), + "--corpus-manifest", str(tmp_path / "manifest.json"), + "--output-dir", str(tmp_path / "output"), + "--evaluate-initial-model", "--dry-run", + ], capture_output=True, text=True, check=False, + ) + assert result.returncode == 0 + assert json.loads(result.stdout) == {"training_command": None, "evaluate_existing": True} + + def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: audit = tmp_path / "audit.json" audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4})) diff --git a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md index f999c828..560e80c8 100644 --- a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md +++ b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md @@ -96,6 +96,11 @@ The checkpointed orchestrator invokes this builder after every rejected iteration, stores its checksum in `training-loop-state.json`, and uses the resulting dataset YAML for the next checkpoint. A restart resumes both the candidate weights and that exact failure-driven training input. +An already completed out-of-band checkpoint enters the same contract with +`--evaluate-initial-model`: the first iteration skips fitting, copies and +hashes the checkpoint, and begins at calibration. A rejection then follows +the identical failure-driven CUDA path and cannot open protected test evidence +early. The orchestrator refuses to start unless every automated frozen-dataset gate passes and the corpus contains zero blank/low-variance positive tiles. The audit status may remain `needs_human_review` while training and objective diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 251a3a8e..5f549738 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11777,6 +11777,11 @@ Deployment evidence: leak-free failure-driven sampler automatically, records its evidence checksum and next dataset YAML in `training-loop-state.json`, and resumes both the candidate weights and exact sampling input after interruption. +- Added `--evaluate-initial-model` for completed checkpoints such as v37. It + skips redundant fitting only for the first iteration, copies and hashes the + supplied weights, runs calibration first, and rejoins the same automatic + sampling/training path after rejection. Protected evidence remains closed + until calibration passes. - Confirmed v37 epoch 1 completed on CUDA with validation precision `0.601`, recall `0.455`, mAP50 `0.474` and mAP50-95 `0.205`; the run remains inactive and these internal-validation metrics are not release evidence. @@ -11786,7 +11791,7 @@ Verified in this pass: - `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_belgium_training_iteration_assessment.py backend/tests/test_belgium_training_portfolio.py` (`12 passed`). - `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_failure_driven_yolo_sampling.py backend/tests/test_belgium_training_iteration_assessment.py` - (`15 passed`). + (`16 passed` after adding the completed-checkpoint entry contract). Open: diff --git a/docs/TODO.md b/docs/TODO.md index 543652af..bf40db20 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -973,6 +973,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Exclude GRB/PICC features created after the corresponding dated imagery period while retaining auditable rejection evidence. - [x] Allow the objective CUDA loop to consume an automatically clean `needs_human_review` corpus while keeping final human sign-off as a separate, mandatory promotion gate. - [x] Persist checksummed train-only failure-driven sampling after every rejected loop iteration and resume the next checkpoint from that exact dataset YAML. +- [x] Add a guarded calibration-first entry point for completed checkpoints so v37 and future externally interrupted runs can rejoin the automated loop without redundant retraining. - [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed. - [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes. diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index 22a85253..5ad3e106 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -191,6 +191,11 @@ def main() -> int: parser.add_argument("--min-region-recall", type=float, default=0.4) parser.add_argument("--max-pure-empty-fp", type=int, default=0) parser.add_argument("--dry-run", action="store_true") + parser.add_argument( + "--evaluate-initial-model", + action="store_true", + help="Gate an already trained initial checkpoint before starting the next training iteration.", + ) args = parser.parse_args() if args.iterations < 1: raise SystemExit("--iterations must be positive") @@ -223,7 +228,8 @@ def main() -> int: name = f"iteration-{index:03d}" iteration_dir = args.output_dir / name train_run = args.output_dir / "runs" / name - command = training_command( + evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"] + command = None if evaluate_existing else training_command( args.yolo, model=model, data=train_yaml, @@ -242,12 +248,18 @@ def main() -> int: translate=args.translate, ) if args.dry_run: - print(json.dumps({"training_command": command}, indent=2)) + print(json.dumps({"training_command": command, "evaluate_existing": evaluate_existing}, indent=2)) return 0 - run(command, iteration_dir / "training.log") - best = train_run / "weights" / "best.pt" - if not best.is_file(): - raise RuntimeError(f"Training produced no best checkpoint: {best}") + if evaluate_existing: + best = model + if not best.is_file(): + raise RuntimeError(f"Initial checkpoint does not exist: {best}") + else: + assert command is not None + run(command, iteration_dir / "training.log") + best = train_run / "weights" / "best.pt" + if not best.is_file(): + raise RuntimeError(f"Training produced no best checkpoint: {best}") candidate = iteration_dir / "candidate.pt" shutil.copy2(best, candidate) @@ -347,6 +359,7 @@ def main() -> int: "iteration": index, "candidate": str(candidate), "candidate_sha256": sha256(candidate), + "training_skipped_for_existing_checkpoint": evaluate_existing, "assessment": str(assessment), "status": decision["status"], "failures": decision["failures"],