Gate completed checkpoints through training loop
This commit is contained in:
@@ -76,6 +76,31 @@ def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_pat
|
|||||||
assert command[command.index("--output-dir") + 1].endswith("failure-driven-training")
|
assert command[command.index("--output-dir") + 1].endswith("failure-driven-training")
|
||||||
|
|
||||||
|
|
||||||
|
def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -> None:
|
||||||
|
audit = tmp_path / "audit.json"
|
||||||
|
audit.write_text(json.dumps({
|
||||||
|
"status": "needs_human_review", "failures": [],
|
||||||
|
"manifest_immutable": True, "spatial_leakage_status": "ok",
|
||||||
|
}))
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable, str(SCRIPT),
|
||||||
|
"--initial-model", str(tmp_path / "candidate.pt"),
|
||||||
|
"--train-yaml", str(tmp_path / "dataset.yaml"),
|
||||||
|
"--train-summary", str(tmp_path / "train-summary.json"),
|
||||||
|
"--dataset-audit", str(audit),
|
||||||
|
"--calibration-summary", str(tmp_path / "cal.json"),
|
||||||
|
"--test-summary", str(tmp_path / "test.json"),
|
||||||
|
"--background-summary", str(tmp_path / "background.json"),
|
||||||
|
"--corpus-manifest", str(tmp_path / "manifest.json"),
|
||||||
|
"--output-dir", str(tmp_path / "output"),
|
||||||
|
"--evaluate-initial-model", "--dry-run",
|
||||||
|
], capture_output=True, text=True, check=False,
|
||||||
|
)
|
||||||
|
assert result.returncode == 0
|
||||||
|
assert json.loads(result.stdout) == {"training_command": None, "evaluate_existing": True}
|
||||||
|
|
||||||
|
|
||||||
def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
||||||
audit = tmp_path / "audit.json"
|
audit = tmp_path / "audit.json"
|
||||||
audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4}))
|
audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4}))
|
||||||
|
|||||||
@@ -96,6 +96,11 @@ The checkpointed orchestrator invokes this builder after every rejected
|
|||||||
iteration, stores its checksum in `training-loop-state.json`, and uses the
|
iteration, stores its checksum in `training-loop-state.json`, and uses the
|
||||||
resulting dataset YAML for the next checkpoint. A restart resumes both the
|
resulting dataset YAML for the next checkpoint. A restart resumes both the
|
||||||
candidate weights and that exact failure-driven training input.
|
candidate weights and that exact failure-driven training input.
|
||||||
|
An already completed out-of-band checkpoint enters the same contract with
|
||||||
|
`--evaluate-initial-model`: the first iteration skips fitting, copies and
|
||||||
|
hashes the checkpoint, and begins at calibration. A rejection then follows
|
||||||
|
the identical failure-driven CUDA path and cannot open protected test evidence
|
||||||
|
early.
|
||||||
The orchestrator refuses to start unless every automated frozen-dataset gate
|
The orchestrator refuses to start unless every automated frozen-dataset gate
|
||||||
passes and the corpus contains zero blank/low-variance positive tiles. The
|
passes and the corpus contains zero blank/low-variance positive tiles. The
|
||||||
audit status may remain `needs_human_review` while training and objective
|
audit status may remain `needs_human_review` while training and objective
|
||||||
|
|||||||
@@ -11777,6 +11777,11 @@ Deployment evidence:
|
|||||||
leak-free failure-driven sampler automatically, records its evidence
|
leak-free failure-driven sampler automatically, records its evidence
|
||||||
checksum and next dataset YAML in `training-loop-state.json`, and resumes
|
checksum and next dataset YAML in `training-loop-state.json`, and resumes
|
||||||
both the candidate weights and exact sampling input after interruption.
|
both the candidate weights and exact sampling input after interruption.
|
||||||
|
- Added `--evaluate-initial-model` for completed checkpoints such as v37. It
|
||||||
|
skips redundant fitting only for the first iteration, copies and hashes the
|
||||||
|
supplied weights, runs calibration first, and rejoins the same automatic
|
||||||
|
sampling/training path after rejection. Protected evidence remains closed
|
||||||
|
until calibration passes.
|
||||||
- Confirmed v37 epoch 1 completed on CUDA with validation precision `0.601`,
|
- Confirmed v37 epoch 1 completed on CUDA with validation precision `0.601`,
|
||||||
recall `0.455`, mAP50 `0.474` and mAP50-95 `0.205`; the run remains inactive
|
recall `0.455`, mAP50 `0.474` and mAP50-95 `0.205`; the run remains inactive
|
||||||
and these internal-validation metrics are not release evidence.
|
and these internal-validation metrics are not release evidence.
|
||||||
@@ -11786,7 +11791,7 @@ Verified in this pass:
|
|||||||
- `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_belgium_training_iteration_assessment.py backend/tests/test_belgium_training_portfolio.py`
|
- `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_belgium_training_iteration_assessment.py backend/tests/test_belgium_training_portfolio.py`
|
||||||
(`12 passed`).
|
(`12 passed`).
|
||||||
- `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_failure_driven_yolo_sampling.py backend/tests/test_belgium_training_iteration_assessment.py`
|
- `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_failure_driven_yolo_sampling.py backend/tests/test_belgium_training_iteration_assessment.py`
|
||||||
(`15 passed`).
|
(`16 passed` after adding the completed-checkpoint entry contract).
|
||||||
|
|
||||||
Open:
|
Open:
|
||||||
|
|
||||||
|
|||||||
@@ -973,6 +973,7 @@ This file now starts with the current implementation status. Older preparation/b
|
|||||||
- [x] Exclude GRB/PICC features created after the corresponding dated imagery period while retaining auditable rejection evidence.
|
- [x] Exclude GRB/PICC features created after the corresponding dated imagery period while retaining auditable rejection evidence.
|
||||||
- [x] Allow the objective CUDA loop to consume an automatically clean `needs_human_review` corpus while keeping final human sign-off as a separate, mandatory promotion gate.
|
- [x] Allow the objective CUDA loop to consume an automatically clean `needs_human_review` corpus while keeping final human sign-off as a separate, mandatory promotion gate.
|
||||||
- [x] Persist checksummed train-only failure-driven sampling after every rejected loop iteration and resume the next checkpoint from that exact dataset YAML.
|
- [x] Persist checksummed train-only failure-driven sampling after every rejected loop iteration and resume the next checkpoint from that exact dataset YAML.
|
||||||
|
- [x] Add a guarded calibration-first entry point for completed checkpoints so v37 and future externally interrupted runs can rejoin the automated loop without redundant retraining.
|
||||||
- [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed.
|
- [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed.
|
||||||
- [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes.
|
- [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes.
|
||||||
|
|
||||||
|
|||||||
@@ -191,6 +191,11 @@ def main() -> int:
|
|||||||
parser.add_argument("--min-region-recall", type=float, default=0.4)
|
parser.add_argument("--min-region-recall", type=float, default=0.4)
|
||||||
parser.add_argument("--max-pure-empty-fp", type=int, default=0)
|
parser.add_argument("--max-pure-empty-fp", type=int, default=0)
|
||||||
parser.add_argument("--dry-run", action="store_true")
|
parser.add_argument("--dry-run", action="store_true")
|
||||||
|
parser.add_argument(
|
||||||
|
"--evaluate-initial-model",
|
||||||
|
action="store_true",
|
||||||
|
help="Gate an already trained initial checkpoint before starting the next training iteration.",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
if args.iterations < 1:
|
if args.iterations < 1:
|
||||||
raise SystemExit("--iterations must be positive")
|
raise SystemExit("--iterations must be positive")
|
||||||
@@ -223,7 +228,8 @@ def main() -> int:
|
|||||||
name = f"iteration-{index:03d}"
|
name = f"iteration-{index:03d}"
|
||||||
iteration_dir = args.output_dir / name
|
iteration_dir = args.output_dir / name
|
||||||
train_run = args.output_dir / "runs" / name
|
train_run = args.output_dir / "runs" / name
|
||||||
command = training_command(
|
evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"]
|
||||||
|
command = None if evaluate_existing else training_command(
|
||||||
args.yolo,
|
args.yolo,
|
||||||
model=model,
|
model=model,
|
||||||
data=train_yaml,
|
data=train_yaml,
|
||||||
@@ -242,12 +248,18 @@ def main() -> int:
|
|||||||
translate=args.translate,
|
translate=args.translate,
|
||||||
)
|
)
|
||||||
if args.dry_run:
|
if args.dry_run:
|
||||||
print(json.dumps({"training_command": command}, indent=2))
|
print(json.dumps({"training_command": command, "evaluate_existing": evaluate_existing}, indent=2))
|
||||||
return 0
|
return 0
|
||||||
run(command, iteration_dir / "training.log")
|
if evaluate_existing:
|
||||||
best = train_run / "weights" / "best.pt"
|
best = model
|
||||||
if not best.is_file():
|
if not best.is_file():
|
||||||
raise RuntimeError(f"Training produced no best checkpoint: {best}")
|
raise RuntimeError(f"Initial checkpoint does not exist: {best}")
|
||||||
|
else:
|
||||||
|
assert command is not None
|
||||||
|
run(command, iteration_dir / "training.log")
|
||||||
|
best = train_run / "weights" / "best.pt"
|
||||||
|
if not best.is_file():
|
||||||
|
raise RuntimeError(f"Training produced no best checkpoint: {best}")
|
||||||
candidate = iteration_dir / "candidate.pt"
|
candidate = iteration_dir / "candidate.pt"
|
||||||
shutil.copy2(best, candidate)
|
shutil.copy2(best, candidate)
|
||||||
|
|
||||||
@@ -347,6 +359,7 @@ def main() -> int:
|
|||||||
"iteration": index,
|
"iteration": index,
|
||||||
"candidate": str(candidate),
|
"candidate": str(candidate),
|
||||||
"candidate_sha256": sha256(candidate),
|
"candidate_sha256": sha256(candidate),
|
||||||
|
"training_skipped_for_existing_checkpoint": evaluate_existing,
|
||||||
"assessment": str(assessment),
|
"assessment": str(assessment),
|
||||||
"status": decision["status"],
|
"status": decision["status"],
|
||||||
"failures": decision["failures"],
|
"failures": decision["failures"],
|
||||||
|
|||||||
Reference in New Issue
Block a user