Recover active training loop checkpoints

scripts/run_belgium_building_training_loop.py

@@ -166,6 +166,10 @@ def failure_sampling_command(
     ]
 
 
+def resumable_training_command(yolo: str, checkpoint: Path) -> list[str]:
+    return [yolo, "train", f"resume={checkpoint}", "device=0"]
+
+
 def run(command: list[str], log_path: Path | None = None, *, allowed: set[int] = {0}) -> int:
     if log_path:
         log_path.parent.mkdir(parents=True, exist_ok=True)
@@ -256,8 +260,12 @@ def main() -> int:
         iteration_dir.mkdir(parents=True, exist_ok=True)
         train_run = args.output_dir / "runs" / name
         evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"]
-        command = None if evaluate_existing else training_command(
-            args.yolo,
+        partial_checkpoint = train_run / "weights" / "last.pt"
+        resume_partial = not evaluate_existing and partial_checkpoint.is_file()
+        command = None if evaluate_existing else (
+            resumable_training_command(args.yolo, partial_checkpoint)
+            if resume_partial else training_command(
+                args.yolo,
             model=model,
             data=train_yaml,
             project=args.output_dir / "runs",
@@ -281,10 +289,14 @@ def main() -> int:
             warmup_bias_lr=args.warmup_bias_lr,
             hsv_h=args.hsv_h,
             hsv_s=args.hsv_s,
-            hsv_v=args.hsv_v,
-        )
+                hsv_v=args.hsv_v,
+            ))
         if args.dry_run:
-            print(json.dumps({"training_command": command, "evaluate_existing": evaluate_existing}, indent=2))
+            print(json.dumps({
+                "training_command": command,
+                "evaluate_existing": evaluate_existing,
+                "resume_partial": resume_partial,
+            }, indent=2))
             return 0
         if evaluate_existing:
             best = model
@@ -396,6 +408,7 @@ def main() -> int:
             "candidate": str(candidate),
             "candidate_sha256": sha256(candidate),
             "training_skipped_for_existing_checkpoint": evaluate_existing,
+            "training_resumed_from_partial_checkpoint": resume_partial,
             "assessment": str(assessment),
             "status": decision["status"],
             "failures": decision["failures"],