Handoff completed training into closed loop

scripts/run_belgium_building_training_loop.py

@@ -101,6 +101,9 @@ def training_command(
     mosaic: float = 1.0,
     scale: float = 0.5,
     translate: float = 0.1,
+    degrees: float = 0.0,
+    flipud: float = 0.0,
+    fliplr: float = 0.5,
 ) -> list[str]:
     command = [
         yolo,
@@ -120,6 +123,9 @@ def training_command(
         f"mosaic={mosaic}",
         f"scale={scale}",
         f"translate={translate}",
+        f"degrees={degrees}",
+        f"flipud={flipud}",
+        f"fliplr={fliplr}",
         f"seed={seed}",
         "deterministic=True",
         f"project={project}",
@@ -183,6 +189,9 @@ def main() -> int:
     parser.add_argument("--mosaic", type=float, default=1.0)
     parser.add_argument("--scale", type=float, default=0.5)
     parser.add_argument("--translate", type=float, default=0.1)
+    parser.add_argument("--degrees", type=float, default=0.0)
+    parser.add_argument("--flipud", type=float, default=0.0)
+    parser.add_argument("--fliplr", type=float, default=0.5)
     parser.add_argument("--seed", type=int, default=20260731)
     parser.add_argument("--yolo", default="yolo")
     parser.add_argument("--min-aggregate-f1", type=float, default=0.55)
@@ -246,6 +255,9 @@ def main() -> int:
             mosaic=args.mosaic,
             scale=args.scale,
             translate=args.translate,
+            degrees=args.degrees,
+            flipud=args.flipud,
+            fliplr=args.fliplr,
         )
         if args.dry_run:
             print(json.dumps({"training_command": command, "evaluate_existing": evaluate_existing}, indent=2))

scripts/supervise_container_yolo_training.py

@@ -46,6 +46,7 @@ def main() -> int:
     parser.add_argument("--yolo", default="/opt/geointel/venv/bin/yolo")
     parser.add_argument("--poll-seconds", type=int, default=30)
     parser.add_argument("--max-resumes", type=int, default=20)
+    parser.add_argument("--completion-command-json", type=Path)
     parser.add_argument("--once", action="store_true")
     args = parser.parse_args()
     if args.poll_seconds < 1 or args.max_resumes < 1:
@@ -58,7 +59,27 @@ def main() -> int:
 
     while True:
         if (args.host_run_dir / "results.png").is_file():
-            state["status"] = "training_finished"
+            if state.get("completion_handoff_started"):
+                state["status"] = "training_finished_handoff_already_started"
+                write_state(state_path, state)
+                return 0
+            if args.completion_command_json:
+                command = json.loads(args.completion_command_json.read_text(encoding="utf-8"))
+                if not isinstance(command, list) or not command or not all(isinstance(x, str) and x for x in command):
+                    state["status"] = "invalid_completion_command"
+                    write_state(state_path, state)
+                    return 4
+                result = subprocess.run(command, check=False)
+                if result.returncode != 0:
+                    state["status"] = "completion_handoff_failed"
+                    state["completion_handoff_returncode"] = result.returncode
+                    write_state(state_path, state)
+                    return 5
+                state["completion_handoff_started"] = True
+                state["completion_command"] = command
+                state["status"] = "training_finished_handoff_started"
+            else:
+                state["status"] = "training_finished"
             write_state(state_path, state)
             return 0
         checkpoint = args.host_run_dir / "weights" / "last.pt"

scripts/tower-v37-completion-command.json

@@ -0,0 +1,59 @@
+[
+  "docker",
+  "exec",
+  "-d",
+  "geointel",
+  "/opt/geointel/venv/bin/python",
+  "/app/scripts/run_belgium_building_training_loop.py",
+  "--initial-model",
+  "/app/storage/training/building-be-v37-yolo11x-failure-driven-r1/iteration-001/weights/best.pt",
+  "--train-yaml",
+  "/app/storage/operator-data/building-be-v37-failure-driven-r1/dataset.yaml",
+  "--train-summary",
+  "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/train/yolo_tile_dataset_summary.json",
+  "--dataset-audit",
+  "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/audit/belgium-building-corpus-audit.json",
+  "--calibration-summary",
+  "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/calibration/yolo_tile_dataset_summary.json",
+  "--test-summary",
+  "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/test/yolo_tile_dataset_summary.json",
+  "--background-summary",
+  "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/background-test/yolo_tile_dataset_summary.json",
+  "--corpus-manifest",
+  "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/operator_samples_manifest.json",
+  "--output-dir",
+  "/app/storage/training/building-be-v37-closed-loop-r1",
+  "--iterations",
+  "20",
+  "--epochs",
+  "80",
+  "--batch",
+  "8",
+  "--workers",
+  "0",
+  "--max-det",
+  "1000",
+  "--imgsz",
+  "640",
+  "--optimizer",
+  "AdamW",
+  "--lr0",
+  "0.0001",
+  "--mosaic",
+  "0",
+  "--scale",
+  "0.15",
+  "--translate",
+  "0.05",
+  "--degrees",
+  "180",
+  "--flipud",
+  "0.5",
+  "--fliplr",
+  "0.5",
+  "--seed",
+  "20260807",
+  "--yolo",
+  "/opt/geointel/venv/bin/yolo",
+  "--evaluate-initial-model"
+]