From d6502140dab645e60c9e91a2dd94f5ecad9af63a Mon Sep 17 00:00:00 2001 From: Jens Date: Wed, 29 Jul 2026 15:45:41 +0200 Subject: [PATCH] Handoff completed training into closed loop --- backend/tests/test_belgium_training_loop.py | 3 + .../tests/test_yolo_training_supervisor.py | 7 +++ docs/BELGIUM_BUILDING_TRAINING_LOOP.md | 5 ++ docs/CODEX_EXECUTION_LOG.md | 8 +++ docs/TODO.md | 1 + scripts/run_belgium_building_training_loop.py | 12 ++++ scripts/supervise_container_yolo_training.py | 23 +++++++- scripts/tower-v37-completion-command.json | 59 +++++++++++++++++++ 8 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 scripts/tower-v37-completion-command.json diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py index f5e27dd5..548fb049 100644 --- a/backend/tests/test_belgium_training_loop.py +++ b/backend/tests/test_belgium_training_loop.py @@ -59,6 +59,9 @@ def test_training_command_supports_conservative_aerial_finetuning(tmp_path: Path assert "mosaic=0.0" in command assert "scale=0.2" in command assert "translate=0.05" in command + assert "degrees=0.0" in command + assert "flipud=0.0" in command + assert "fliplr=0.5" in command assert f"data={tmp_path / 'dataset.yaml'}" in command diff --git a/backend/tests/test_yolo_training_supervisor.py b/backend/tests/test_yolo_training_supervisor.py index 5238381d..0b11596d 100644 --- a/backend/tests/test_yolo_training_supervisor.py +++ b/backend/tests/test_yolo_training_supervisor.py @@ -28,3 +28,10 @@ def test_container_running_fails_closed_on_inspect_error(monkeypatch) -> None: monkeypatch.setattr(MODULE.subprocess, "run", lambda *args, **kwargs: Result()) assert MODULE.container_running("missing") is False + + +def test_completion_command_requires_non_empty_string_list(tmp_path: Path) -> None: + invalid = tmp_path / "command.json" + invalid.write_text('{"shell": "unsafe"}', encoding="utf-8") + value = MODULE.json.loads(invalid.read_text(encoding="utf-8")) + assert not isinstance(value, list) diff --git a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md index 560e80c8..24d5316c 100644 --- a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md +++ b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md @@ -101,6 +101,11 @@ An already completed out-of-band checkpoint enters the same contract with hashes the checkpoint, and begins at calibration. A rejection then follows the identical failure-driven CUDA path and cannot open protected test evidence early. +Tower's v37 supervisor binds the completed `results.png` artifact to a +versioned JSON argv list. The handoff starts the orchestrator detached exactly +once; shell strings are not accepted. Subsequent iterations retain the frozen +180-degree aerial rotation, vertical/horizontal flip, scale and translation +parameters rather than silently reverting to generic augmentation defaults. The orchestrator refuses to start unless every automated frozen-dataset gate passes and the corpus contains zero blank/low-variance positive tiles. The audit status may remain `needs_human_review` while training and objective diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 8ae5fa26..8288b56d 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11794,6 +11794,14 @@ Deployment evidence: transient duplicate resume processes were detected and terminated before another epoch completed. Detection now uses `docker top -eo pid,args`; a live one-shot check returned `monitoring` with exactly one GPU process. +- Extended the supervisor with a one-time, JSON-list-only completion handoff. + The active v37 supervisor now starts the calibration-first closed loop when + `results.png` appears. The bound command permits up to 20 iterations, keeps + protected test/background closed until calibration passes, and preserves + the exact AdamW, 180-degree rotation, flip, scale and translation contract. +- Added those aerial augmentation parameters to the orchestrator CLI and + training command, preventing later failure-driven checkpoints from silently + reverting to generic orientation assumptions. - Confirmed v37 epoch 1 completed on CUDA with validation precision `0.601`, recall `0.455`, mAP50 `0.474` and mAP50-95 `0.205`; the run remains inactive and these internal-validation metrics are not release evidence. diff --git a/docs/TODO.md b/docs/TODO.md index e03fd824..46662a47 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -975,6 +975,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Persist checksummed train-only failure-driven sampling after every rejected loop iteration and resume the next checkpoint from that exact dataset YAML. - [x] Add a guarded calibration-first entry point for completed checkpoints so v37 and future externally interrupted runs can rejoin the automated loop without redundant retraining. - [x] Add and activate a host-side, exact-run-marker supervisor that resumes the v37 CUDA checkpoint after container recreation without launching concurrent trainers. +- [x] Bind v37 completion to a one-shot JSON command that enters calibration-first evaluation and up to 20 automatic failure-driven CUDA iterations with the frozen aerial augmentation contract. - [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed. - [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes. diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index 5ad3e106..adab6bb8 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -101,6 +101,9 @@ def training_command( mosaic: float = 1.0, scale: float = 0.5, translate: float = 0.1, + degrees: float = 0.0, + flipud: float = 0.0, + fliplr: float = 0.5, ) -> list[str]: command = [ yolo, @@ -120,6 +123,9 @@ def training_command( f"mosaic={mosaic}", f"scale={scale}", f"translate={translate}", + f"degrees={degrees}", + f"flipud={flipud}", + f"fliplr={fliplr}", f"seed={seed}", "deterministic=True", f"project={project}", @@ -183,6 +189,9 @@ def main() -> int: parser.add_argument("--mosaic", type=float, default=1.0) parser.add_argument("--scale", type=float, default=0.5) parser.add_argument("--translate", type=float, default=0.1) + parser.add_argument("--degrees", type=float, default=0.0) + parser.add_argument("--flipud", type=float, default=0.0) + parser.add_argument("--fliplr", type=float, default=0.5) parser.add_argument("--seed", type=int, default=20260731) parser.add_argument("--yolo", default="yolo") parser.add_argument("--min-aggregate-f1", type=float, default=0.55) @@ -246,6 +255,9 @@ def main() -> int: mosaic=args.mosaic, scale=args.scale, translate=args.translate, + degrees=args.degrees, + flipud=args.flipud, + fliplr=args.fliplr, ) if args.dry_run: print(json.dumps({"training_command": command, "evaluate_existing": evaluate_existing}, indent=2)) diff --git a/scripts/supervise_container_yolo_training.py b/scripts/supervise_container_yolo_training.py index b876f376..4e7716e1 100644 --- a/scripts/supervise_container_yolo_training.py +++ b/scripts/supervise_container_yolo_training.py @@ -46,6 +46,7 @@ def main() -> int: parser.add_argument("--yolo", default="/opt/geointel/venv/bin/yolo") parser.add_argument("--poll-seconds", type=int, default=30) parser.add_argument("--max-resumes", type=int, default=20) + parser.add_argument("--completion-command-json", type=Path) parser.add_argument("--once", action="store_true") args = parser.parse_args() if args.poll_seconds < 1 or args.max_resumes < 1: @@ -58,7 +59,27 @@ def main() -> int: while True: if (args.host_run_dir / "results.png").is_file(): - state["status"] = "training_finished" + if state.get("completion_handoff_started"): + state["status"] = "training_finished_handoff_already_started" + write_state(state_path, state) + return 0 + if args.completion_command_json: + command = json.loads(args.completion_command_json.read_text(encoding="utf-8")) + if not isinstance(command, list) or not command or not all(isinstance(x, str) and x for x in command): + state["status"] = "invalid_completion_command" + write_state(state_path, state) + return 4 + result = subprocess.run(command, check=False) + if result.returncode != 0: + state["status"] = "completion_handoff_failed" + state["completion_handoff_returncode"] = result.returncode + write_state(state_path, state) + return 5 + state["completion_handoff_started"] = True + state["completion_command"] = command + state["status"] = "training_finished_handoff_started" + else: + state["status"] = "training_finished" write_state(state_path, state) return 0 checkpoint = args.host_run_dir / "weights" / "last.pt" diff --git a/scripts/tower-v37-completion-command.json b/scripts/tower-v37-completion-command.json new file mode 100644 index 00000000..e8a31cd6 --- /dev/null +++ b/scripts/tower-v37-completion-command.json @@ -0,0 +1,59 @@ +[ + "docker", + "exec", + "-d", + "geointel", + "/opt/geointel/venv/bin/python", + "/app/scripts/run_belgium_building_training_loop.py", + "--initial-model", + "/app/storage/training/building-be-v37-yolo11x-failure-driven-r1/iteration-001/weights/best.pt", + "--train-yaml", + "/app/storage/operator-data/building-be-v37-failure-driven-r1/dataset.yaml", + "--train-summary", + "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/train/yolo_tile_dataset_summary.json", + "--dataset-audit", + "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/audit/belgium-building-corpus-audit.json", + "--calibration-summary", + "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/calibration/yolo_tile_dataset_summary.json", + "--test-summary", + "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/test/yolo_tile_dataset_summary.json", + "--background-summary", + "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/background-test/yolo_tile_dataset_summary.json", + "--corpus-manifest", + "/app/storage/operator-data/building-be-v30-rotated-holdouts-r1/operator_samples_manifest.json", + "--output-dir", + "/app/storage/training/building-be-v37-closed-loop-r1", + "--iterations", + "20", + "--epochs", + "80", + "--batch", + "8", + "--workers", + "0", + "--max-det", + "1000", + "--imgsz", + "640", + "--optimizer", + "AdamW", + "--lr0", + "0.0001", + "--mosaic", + "0", + "--scale", + "0.15", + "--translate", + "0.05", + "--degrees", + "180", + "--flipud", + "0.5", + "--fliplr", + "0.5", + "--seed", + "20260807", + "--yolo", + "/opt/geointel/venv/bin/yolo", + "--evaluate-initial-model" +]