From 18387448dec7a81c9c39c89f0c653d553db868c0 Mon Sep 17 00:00:00 2001 From: Jens Date: Wed, 29 Jul 2026 16:27:30 +0200 Subject: [PATCH] Freeze aerial warmup across training loop --- backend/tests/test_belgium_training_loop.py | 5 +++++ docs/CODEX_EXECUTION_LOG.md | 10 ++++++++++ docs/TODO.md | 1 + scripts/run_belgium_building_training_loop.py | 20 +++++++++++++++++++ scripts/tower-v37-completion-command.json | 10 ++++++++++ 5 files changed, 46 insertions(+) diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py index f02123f1..ca8eb260 100644 --- a/backend/tests/test_belgium_training_loop.py +++ b/backend/tests/test_belgium_training_loop.py @@ -63,6 +63,11 @@ def test_training_command_supports_conservative_aerial_finetuning(tmp_path: Path assert "degrees=0.0" in command assert "flipud=0.0" in command assert "fliplr=0.5" in command + assert "warmup_epochs=1.0" in command + assert "warmup_bias_lr=0.01" in command + assert "hsv_h=0.01" in command + assert "hsv_s=0.2" in command + assert "hsv_v=0.15" in command assert f"data={tmp_path / 'dataset.yaml'}" in command diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 0b88de94..16411365 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11817,6 +11817,16 @@ Deployment evidence: 2 process exposing the old hardcoded value `35` was terminated before any result row existed; iteration 2 restarted from the identical checkpoint and sampling with the frozen `patience=18` contract on CUDA. +- The first iteration-2 result row then exposed further generic-default drift: + bias LR reached `0.066775` because the direct v37 run's warmup bias `0.01` + and aerial HSV settings were not represented in the orchestrator. Both exact + processes were stopped after one row, and the invalid run was retained as + `iteration-002-invalid-warmup` with a reason marker. +- Added explicit warmup epochs, warmup bias LR and HSV hue/saturation/value to + the orchestrator and versioned completion command. Iteration 2 restarted + cleanly from the same candidate and checksummed sampling with + `warmup_epochs=1`, `warmup_bias_lr=0.01`, HSV `0.01/0.2/0.15`, patience 18 + and the existing aerial rotation/flip contract. - Added those aerial augmentation parameters to the orchestrator CLI and training command, preventing later failure-driven checkpoints from silently reverting to generic orientation assumptions. diff --git a/docs/TODO.md b/docs/TODO.md index 35cdbb30..9f24a682 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -978,6 +978,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Bind v37 completion to a one-shot JSON command that enters calibration-first evaluation and up to 20 automatic failure-driven CUDA iterations with the frozen aerial augmentation contract. - [x] Complete v37 by early stopping at epoch 23, reject its best epoch-5 checkpoint calibration-first, and start closed-loop iteration 2 from checksummed failure-driven sampling. - [x] Make orchestrator patience explicit and retain the frozen value `18` in every subsequent CUDA iteration. +- [x] Preserve v37 warmup and aerial HSV augmentation in every loop iteration; archive and invalidate the pre-epoch-2 run that exposed generic Ultralytics warmup drift. - [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed. - [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes. diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index d763a1be..de8df250 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -105,6 +105,11 @@ def training_command( degrees: float = 0.0, flipud: float = 0.0, fliplr: float = 0.5, + warmup_epochs: float = 1.0, + warmup_bias_lr: float = 0.01, + hsv_h: float = 0.01, + hsv_s: float = 0.2, + hsv_v: float = 0.15, ) -> list[str]: command = [ yolo, @@ -127,6 +132,11 @@ def training_command( f"degrees={degrees}", f"flipud={flipud}", f"fliplr={fliplr}", + f"warmup_epochs={warmup_epochs}", + f"warmup_bias_lr={warmup_bias_lr}", + f"hsv_h={hsv_h}", + f"hsv_s={hsv_s}", + f"hsv_v={hsv_v}", f"seed={seed}", "deterministic=True", f"project={project}", @@ -194,6 +204,11 @@ def main() -> int: parser.add_argument("--degrees", type=float, default=0.0) parser.add_argument("--flipud", type=float, default=0.0) parser.add_argument("--fliplr", type=float, default=0.5) + parser.add_argument("--warmup-epochs", type=float, default=1.0) + parser.add_argument("--warmup-bias-lr", type=float, default=0.01) + parser.add_argument("--hsv-h", type=float, default=0.01) + parser.add_argument("--hsv-s", type=float, default=0.2) + parser.add_argument("--hsv-v", type=float, default=0.15) parser.add_argument("--seed", type=int, default=20260731) parser.add_argument("--yolo", default="yolo") parser.add_argument("--min-aggregate-f1", type=float, default=0.55) @@ -262,6 +277,11 @@ def main() -> int: degrees=args.degrees, flipud=args.flipud, fliplr=args.fliplr, + warmup_epochs=args.warmup_epochs, + warmup_bias_lr=args.warmup_bias_lr, + hsv_h=args.hsv_h, + hsv_s=args.hsv_s, + hsv_v=args.hsv_v, ) if args.dry_run: print(json.dumps({"training_command": command, "evaluate_existing": evaluate_existing}, indent=2)) diff --git a/scripts/tower-v37-completion-command.json b/scripts/tower-v37-completion-command.json index fb024f4c..1801084c 100644 --- a/scripts/tower-v37-completion-command.json +++ b/scripts/tower-v37-completion-command.json @@ -53,6 +53,16 @@ "0.5", "--fliplr", "0.5", + "--warmup-epochs", + "1", + "--warmup-bias-lr", + "0.01", + "--hsv-h", + "0.01", + "--hsv-s", + "0.2", + "--hsv-v", + "0.15", "--seed", "20260807", "--yolo",