From b393d9f65d5e7b5a94042977c70b94d397cc1ca7 Mon Sep 17 00:00:00 2001 From: Jens Date: Wed, 29 Jul 2026 16:23:28 +0200 Subject: [PATCH] Continue rejected v37 through frozen loop --- backend/tests/test_belgium_training_loop.py | 7 +++++++ docs/CODEX_EXECUTION_LOG.md | 13 +++++++++++++ docs/TODO.md | 2 ++ scripts/run_belgium_building_training_loop.py | 8 ++++++-- scripts/tower-v37-completion-command.json | 2 ++ 5 files changed, 30 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py index 548fb049..f02123f1 100644 --- a/backend/tests/test_belgium_training_loop.py +++ b/backend/tests/test_belgium_training_loop.py @@ -31,6 +31,7 @@ def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_p assert "deterministic=True" in command assert "seed=42" in command assert "epochs=160" in command + assert "patience=18" in command assert "max_det=1000" in command assert "imgsz=640" in command assert "optimizer=auto" in command @@ -104,6 +105,12 @@ def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) - assert json.loads(result.stdout) == {"training_command": None, "evaluate_existing": True} +def test_existing_checkpoint_iteration_directory_can_be_created_without_yolo(tmp_path: Path) -> None: + iteration_dir = tmp_path / "closed-loop" / "iteration-001" + iteration_dir.mkdir(parents=True, exist_ok=True) + assert iteration_dir.is_dir() + + def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: audit = tmp_path / "audit.json" audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4})) diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 10f291dc..0b88de94 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11804,6 +11804,19 @@ Deployment evidence: malformed JSON and filesystem errors produce an explicit fail-closed supervisor state before any subprocess is started. The updated supervisor was activated live with one remaining v37 GPU process. +- V37 early-stopped normally at epoch 23; epoch 5 remained the immutable best + checkpoint. The supervisor observed `results.png` and executed its one-shot + completion handoff. +- Fixed a completed-checkpoint entry defect exposed by that live handoff: the + iteration directory is now created before copying/hash-binding an existing + model. The repaired calibration ran without opening test/background. +- V37 improved calibration aggregate F1 to `0.568` and Flanders F1 to `0.307`, + but failed Flanders F1/precision/recall and Wallonia precision. It was + rejected and produced checksummed failure-driven sampling for iteration 2. +- Made training patience an explicit orchestrator input. A pre-epoch iteration + 2 process exposing the old hardcoded value `35` was terminated before any + result row existed; iteration 2 restarted from the identical checkpoint and + sampling with the frozen `patience=18` contract on CUDA. - Added those aerial augmentation parameters to the orchestrator CLI and training command, preventing later failure-driven checkpoints from silently reverting to generic orientation assumptions. diff --git a/docs/TODO.md b/docs/TODO.md index 46662a47..35cdbb30 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -976,6 +976,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Add a guarded calibration-first entry point for completed checkpoints so v37 and future externally interrupted runs can rejoin the automated loop without redundant retraining. - [x] Add and activate a host-side, exact-run-marker supervisor that resumes the v37 CUDA checkpoint after container recreation without launching concurrent trainers. - [x] Bind v37 completion to a one-shot JSON command that enters calibration-first evaluation and up to 20 automatic failure-driven CUDA iterations with the frozen aerial augmentation contract. +- [x] Complete v37 by early stopping at epoch 23, reject its best epoch-5 checkpoint calibration-first, and start closed-loop iteration 2 from checksummed failure-driven sampling. +- [x] Make orchestrator patience explicit and retain the frozen value `18` in every subsequent CUDA iteration. - [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed. - [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes. diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index adab6bb8..d763a1be 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -91,6 +91,7 @@ def training_command( project: Path, name: str, epochs: int, + patience: int = 18, seed: int, batch: int, workers: int, @@ -115,7 +116,7 @@ def training_command( f"batch={batch}", "device=0", f"workers={workers}", - "patience=35", + f"patience={patience}", "cache=disk", "close_mosaic=20", f"max_det={max_det}", @@ -180,6 +181,7 @@ def main() -> int: parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--iterations", type=int, default=1) parser.add_argument("--epochs", type=int, default=160) + parser.add_argument("--patience", type=int, default=18) parser.add_argument("--batch", type=int, default=2) parser.add_argument("--workers", type=int, default=4) parser.add_argument("--max-det", type=int, default=1000) @@ -236,6 +238,7 @@ def main() -> int: index = first_index + offset name = f"iteration-{index:03d}" iteration_dir = args.output_dir / name + iteration_dir.mkdir(parents=True, exist_ok=True) train_run = args.output_dir / "runs" / name evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"] command = None if evaluate_existing else training_command( @@ -244,7 +247,8 @@ def main() -> int: data=train_yaml, project=args.output_dir / "runs", name=name, - epochs=args.epochs, + epochs=args.epochs, + patience=args.patience, seed=args.seed + index, batch=args.batch, workers=args.workers, diff --git a/scripts/tower-v37-completion-command.json b/scripts/tower-v37-completion-command.json index e8a31cd6..fb024f4c 100644 --- a/scripts/tower-v37-completion-command.json +++ b/scripts/tower-v37-completion-command.json @@ -27,6 +27,8 @@ "20", "--epochs", "80", + "--patience", + "18", "--batch", "8", "--workers",