Continue rejected v37 through frozen loop
This commit is contained in:
@@ -31,6 +31,7 @@ def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_p
|
||||
assert "deterministic=True" in command
|
||||
assert "seed=42" in command
|
||||
assert "epochs=160" in command
|
||||
assert "patience=18" in command
|
||||
assert "max_det=1000" in command
|
||||
assert "imgsz=640" in command
|
||||
assert "optimizer=auto" in command
|
||||
@@ -104,6 +105,12 @@ def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -
|
||||
assert json.loads(result.stdout) == {"training_command": None, "evaluate_existing": True}
|
||||
|
||||
|
||||
def test_existing_checkpoint_iteration_directory_can_be_created_without_yolo(tmp_path: Path) -> None:
|
||||
iteration_dir = tmp_path / "closed-loop" / "iteration-001"
|
||||
iteration_dir.mkdir(parents=True, exist_ok=True)
|
||||
assert iteration_dir.is_dir()
|
||||
|
||||
|
||||
def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
||||
audit = tmp_path / "audit.json"
|
||||
audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4}))
|
||||
|
||||
@@ -11804,6 +11804,19 @@ Deployment evidence:
|
||||
malformed JSON and filesystem errors produce an explicit fail-closed
|
||||
supervisor state before any subprocess is started. The updated supervisor
|
||||
was activated live with one remaining v37 GPU process.
|
||||
- V37 early-stopped normally at epoch 23; epoch 5 remained the immutable best
|
||||
checkpoint. The supervisor observed `results.png` and executed its one-shot
|
||||
completion handoff.
|
||||
- Fixed a completed-checkpoint entry defect exposed by that live handoff: the
|
||||
iteration directory is now created before copying/hash-binding an existing
|
||||
model. The repaired calibration ran without opening test/background.
|
||||
- V37 improved calibration aggregate F1 to `0.568` and Flanders F1 to `0.307`,
|
||||
but failed Flanders F1/precision/recall and Wallonia precision. It was
|
||||
rejected and produced checksummed failure-driven sampling for iteration 2.
|
||||
- Made training patience an explicit orchestrator input. A pre-epoch iteration
|
||||
2 process exposing the old hardcoded value `35` was terminated before any
|
||||
result row existed; iteration 2 restarted from the identical checkpoint and
|
||||
sampling with the frozen `patience=18` contract on CUDA.
|
||||
- Added those aerial augmentation parameters to the orchestrator CLI and
|
||||
training command, preventing later failure-driven checkpoints from silently
|
||||
reverting to generic orientation assumptions.
|
||||
|
||||
@@ -976,6 +976,8 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
- [x] Add a guarded calibration-first entry point for completed checkpoints so v37 and future externally interrupted runs can rejoin the automated loop without redundant retraining.
|
||||
- [x] Add and activate a host-side, exact-run-marker supervisor that resumes the v37 CUDA checkpoint after container recreation without launching concurrent trainers.
|
||||
- [x] Bind v37 completion to a one-shot JSON command that enters calibration-first evaluation and up to 20 automatic failure-driven CUDA iterations with the frozen aerial augmentation contract.
|
||||
- [x] Complete v37 by early stopping at epoch 23, reject its best epoch-5 checkpoint calibration-first, and start closed-loop iteration 2 from checksummed failure-driven sampling.
|
||||
- [x] Make orchestrator patience explicit and retain the frozen value `18` in every subsequent CUDA iteration.
|
||||
- [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed.
|
||||
- [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes.
|
||||
|
||||
|
||||
@@ -91,6 +91,7 @@ def training_command(
|
||||
project: Path,
|
||||
name: str,
|
||||
epochs: int,
|
||||
patience: int = 18,
|
||||
seed: int,
|
||||
batch: int,
|
||||
workers: int,
|
||||
@@ -115,7 +116,7 @@ def training_command(
|
||||
f"batch={batch}",
|
||||
"device=0",
|
||||
f"workers={workers}",
|
||||
"patience=35",
|
||||
f"patience={patience}",
|
||||
"cache=disk",
|
||||
"close_mosaic=20",
|
||||
f"max_det={max_det}",
|
||||
@@ -180,6 +181,7 @@ def main() -> int:
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--iterations", type=int, default=1)
|
||||
parser.add_argument("--epochs", type=int, default=160)
|
||||
parser.add_argument("--patience", type=int, default=18)
|
||||
parser.add_argument("--batch", type=int, default=2)
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
parser.add_argument("--max-det", type=int, default=1000)
|
||||
@@ -236,6 +238,7 @@ def main() -> int:
|
||||
index = first_index + offset
|
||||
name = f"iteration-{index:03d}"
|
||||
iteration_dir = args.output_dir / name
|
||||
iteration_dir.mkdir(parents=True, exist_ok=True)
|
||||
train_run = args.output_dir / "runs" / name
|
||||
evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"]
|
||||
command = None if evaluate_existing else training_command(
|
||||
@@ -244,7 +247,8 @@ def main() -> int:
|
||||
data=train_yaml,
|
||||
project=args.output_dir / "runs",
|
||||
name=name,
|
||||
epochs=args.epochs,
|
||||
epochs=args.epochs,
|
||||
patience=args.patience,
|
||||
seed=args.seed + index,
|
||||
batch=args.batch,
|
||||
workers=args.workers,
|
||||
|
||||
@@ -27,6 +27,8 @@
|
||||
"20",
|
||||
"--epochs",
|
||||
"80",
|
||||
"--patience",
|
||||
"18",
|
||||
"--batch",
|
||||
"8",
|
||||
"--workers",
|
||||
|
||||
Reference in New Issue
Block a user