Continue rejected v37 through frozen loop
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Jens
2026-07-29 16:23:28 +02:00
parent f18a23c060
commit b393d9f65d
5 changed files with 30 additions and 2 deletions
@@ -31,6 +31,7 @@ def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_p
assert "deterministic=True" in command assert "deterministic=True" in command
assert "seed=42" in command assert "seed=42" in command
assert "epochs=160" in command assert "epochs=160" in command
assert "patience=18" in command
assert "max_det=1000" in command assert "max_det=1000" in command
assert "imgsz=640" in command assert "imgsz=640" in command
assert "optimizer=auto" in command assert "optimizer=auto" in command
@@ -104,6 +105,12 @@ def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -
assert json.loads(result.stdout) == {"training_command": None, "evaluate_existing": True} assert json.loads(result.stdout) == {"training_command": None, "evaluate_existing": True}
def test_existing_checkpoint_iteration_directory_can_be_created_without_yolo(tmp_path: Path) -> None:
iteration_dir = tmp_path / "closed-loop" / "iteration-001"
iteration_dir.mkdir(parents=True, exist_ok=True)
assert iteration_dir.is_dir()
def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
audit = tmp_path / "audit.json" audit = tmp_path / "audit.json"
audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4})) audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4}))
+13
View File
@@ -11804,6 +11804,19 @@ Deployment evidence:
malformed JSON and filesystem errors produce an explicit fail-closed malformed JSON and filesystem errors produce an explicit fail-closed
supervisor state before any subprocess is started. The updated supervisor supervisor state before any subprocess is started. The updated supervisor
was activated live with one remaining v37 GPU process. was activated live with one remaining v37 GPU process.
- V37 early-stopped normally at epoch 23; epoch 5 remained the immutable best
checkpoint. The supervisor observed `results.png` and executed its one-shot
completion handoff.
- Fixed a completed-checkpoint entry defect exposed by that live handoff: the
iteration directory is now created before copying/hash-binding an existing
model. The repaired calibration ran without opening test/background.
- V37 improved calibration aggregate F1 to `0.568` and Flanders F1 to `0.307`,
but failed Flanders F1/precision/recall and Wallonia precision. It was
rejected and produced checksummed failure-driven sampling for iteration 2.
- Made training patience an explicit orchestrator input. A pre-epoch iteration
2 process exposing the old hardcoded value `35` was terminated before any
result row existed; iteration 2 restarted from the identical checkpoint and
sampling with the frozen `patience=18` contract on CUDA.
- Added those aerial augmentation parameters to the orchestrator CLI and - Added those aerial augmentation parameters to the orchestrator CLI and
training command, preventing later failure-driven checkpoints from silently training command, preventing later failure-driven checkpoints from silently
reverting to generic orientation assumptions. reverting to generic orientation assumptions.
+2
View File
@@ -976,6 +976,8 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Add a guarded calibration-first entry point for completed checkpoints so v37 and future externally interrupted runs can rejoin the automated loop without redundant retraining. - [x] Add a guarded calibration-first entry point for completed checkpoints so v37 and future externally interrupted runs can rejoin the automated loop without redundant retraining.
- [x] Add and activate a host-side, exact-run-marker supervisor that resumes the v37 CUDA checkpoint after container recreation without launching concurrent trainers. - [x] Add and activate a host-side, exact-run-marker supervisor that resumes the v37 CUDA checkpoint after container recreation without launching concurrent trainers.
- [x] Bind v37 completion to a one-shot JSON command that enters calibration-first evaluation and up to 20 automatic failure-driven CUDA iterations with the frozen aerial augmentation contract. - [x] Bind v37 completion to a one-shot JSON command that enters calibration-first evaluation and up to 20 automatic failure-driven CUDA iterations with the frozen aerial augmentation contract.
- [x] Complete v37 by early stopping at epoch 23, reject its best epoch-5 checkpoint calibration-first, and start closed-loop iteration 2 from checksummed failure-driven sampling.
- [x] Make orchestrator patience explicit and retain the frozen value `18` in every subsequent CUDA iteration.
- [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed. - [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed.
- [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes. - [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes.
@@ -91,6 +91,7 @@ def training_command(
project: Path, project: Path,
name: str, name: str,
epochs: int, epochs: int,
patience: int = 18,
seed: int, seed: int,
batch: int, batch: int,
workers: int, workers: int,
@@ -115,7 +116,7 @@ def training_command(
f"batch={batch}", f"batch={batch}",
"device=0", "device=0",
f"workers={workers}", f"workers={workers}",
"patience=35", f"patience={patience}",
"cache=disk", "cache=disk",
"close_mosaic=20", "close_mosaic=20",
f"max_det={max_det}", f"max_det={max_det}",
@@ -180,6 +181,7 @@ def main() -> int:
parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--iterations", type=int, default=1) parser.add_argument("--iterations", type=int, default=1)
parser.add_argument("--epochs", type=int, default=160) parser.add_argument("--epochs", type=int, default=160)
parser.add_argument("--patience", type=int, default=18)
parser.add_argument("--batch", type=int, default=2) parser.add_argument("--batch", type=int, default=2)
parser.add_argument("--workers", type=int, default=4) parser.add_argument("--workers", type=int, default=4)
parser.add_argument("--max-det", type=int, default=1000) parser.add_argument("--max-det", type=int, default=1000)
@@ -236,6 +238,7 @@ def main() -> int:
index = first_index + offset index = first_index + offset
name = f"iteration-{index:03d}" name = f"iteration-{index:03d}"
iteration_dir = args.output_dir / name iteration_dir = args.output_dir / name
iteration_dir.mkdir(parents=True, exist_ok=True)
train_run = args.output_dir / "runs" / name train_run = args.output_dir / "runs" / name
evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"] evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"]
command = None if evaluate_existing else training_command( command = None if evaluate_existing else training_command(
@@ -245,6 +248,7 @@ def main() -> int:
project=args.output_dir / "runs", project=args.output_dir / "runs",
name=name, name=name,
epochs=args.epochs, epochs=args.epochs,
patience=args.patience,
seed=args.seed + index, seed=args.seed + index,
batch=args.batch, batch=args.batch,
workers=args.workers, workers=args.workers,
@@ -27,6 +27,8 @@
"20", "20",
"--epochs", "--epochs",
"80", "80",
"--patience",
"18",
"--batch", "--batch",
"8", "8",
"--workers", "--workers",