From b5155c702f07fbced37672dcef05f992344f2903 Mon Sep 17 00:00:00 2001 From: Jens Date: Wed, 29 Jul 2026 15:34:09 +0200 Subject: [PATCH] Automate failure-driven training continuation --- backend/tests/test_belgium_training_loop.py | 16 +++++++ docs/BELGIUM_BUILDING_TRAINING_LOOP.md | 4 ++ docs/CODEX_EXECUTION_LOG.md | 9 ++++ docs/TODO.md | 1 + scripts/run_belgium_building_training_loop.py | 42 ++++++++++++++++++- 5 files changed, 71 insertions(+), 1 deletion(-) diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py index 0d4e5a6e..54578540 100644 --- a/backend/tests/test_belgium_training_loop.py +++ b/backend/tests/test_belgium_training_loop.py @@ -62,6 +62,20 @@ def test_training_command_supports_conservative_aerial_finetuning(tmp_path: Path assert f"data={tmp_path / 'dataset.yaml'}" in command +def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_path: Path) -> None: + command = MODULE.failure_sampling_command( + scripts_dir=tmp_path / "scripts", + train_summary=tmp_path / "train-summary.json", + corpus_manifest=tmp_path / "manifest.json", + assessment=tmp_path / "assessment.json", + output_dir=tmp_path / "iteration-001" / "failure-driven-training", + ) + assert command[1].endswith("build_failure_driven_yolo_sampling.py") + assert command[command.index("--summary") + 1].endswith("train-summary.json") + assert command[command.index("--assessment") + 1].endswith("assessment.json") + assert command[command.index("--output-dir") + 1].endswith("failure-driven-training") + + def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: audit = tmp_path / "audit.json" audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4})) @@ -73,6 +87,8 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: str(tmp_path / "base.pt"), "--train-yaml", str(tmp_path / "dataset.yaml"), + "--train-summary", + str(tmp_path / "train-summary.json"), "--dataset-audit", str(audit), "--calibration-summary", diff --git a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md index 988fa74a..f999c828 100644 --- a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md +++ b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md @@ -92,6 +92,10 @@ recall are repeated, while true negative train tiles are repeated when a regional precision gate or the pure-background gate fails. Calibration, test, background-test and validation AOIs are excluded by their frozen corpus split; the generated evidence records that no protected sample entered training. +The checkpointed orchestrator invokes this builder after every rejected +iteration, stores its checksum in `training-loop-state.json`, and uses the +resulting dataset YAML for the next checkpoint. A restart resumes both the +candidate weights and that exact failure-driven training input. The orchestrator refuses to start unless every automated frozen-dataset gate passes and the corpus contains zero blank/low-variance positive tiles. The audit status may remain `needs_human_review` while training and objective diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index ff00c416..251a3a8e 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11773,11 +11773,20 @@ Deployment evidence: immutable, automated failures are empty, spatial leakage is `ok`, and blank positive-tile count is zero. Human sign-off remains a separate mandatory final promotion gate. +- Closed the next orchestration gap: a rejected iteration now invokes the + leak-free failure-driven sampler automatically, records its evidence + checksum and next dataset YAML in `training-loop-state.json`, and resumes + both the candidate weights and exact sampling input after interruption. +- Confirmed v37 epoch 1 completed on CUDA with validation precision `0.601`, + recall `0.455`, mAP50 `0.474` and mAP50-95 `0.205`; the run remains inactive + and these internal-validation metrics are not release evidence. Verified in this pass: - `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_belgium_training_iteration_assessment.py backend/tests/test_belgium_training_portfolio.py` (`12 passed`). +- `py -3 -m pytest -q backend/tests/test_belgium_training_loop.py backend/tests/test_failure_driven_yolo_sampling.py backend/tests/test_belgium_training_iteration_assessment.py` + (`15 passed`). Open: diff --git a/docs/TODO.md b/docs/TODO.md index 203a8deb..543652af 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -972,6 +972,7 @@ This file now starts with the current implementation status. Older preparation/b - [x] Reject positive labels over blank/no-data imagery and replace partial SPW 2024 coverage with the complete dated SPW 2023 campaign. - [x] Exclude GRB/PICC features created after the corresponding dated imagery period while retaining auditable rejection evidence. - [x] Allow the objective CUDA loop to consume an automatically clean `needs_human_review` corpus while keeping final human sign-off as a separate, mandatory promotion gate. +- [x] Persist checksummed train-only failure-driven sampling after every rejected loop iteration and resume the next checkpoint from that exact dataset YAML. - [x] Evaluate the completed v36 YOLO11x checkpoint calibration-first on the rotated v30 holdouts; reject it before opening test/background because the regional calibration gate failed. - [ ] Finish and assess the leak-free v37 YOLO11x failure-driven CUDA iteration; open test/background evidence only if every calibration gate passes. diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index 897493c9..22a85253 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -131,6 +131,24 @@ def training_command( return command +def failure_sampling_command( + *, + scripts_dir: Path, + train_summary: Path, + corpus_manifest: Path, + assessment: Path, + output_dir: Path, +) -> list[str]: + return [ + sys.executable, + str(scripts_dir / "build_failure_driven_yolo_sampling.py"), + "--summary", str(train_summary), + "--corpus-manifest", str(corpus_manifest), + "--assessment", str(assessment), + "--output-dir", str(output_dir), + ] + + def run(command: list[str], log_path: Path | None = None, *, allowed: set[int] = {0}) -> int: if log_path: log_path.parent.mkdir(parents=True, exist_ok=True) @@ -147,6 +165,7 @@ def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--initial-model", type=Path, required=True) parser.add_argument("--train-yaml", type=Path, required=True) + parser.add_argument("--train-summary", type=Path, required=True) parser.add_argument("--dataset-audit", type=Path, required=True) parser.add_argument("--calibration-summary", type=Path, required=True) parser.add_argument("--test-summary", type=Path, required=True) @@ -195,6 +214,7 @@ def main() -> int: state = json.loads(state_path.read_text(encoding="utf-8")) state["status"] = "running" model = Path(state.get("next_model") or args.initial_model) + train_yaml = Path(state.get("next_train_yaml") or args.train_yaml) first_index = len(state["iterations"]) + 1 scripts_dir = Path(__file__).resolve().parent @@ -206,7 +226,7 @@ def main() -> int: command = training_command( args.yolo, model=model, - data=args.train_yaml, + data=train_yaml, project=args.output_dir / "runs", name=name, epochs=args.epochs, @@ -339,7 +359,27 @@ def main() -> int: write_json(state_path, state) print(json.dumps(state, indent=2)) return 0 + sampling_dir = iteration_dir / "failure-driven-training" + run( + failure_sampling_command( + scripts_dir=scripts_dir, + train_summary=args.train_summary, + corpus_manifest=args.corpus_manifest, + assessment=assessment, + output_dir=sampling_dir, + ), + iteration_dir / "failure-driven-sampling.log", + ) + sampling_evidence = sampling_dir / "failure-driven-sampling.json" + next_train_yaml = sampling_dir / "dataset.yaml" + if not sampling_evidence.is_file() or not next_train_yaml.is_file(): + raise RuntimeError("Failure-driven sampling produced incomplete evidence") + record["failure_driven_sampling"] = str(sampling_evidence) + record["failure_driven_sampling_sha256"] = sha256(sampling_evidence) + record["next_train_yaml"] = str(next_train_yaml) + state["next_train_yaml"] = str(next_train_yaml) model = candidate + train_yaml = next_train_yaml write_json(state_path, state) state["status"] = "continue_training_loop"