Automate failure-driven training continuation

scripts/run_belgium_building_training_loop.py

@@ -131,6 +131,24 @@ def training_command(
     return command
 
 
+def failure_sampling_command(
+    *,
+    scripts_dir: Path,
+    train_summary: Path,
+    corpus_manifest: Path,
+    assessment: Path,
+    output_dir: Path,
+) -> list[str]:
+    return [
+        sys.executable,
+        str(scripts_dir / "build_failure_driven_yolo_sampling.py"),
+        "--summary", str(train_summary),
+        "--corpus-manifest", str(corpus_manifest),
+        "--assessment", str(assessment),
+        "--output-dir", str(output_dir),
+    ]
+
+
 def run(command: list[str], log_path: Path | None = None, *, allowed: set[int] = {0}) -> int:
     if log_path:
         log_path.parent.mkdir(parents=True, exist_ok=True)
@@ -147,6 +165,7 @@ def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--initial-model", type=Path, required=True)
     parser.add_argument("--train-yaml", type=Path, required=True)
+    parser.add_argument("--train-summary", type=Path, required=True)
     parser.add_argument("--dataset-audit", type=Path, required=True)
     parser.add_argument("--calibration-summary", type=Path, required=True)
     parser.add_argument("--test-summary", type=Path, required=True)
@@ -195,6 +214,7 @@ def main() -> int:
         state = json.loads(state_path.read_text(encoding="utf-8"))
         state["status"] = "running"
     model = Path(state.get("next_model") or args.initial_model)
+    train_yaml = Path(state.get("next_train_yaml") or args.train_yaml)
     first_index = len(state["iterations"]) + 1
     scripts_dir = Path(__file__).resolve().parent
 
@@ -206,7 +226,7 @@ def main() -> int:
         command = training_command(
             args.yolo,
             model=model,
-            data=args.train_yaml,
+            data=train_yaml,
             project=args.output_dir / "runs",
             name=name,
             epochs=args.epochs,
@@ -339,7 +359,27 @@ def main() -> int:
             write_json(state_path, state)
             print(json.dumps(state, indent=2))
             return 0
+        sampling_dir = iteration_dir / "failure-driven-training"
+        run(
+            failure_sampling_command(
+                scripts_dir=scripts_dir,
+                train_summary=args.train_summary,
+                corpus_manifest=args.corpus_manifest,
+                assessment=assessment,
+                output_dir=sampling_dir,
+            ),
+            iteration_dir / "failure-driven-sampling.log",
+        )
+        sampling_evidence = sampling_dir / "failure-driven-sampling.json"
+        next_train_yaml = sampling_dir / "dataset.yaml"
+        if not sampling_evidence.is_file() or not next_train_yaml.is_file():
+            raise RuntimeError("Failure-driven sampling produced incomplete evidence")
+        record["failure_driven_sampling"] = str(sampling_evidence)
+        record["failure_driven_sampling_sha256"] = sha256(sampling_evidence)
+        record["next_train_yaml"] = str(next_train_yaml)
+        state["next_train_yaml"] = str(next_train_yaml)
         model = candidate
+        train_yaml = next_train_yaml
         write_json(state_path, state)
 
     state["status"] = "continue_training_loop"