diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py index b12a7d11..2ff52d90 100644 --- a/backend/tests/test_belgium_training_loop.py +++ b/backend/tests/test_belgium_training_loop.py @@ -1,6 +1,9 @@ from __future__ import annotations import importlib.util +import json +import subprocess +import sys from pathlib import Path @@ -29,3 +32,35 @@ def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_p assert "seed=42" in command assert "epochs=160" in command assert f"data={tmp_path / 'dataset.yaml'}" in command + + +def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: + audit = tmp_path / "audit.json" + audit.write_text(json.dumps({"status": "needs_attention", "low_variance_positive_tile_count": 4})) + result = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--initial-model", + str(tmp_path / "base.pt"), + "--train-yaml", + str(tmp_path / "dataset.yaml"), + "--dataset-audit", + str(audit), + "--calibration-summary", + str(tmp_path / "cal.json"), + "--test-summary", + str(tmp_path / "test.json"), + "--background-summary", + str(tmp_path / "background.json"), + "--corpus-manifest", + str(tmp_path / "manifest.json"), + "--output-dir", + str(tmp_path / "output"), + ], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode != 0 + assert "Dataset audit is not ok" in result.stderr diff --git a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md index 11a6a76b..9e822036 100644 --- a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md +++ b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md @@ -79,3 +79,5 @@ The active production model remains unchanged while any gate fails. Every failed assessment returns `continue_training_loop`. Only a report with `training_complete` may proceed to final human review and guarded activation. +The orchestrator refuses to start unless the frozen dataset audit is `ok` and +contains zero blank/low-variance positive tiles. diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 2331411f..2526d9bf 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -11570,3 +11570,20 @@ Next gate: became Brussels `0.566`, Wallonia `0.427`, Flanders `0.222`. The generic-base iteration did not beat it. Both remained rejected, and iteration 4 started from the stronger candidate on the dated 75-AOI corpus. + +## 2026-07-27 - Positive-imagery gate and complete SPW campaign + +- Visual inspection caught 20 positive Walloon tiles with labels over white + no-data imagery. SPW 2024 is an official but partial campaign, so that + training run was stopped and its candidate is invalid. +- Dataset QA now fails whenever any positive tile is blank/low-variance. The + checkpointed training orchestrator also refuses to start unless this audit + is `ok` with zero affected positive tiles. +- Replaced the partial product with the official, territory-complete SPW summer + 2023 campaign (27 May through 25 June, 25 cm). The frozen replacement corpus + `building-be-v5-dated-20260727-r1` retains 75 AOIs and 13,765 accepted labels. + Manifest SHA-256 is + `eade90d3ba22b426b72a300fcaa9a01f6eb1d18567c4a53413b72b8048f5f021`. +- The v5 tile audit passed with zero blank positive tiles. The automated CUDA + loop started from the strongest prior candidate and will checkpoint every + train/calibrate/test/background assessment without promoting failed models. diff --git a/docs/TODO.md b/docs/TODO.md index 3da71425..8e7e39f1 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -953,4 +953,5 @@ This file now starts with the current implementation status. Older preparation/b - [x] Freeze the objective train/evaluate/error-analysis loop and regional exit gates. - [x] Expand the second corpus wave to 60 independent AOIs and 10,262 accepted labels. - [ ] Resolve the v3 Flanders and Wallonia generalisation failures through additional training-only evidence and retraining. -- [x] Replace rolling-mosaic training inputs with governed dated 2025 Flanders/Brussels and 2024 SPW imagery; retain exact flight-day limitations. +- [x] Replace rolling-mosaic training inputs with governed dated 2025 Flanders/Brussels and complete 2023 SPW imagery; retain exact flight-day limitations. +- [x] Reject positive labels over blank/no-data imagery and replace partial SPW 2024 coverage with the complete dated SPW 2023 campaign. diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index a1d7427f..6129a35d 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -78,6 +78,7 @@ def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--initial-model", type=Path, required=True) parser.add_argument("--train-yaml", type=Path, required=True) + parser.add_argument("--dataset-audit", type=Path, required=True) parser.add_argument("--calibration-summary", type=Path, required=True) parser.add_argument("--test-summary", type=Path, required=True) parser.add_argument("--background-summary", type=Path, required=True) @@ -93,6 +94,11 @@ def main() -> int: args = parser.parse_args() if args.iterations < 1: raise SystemExit("--iterations must be positive") + dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8")) + if dataset_audit.get("status") != "ok": + raise SystemExit(f"Dataset audit is not ok: {args.dataset_audit}") + if int(dataset_audit.get("low_variance_positive_tile_count") or 0) != 0: + raise SystemExit("Dataset audit contains blank/low-variance positive tiles") state_path = args.output_dir / "training-loop-state.json" state: dict[str, Any] = { @@ -101,6 +107,7 @@ def main() -> int: "started_at": datetime.now(UTC).isoformat(), "initial_model": str(args.initial_model), "train_yaml": str(args.train_yaml), + "dataset_audit": str(args.dataset_audit), "corpus_manifest": str(args.corpus_manifest), "iterations": [], }