Reject blank positive imagery in training loop

backend/app/services/orthophoto_acquisition_service.py

@@ -117,6 +117,27 @@ class OrthophotoAcquisitionService:
                     valid_from=datetime(2024, 4, 6, tzinfo=UTC),
                     valid_to=datetime(2024, 9, 21, 23, 59, 59, tzinfo=UTC),
                 ),
+                OrthophotoProduct(
+                    key="wallonia_2023",
+                    display_name="Zomerorthofoto Wallonië 2023",
+                    observation_label="27 mei tot 25 juni 2023",
+                    temporal_granularity="period",
+                    native_resolution_m=0.25,
+                    wms_url="https://geoservices.wallonie.be/arcgis/services/IMAGERIE/ORTHO_2023_ETE/MapServer/WMSServer",
+                    layer="0",
+                    catalog_url="https://geoportail.wallonie.be/catalogue/ad55c2ce-62ad-4c3c-b3cf-8fbc270a6b6e.html",
+                    limitation_message="Officiële gebiedsdekkende SPW-zomercampagne 2023; exacte vliegdata zijn beschikbaar in het afzonderlijke maillage- en tuilageproduct.",
+                    provider="spw_orthophoto",
+                    source_label="SPW ORTHO_2023_ETE WMS",
+                    attribution="Bron: Service public de Wallonie (SPW), Orthophotos 2023 Été",
+                    license_note="CC BY 4.0; citeer SPW en vermeld wijzigingen.",
+                    series_namespace="spw",
+                    coverage_zone="wallonia",
+                    supports_detection=True,
+                    observed_at=datetime(2023, 5, 27, tzinfo=UTC),
+                    valid_from=datetime(2023, 5, 27, tzinfo=UTC),
+                    valid_to=datetime(2023, 6, 25, 23, 59, 59, tzinfo=UTC),
+                ),
                 OrthophotoProduct(
                     key="brussels_latest",
                     display_name="Meest recente orthofoto Brussel",

backend/tests/test_belgium_training_loop.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).parents[2] / "scripts" / "run_belgium_building_training_loop.py"
+SPEC = importlib.util.spec_from_file_location("training_loop", SCRIPT)
+assert SPEC and SPEC.loader
+MODULE = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MODULE)
+
+
+def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_path: Path) -> None:
+    command = MODULE.training_command(
+        "yolo",
+        model=tmp_path / "base.pt",
+        data=tmp_path / "dataset.yaml",
+        project=tmp_path / "runs",
+        name="iteration-001",
+        epochs=160,
+        seed=42,
+        batch=2,
+        workers=4,
+    )
+    assert command[:2] == ["yolo", "train"]
+    assert "device=0" in command
+    assert "deterministic=True" in command
+    assert "seed=42" in command
+    assert "epochs=160" in command
+    assert f"data={tmp_path / 'dataset.yaml'}" in command

backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py

@@ -65,6 +65,7 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat
                         "label_count": 2,
                         "is_negative": False,
                         "is_repeated_background_negative": False,
+                        "low_visual_variance": True,
                     },
                     {
                         "sample_slug": "postel_bos",
@@ -144,6 +145,7 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat
     assert report["positive_sample_count"] == 2
     assert report["background_sample_count"] == 1
     assert report["train_negative_tile_count"] == 2
+    assert report["low_variance_positive_tile_count"] == 1
     assert report["repeated_background_negative_tile_count"] == 1
     assert report["label_stats"]["parsed_label_count"] == 3
     assert report["label_stats"]["invalid_label_count"] == 0
@@ -169,6 +171,7 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat
     assert "repeated_background_negative_share_above_gate" in warning_codes
     assert "median_box_area_below_gate" in warning_codes
     assert "small_box_share_above_gate" in warning_codes
+    assert "positive_tiles_have_low_visual_variance" in warning_codes
 
     markdown = (output_dir / "operator_yolo_dataset_quality_audit.md").read_text(encoding="utf-8")
     assert "Operator YOLO Dataset Quality Audit" in markdown

backend/tests/test_sprint196_map_orthophoto_analysis.py

@@ -169,7 +169,7 @@ def test_orthophoto_product_registry_exposes_only_governed_official_layers() ->
     assert {"2025", "2012", "2008_2011", "2000_2003", "1979_1990", "1971"}.issubset(keys)
     assert next(item for item in products if item["key"] == "most_recent")["supports_detection"] is True
     detection_keys = {item["key"] for item in products if item["supports_detection"]}
-    assert {"most_recent", "wallonia_latest", "wallonia_2024", "brussels_latest", "brussels_2025", "2025"} <= detection_keys
+    assert {"most_recent", "wallonia_latest", "wallonia_2024", "wallonia_2023", "brussels_latest", "brussels_2025", "2025"} <= detection_keys
     by_key = {item["key"]: item for item in products}
     assert by_key["wallonia_latest"]["provider"] == "spw_orthophoto"
     assert by_key["wallonia_latest"]["coverage_zone"] == "wallonia"

deploy/unraid/Dockerfile.all-in-one

@@ -126,6 +126,7 @@ COPY scripts/provision_belgium_building_training_portfolio.py /app/scripts/provi
 COPY scripts/audit_belgium_building_corpus.py /app/scripts/audit_belgium_building_corpus.py
 COPY scripts/evaluate_belgium_building_candidate.py /app/scripts/evaluate_belgium_building_candidate.py
 COPY scripts/assess_belgium_building_training_iteration.py /app/scripts/assess_belgium_building_training_iteration.py
+COPY scripts/run_belgium_building_training_loop.py /app/scripts/run_belgium_building_training_loop.py
 COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
 COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
 COPY scripts/train_operator_yolo_detector.sh /app/scripts/train_operator_yolo_detector.sh

docs/BELGIUM_BUILDING_TRAINING_LOOP.md

@@ -74,6 +74,8 @@ The active production model remains unchanged while any gate fails.
 - per-AOI evaluator: `scripts/evaluate_belgium_building_candidate.py`;
 - calibration-only selection and release gates:
   `scripts/assess_belgium_building_training_iteration.py`.
+- checkpointed CUDA orchestration:
+  `scripts/run_belgium_building_training_loop.py`.
 
 Every failed assessment returns `continue_training_loop`. Only a report with
 `training_complete` may proceed to final human review and guarded activation.

scripts/audit_operator_yolo_dataset_quality.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import argparse
 import json
 import statistics
-from collections import Counter, defaultdict
+from collections import Counter
 from pathlib import Path
 from typing import Any
 
@@ -246,6 +246,7 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
     ]
     positive_tiles = [tile for tile in tiles if tile not in negative_tiles]
     train_negative_tiles = [tile for tile in negative_tiles if tile.get("split") == "train"]
+    low_variance_positive_tiles = [tile for tile in positive_tiles if tile.get("low_visual_variance")]
     val_positive_samples = {
         str(tile.get("sample_slug"))
         for tile in positive_tiles
@@ -288,6 +289,15 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
                 f"gate is {args.max_repeated_negative_share:.3f}."
             ),
         )
+    if low_variance_positive_tiles:
+        add_warning(
+            warnings,
+            "positive_tiles_have_low_visual_variance",
+            (
+                f"{len(low_variance_positive_tiles)} positive tiles are visually blank/low-variance; "
+                "the imagery source does not support their labels."
+            ),
+        )
     if label_stats["missing_label_file_count"]:
         add_warning(
             warnings,
@@ -337,6 +347,7 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
         "train_tile_count": split_counts.get("train", 0),
         "val_tile_count": split_counts.get("val", 0),
         "train_negative_tile_count": len(train_negative_tiles),
+        "low_variance_positive_tile_count": len(low_variance_positive_tiles),
         "repeated_background_negative_tile_count": repeated_negative_count,
         "repeated_background_negative_share_of_negatives": repeated_negative_share,
         "positive_tile_share": len(positive_tiles) / len(tiles) if tiles else 0.0,
@@ -371,6 +382,10 @@ def build_recommendations(warnings: list[dict[str, str]]) -> list[str]:
         )
     if "label_files_missing" in codes or "invalid_label_rows" in codes:
         recommendations.append("Regenerate the YOLO tile dataset and review exporter path/label integrity.")
+    if "positive_tiles_have_low_visual_variance" in codes:
+        recommendations.append(
+            "Reject the raster product for affected AOIs or replace it with an officially complete imagery edition before training."
+        )
     if not recommendations:
         recommendations.append("Dataset audit passed the configured gates; continue with benchmarked training.")
     return recommendations
@@ -389,6 +404,7 @@ def write_markdown(report: dict[str, Any], path: Path) -> None:
         f"- Samples: {report['sample_count']} ({report['positive_sample_count']} positive, {report['background_sample_count']} background)",
         f"- Repeated background negative share: {report['repeated_background_negative_share_of_negatives']:.3f}",
         f"- Minimum visible label ratio: {format_optional_float(report.get('min_label_visible_ratio'))}",
+        f"- Blank/low-variance positive tiles: {report['low_variance_positive_tile_count']}",
         "",
         "## Label Quality",
         "",

scripts/provision_belgium_building_training_portfolio.py

@@ -115,7 +115,7 @@ REGION_CONTRACT = {
     },
     "wallonia": {
         "area_id": "e5fd742a-ca18-4520-abe0-d28416aa2ece",
-        "orthophoto_product": "wallonia_2024",
+        "orthophoto_product": "wallonia_2023",
         "reference_path": "datasets/official-vector/acquire",
         "reference_product": "spw_picc_buildings",
     },

scripts/run_belgium_building_training_loop.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""Run checkpointed CUDA train/evaluate iterations until gates pass or a batch yields."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import shutil
+import subprocess
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def write_json(path: Path, value: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    temporary.write_text(json.dumps(value, indent=2), encoding="utf-8")
+    temporary.replace(path)
+
+
+def training_command(
+    yolo: str,
+    *,
+    model: Path,
+    data: Path,
+    project: Path,
+    name: str,
+    epochs: int,
+    seed: int,
+    batch: int,
+    workers: int,
+) -> list[str]:
+    return [
+        yolo,
+        "train",
+        f"model={model}",
+        f"data={data}",
+        f"epochs={epochs}",
+        "imgsz=640",
+        f"batch={batch}",
+        "device=0",
+        f"workers={workers}",
+        "patience=35",
+        "cache=disk",
+        "close_mosaic=20",
+        f"seed={seed}",
+        "deterministic=True",
+        f"project={project}",
+        f"name={name}",
+        "exist_ok=True",
+    ]
+
+
+def run(command: list[str], log_path: Path | None = None, *, allowed: set[int] = {0}) -> int:
+    if log_path:
+        log_path.parent.mkdir(parents=True, exist_ok=True)
+        with log_path.open("a", encoding="utf-8") as log:
+            completed = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, check=False)
+    else:
+        completed = subprocess.run(command, check=False)
+    if completed.returncode not in allowed:
+        raise RuntimeError(f"Command failed ({completed.returncode}): {' '.join(command)}")
+    return completed.returncode
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--initial-model", type=Path, required=True)
+    parser.add_argument("--train-yaml", type=Path, required=True)
+    parser.add_argument("--calibration-summary", type=Path, required=True)
+    parser.add_argument("--test-summary", type=Path, required=True)
+    parser.add_argument("--background-summary", type=Path, required=True)
+    parser.add_argument("--corpus-manifest", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--iterations", type=int, default=1)
+    parser.add_argument("--epochs", type=int, default=160)
+    parser.add_argument("--batch", type=int, default=2)
+    parser.add_argument("--workers", type=int, default=4)
+    parser.add_argument("--seed", type=int, default=20260731)
+    parser.add_argument("--yolo", default="yolo")
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+    if args.iterations < 1:
+        raise SystemExit("--iterations must be positive")
+
+    state_path = args.output_dir / "training-loop-state.json"
+    state: dict[str, Any] = {
+        "schema_version": 1,
+        "status": "running",
+        "started_at": datetime.now(UTC).isoformat(),
+        "initial_model": str(args.initial_model),
+        "train_yaml": str(args.train_yaml),
+        "corpus_manifest": str(args.corpus_manifest),
+        "iterations": [],
+    }
+    if state_path.is_file():
+        state = json.loads(state_path.read_text(encoding="utf-8"))
+        state["status"] = "running"
+    model = Path(state.get("next_model") or args.initial_model)
+    first_index = len(state["iterations"]) + 1
+    scripts_dir = Path(__file__).resolve().parent
+
+    for offset in range(args.iterations):
+        index = first_index + offset
+        name = f"iteration-{index:03d}"
+        iteration_dir = args.output_dir / name
+        train_run = args.output_dir / "runs" / name
+        command = training_command(
+            args.yolo,
+            model=model,
+            data=args.train_yaml,
+            project=args.output_dir / "runs",
+            name=name,
+            epochs=args.epochs,
+            seed=args.seed + index,
+            batch=args.batch,
+            workers=args.workers,
+        )
+        if args.dry_run:
+            print(json.dumps({"training_command": command}, indent=2))
+            return 0
+        run(command, iteration_dir / "training.log")
+        best = train_run / "weights" / "best.pt"
+        if not best.is_file():
+            raise RuntimeError(f"Training produced no best checkpoint: {best}")
+        candidate = iteration_dir / "candidate.pt"
+        shutil.copy2(best, candidate)
+
+        reports: dict[str, Path] = {}
+        for role, summary in (
+            ("calibration", args.calibration_summary),
+            ("test", args.test_summary),
+            ("background", args.background_summary),
+        ):
+            report = iteration_dir / f"{role}.json"
+            reports[role] = report
+            run(
+                [
+                    sys.executable,
+                    str(scripts_dir / "evaluate_belgium_building_candidate.py"),
+                    "--model",
+                    str(candidate),
+                    "--summary",
+                    str(summary),
+                    "--corpus-manifest",
+                    str(args.corpus_manifest),
+                    "--output",
+                    str(report),
+                    "--device",
+                    "cuda:0",
+                ],
+                iteration_dir / f"{role}.log",
+            )
+        assessment = iteration_dir / "assessment.json"
+        run(
+            [
+                sys.executable,
+                str(scripts_dir / "assess_belgium_building_training_iteration.py"),
+                "--calibration",
+                str(reports["calibration"]),
+                "--test",
+                str(reports["test"]),
+                "--background",
+                str(reports["background"]),
+                "--output",
+                str(assessment),
+            ],
+            iteration_dir / "assessment.log",
+            allowed={0, 2},
+        )
+        decision = json.loads(assessment.read_text(encoding="utf-8"))
+        record = {
+            "iteration": index,
+            "candidate": str(candidate),
+            "candidate_sha256": sha256(candidate),
+            "assessment": str(assessment),
+            "status": decision["status"],
+            "failures": decision["failures"],
+        }
+        state["iterations"].append(record)
+        state["next_model"] = str(candidate)
+        if decision["status"] == "training_complete":
+            state["status"] = "training_complete"
+            state["completed_at"] = datetime.now(UTC).isoformat()
+            write_json(state_path, state)
+            print(json.dumps(state, indent=2))
+            return 0
+        model = candidate
+        write_json(state_path, state)
+
+    state["status"] = "continue_training_loop"
+    state["yielded_at"] = datetime.now(UTC).isoformat()
+    write_json(state_path, state)
+    print(json.dumps(state, indent=2))
+    return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())