From 70897a326582fa2c93caa46a4f965dba69786553 Mon Sep 17 00:00:00 2001 From: Jens Date: Mon, 27 Jul 2026 02:57:36 +0200 Subject: [PATCH] Reject blank positive imagery in training loop --- .../orthophoto_acquisition_service.py | 21 ++ backend/tests/test_belgium_training_loop.py | 31 +++ ...146_operator_yolo_dataset_quality_audit.py | 3 + .../test_sprint196_map_orthophoto_analysis.py | 2 +- deploy/unraid/Dockerfile.all-in-one | 1 + docs/BELGIUM_BUILDING_TRAINING_LOOP.md | 2 + .../audit_operator_yolo_dataset_quality.py | 18 +- ...ion_belgium_building_training_portfolio.py | 2 +- scripts/run_belgium_building_training_loop.py | 210 ++++++++++++++++++ 9 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 backend/tests/test_belgium_training_loop.py create mode 100644 scripts/run_belgium_building_training_loop.py diff --git a/backend/app/services/orthophoto_acquisition_service.py b/backend/app/services/orthophoto_acquisition_service.py index 71e15a64..52c51816 100644 --- a/backend/app/services/orthophoto_acquisition_service.py +++ b/backend/app/services/orthophoto_acquisition_service.py @@ -117,6 +117,27 @@ class OrthophotoAcquisitionService: valid_from=datetime(2024, 4, 6, tzinfo=UTC), valid_to=datetime(2024, 9, 21, 23, 59, 59, tzinfo=UTC), ), + OrthophotoProduct( + key="wallonia_2023", + display_name="Zomerorthofoto Wallonië 2023", + observation_label="27 mei tot 25 juni 2023", + temporal_granularity="period", + native_resolution_m=0.25, + wms_url="https://geoservices.wallonie.be/arcgis/services/IMAGERIE/ORTHO_2023_ETE/MapServer/WMSServer", + layer="0", + catalog_url="https://geoportail.wallonie.be/catalogue/ad55c2ce-62ad-4c3c-b3cf-8fbc270a6b6e.html", + limitation_message="Officiële gebiedsdekkende SPW-zomercampagne 2023; exacte vliegdata zijn beschikbaar in het afzonderlijke maillage- en tuilageproduct.", + provider="spw_orthophoto", + source_label="SPW ORTHO_2023_ETE WMS", + attribution="Bron: Service public de Wallonie (SPW), Orthophotos 2023 Été", + license_note="CC BY 4.0; citeer SPW en vermeld wijzigingen.", + series_namespace="spw", + coverage_zone="wallonia", + supports_detection=True, + observed_at=datetime(2023, 5, 27, tzinfo=UTC), + valid_from=datetime(2023, 5, 27, tzinfo=UTC), + valid_to=datetime(2023, 6, 25, 23, 59, 59, tzinfo=UTC), + ), OrthophotoProduct( key="brussels_latest", display_name="Meest recente orthofoto Brussel", diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py new file mode 100644 index 00000000..b12a7d11 --- /dev/null +++ b/backend/tests/test_belgium_training_loop.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +SCRIPT = Path(__file__).parents[2] / "scripts" / "run_belgium_building_training_loop.py" +SPEC = importlib.util.spec_from_file_location("training_loop", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_path: Path) -> None: + command = MODULE.training_command( + "yolo", + model=tmp_path / "base.pt", + data=tmp_path / "dataset.yaml", + project=tmp_path / "runs", + name="iteration-001", + epochs=160, + seed=42, + batch=2, + workers=4, + ) + assert command[:2] == ["yolo", "train"] + assert "device=0" in command + assert "deterministic=True" in command + assert "seed=42" in command + assert "epochs=160" in command + assert f"data={tmp_path / 'dataset.yaml'}" in command diff --git a/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py b/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py index 288c7e83..d21357ce 100644 --- a/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py +++ b/backend/tests/test_sprint146_operator_yolo_dataset_quality_audit.py @@ -65,6 +65,7 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat "label_count": 2, "is_negative": False, "is_repeated_background_negative": False, + "low_visual_variance": True, }, { "sample_slug": "postel_bos", @@ -144,6 +145,7 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat assert report["positive_sample_count"] == 2 assert report["background_sample_count"] == 1 assert report["train_negative_tile_count"] == 2 + assert report["low_variance_positive_tile_count"] == 1 assert report["repeated_background_negative_tile_count"] == 1 assert report["label_stats"]["parsed_label_count"] == 3 assert report["label_stats"]["invalid_label_count"] == 0 @@ -169,6 +171,7 @@ def test_operator_yolo_dataset_quality_audit_reports_dataset_risks(tmp_path: Pat assert "repeated_background_negative_share_above_gate" in warning_codes assert "median_box_area_below_gate" in warning_codes assert "small_box_share_above_gate" in warning_codes + assert "positive_tiles_have_low_visual_variance" in warning_codes markdown = (output_dir / "operator_yolo_dataset_quality_audit.md").read_text(encoding="utf-8") assert "Operator YOLO Dataset Quality Audit" in markdown diff --git a/backend/tests/test_sprint196_map_orthophoto_analysis.py b/backend/tests/test_sprint196_map_orthophoto_analysis.py index ede5a747..d4bec672 100644 --- a/backend/tests/test_sprint196_map_orthophoto_analysis.py +++ b/backend/tests/test_sprint196_map_orthophoto_analysis.py @@ -169,7 +169,7 @@ def test_orthophoto_product_registry_exposes_only_governed_official_layers() -> assert {"2025", "2012", "2008_2011", "2000_2003", "1979_1990", "1971"}.issubset(keys) assert next(item for item in products if item["key"] == "most_recent")["supports_detection"] is True detection_keys = {item["key"] for item in products if item["supports_detection"]} - assert {"most_recent", "wallonia_latest", "wallonia_2024", "brussels_latest", "brussels_2025", "2025"} <= detection_keys + assert {"most_recent", "wallonia_latest", "wallonia_2024", "wallonia_2023", "brussels_latest", "brussels_2025", "2025"} <= detection_keys by_key = {item["key"]: item for item in products} assert by_key["wallonia_latest"]["provider"] == "spw_orthophoto" assert by_key["wallonia_latest"]["coverage_zone"] == "wallonia" diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index d607999b..cd3dc9fd 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -126,6 +126,7 @@ COPY scripts/provision_belgium_building_training_portfolio.py /app/scripts/provi COPY scripts/audit_belgium_building_corpus.py /app/scripts/audit_belgium_building_corpus.py COPY scripts/evaluate_belgium_building_candidate.py /app/scripts/evaluate_belgium_building_candidate.py COPY scripts/assess_belgium_building_training_iteration.py /app/scripts/assess_belgium_building_training_iteration.py +COPY scripts/run_belgium_building_training_loop.py /app/scripts/run_belgium_building_training_loop.py COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py COPY scripts/train_operator_yolo_detector.sh /app/scripts/train_operator_yolo_detector.sh diff --git a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md index 31b4e01c..11a6a76b 100644 --- a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md +++ b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md @@ -74,6 +74,8 @@ The active production model remains unchanged while any gate fails. - per-AOI evaluator: `scripts/evaluate_belgium_building_candidate.py`; - calibration-only selection and release gates: `scripts/assess_belgium_building_training_iteration.py`. +- checkpointed CUDA orchestration: + `scripts/run_belgium_building_training_loop.py`. Every failed assessment returns `continue_training_loop`. Only a report with `training_complete` may proceed to final human review and guarded activation. diff --git a/scripts/audit_operator_yolo_dataset_quality.py b/scripts/audit_operator_yolo_dataset_quality.py index 218e9c71..b8b0db5e 100644 --- a/scripts/audit_operator_yolo_dataset_quality.py +++ b/scripts/audit_operator_yolo_dataset_quality.py @@ -4,7 +4,7 @@ from __future__ import annotations import argparse import json import statistics -from collections import Counter, defaultdict +from collections import Counter from pathlib import Path from typing import Any @@ -246,6 +246,7 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name ] positive_tiles = [tile for tile in tiles if tile not in negative_tiles] train_negative_tiles = [tile for tile in negative_tiles if tile.get("split") == "train"] + low_variance_positive_tiles = [tile for tile in positive_tiles if tile.get("low_visual_variance")] val_positive_samples = { str(tile.get("sample_slug")) for tile in positive_tiles @@ -288,6 +289,15 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name f"gate is {args.max_repeated_negative_share:.3f}." ), ) + if low_variance_positive_tiles: + add_warning( + warnings, + "positive_tiles_have_low_visual_variance", + ( + f"{len(low_variance_positive_tiles)} positive tiles are visually blank/low-variance; " + "the imagery source does not support their labels." + ), + ) if label_stats["missing_label_file_count"]: add_warning( warnings, @@ -337,6 +347,7 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name "train_tile_count": split_counts.get("train", 0), "val_tile_count": split_counts.get("val", 0), "train_negative_tile_count": len(train_negative_tiles), + "low_variance_positive_tile_count": len(low_variance_positive_tiles), "repeated_background_negative_tile_count": repeated_negative_count, "repeated_background_negative_share_of_negatives": repeated_negative_share, "positive_tile_share": len(positive_tiles) / len(tiles) if tiles else 0.0, @@ -371,6 +382,10 @@ def build_recommendations(warnings: list[dict[str, str]]) -> list[str]: ) if "label_files_missing" in codes or "invalid_label_rows" in codes: recommendations.append("Regenerate the YOLO tile dataset and review exporter path/label integrity.") + if "positive_tiles_have_low_visual_variance" in codes: + recommendations.append( + "Reject the raster product for affected AOIs or replace it with an officially complete imagery edition before training." + ) if not recommendations: recommendations.append("Dataset audit passed the configured gates; continue with benchmarked training.") return recommendations @@ -389,6 +404,7 @@ def write_markdown(report: dict[str, Any], path: Path) -> None: f"- Samples: {report['sample_count']} ({report['positive_sample_count']} positive, {report['background_sample_count']} background)", f"- Repeated background negative share: {report['repeated_background_negative_share_of_negatives']:.3f}", f"- Minimum visible label ratio: {format_optional_float(report.get('min_label_visible_ratio'))}", + f"- Blank/low-variance positive tiles: {report['low_variance_positive_tile_count']}", "", "## Label Quality", "", diff --git a/scripts/provision_belgium_building_training_portfolio.py b/scripts/provision_belgium_building_training_portfolio.py index 6759a923..f4502e47 100644 --- a/scripts/provision_belgium_building_training_portfolio.py +++ b/scripts/provision_belgium_building_training_portfolio.py @@ -115,7 +115,7 @@ REGION_CONTRACT = { }, "wallonia": { "area_id": "e5fd742a-ca18-4520-abe0-d28416aa2ece", - "orthophoto_product": "wallonia_2024", + "orthophoto_product": "wallonia_2023", "reference_path": "datasets/official-vector/acquire", "reference_product": "spw_picc_buildings", }, diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py new file mode 100644 index 00000000..a1d7427f --- /dev/null +++ b/scripts/run_belgium_building_training_loop.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Run checkpointed CUDA train/evaluate iterations until gates pass or a batch yields.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import subprocess +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(value, indent=2), encoding="utf-8") + temporary.replace(path) + + +def training_command( + yolo: str, + *, + model: Path, + data: Path, + project: Path, + name: str, + epochs: int, + seed: int, + batch: int, + workers: int, +) -> list[str]: + return [ + yolo, + "train", + f"model={model}", + f"data={data}", + f"epochs={epochs}", + "imgsz=640", + f"batch={batch}", + "device=0", + f"workers={workers}", + "patience=35", + "cache=disk", + "close_mosaic=20", + f"seed={seed}", + "deterministic=True", + f"project={project}", + f"name={name}", + "exist_ok=True", + ] + + +def run(command: list[str], log_path: Path | None = None, *, allowed: set[int] = {0}) -> int: + if log_path: + log_path.parent.mkdir(parents=True, exist_ok=True) + with log_path.open("a", encoding="utf-8") as log: + completed = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, check=False) + else: + completed = subprocess.run(command, check=False) + if completed.returncode not in allowed: + raise RuntimeError(f"Command failed ({completed.returncode}): {' '.join(command)}") + return completed.returncode + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--initial-model", type=Path, required=True) + parser.add_argument("--train-yaml", type=Path, required=True) + parser.add_argument("--calibration-summary", type=Path, required=True) + parser.add_argument("--test-summary", type=Path, required=True) + parser.add_argument("--background-summary", type=Path, required=True) + parser.add_argument("--corpus-manifest", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--iterations", type=int, default=1) + parser.add_argument("--epochs", type=int, default=160) + parser.add_argument("--batch", type=int, default=2) + parser.add_argument("--workers", type=int, default=4) + parser.add_argument("--seed", type=int, default=20260731) + parser.add_argument("--yolo", default="yolo") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + if args.iterations < 1: + raise SystemExit("--iterations must be positive") + + state_path = args.output_dir / "training-loop-state.json" + state: dict[str, Any] = { + "schema_version": 1, + "status": "running", + "started_at": datetime.now(UTC).isoformat(), + "initial_model": str(args.initial_model), + "train_yaml": str(args.train_yaml), + "corpus_manifest": str(args.corpus_manifest), + "iterations": [], + } + if state_path.is_file(): + state = json.loads(state_path.read_text(encoding="utf-8")) + state["status"] = "running" + model = Path(state.get("next_model") or args.initial_model) + first_index = len(state["iterations"]) + 1 + scripts_dir = Path(__file__).resolve().parent + + for offset in range(args.iterations): + index = first_index + offset + name = f"iteration-{index:03d}" + iteration_dir = args.output_dir / name + train_run = args.output_dir / "runs" / name + command = training_command( + args.yolo, + model=model, + data=args.train_yaml, + project=args.output_dir / "runs", + name=name, + epochs=args.epochs, + seed=args.seed + index, + batch=args.batch, + workers=args.workers, + ) + if args.dry_run: + print(json.dumps({"training_command": command}, indent=2)) + return 0 + run(command, iteration_dir / "training.log") + best = train_run / "weights" / "best.pt" + if not best.is_file(): + raise RuntimeError(f"Training produced no best checkpoint: {best}") + candidate = iteration_dir / "candidate.pt" + shutil.copy2(best, candidate) + + reports: dict[str, Path] = {} + for role, summary in ( + ("calibration", args.calibration_summary), + ("test", args.test_summary), + ("background", args.background_summary), + ): + report = iteration_dir / f"{role}.json" + reports[role] = report + run( + [ + sys.executable, + str(scripts_dir / "evaluate_belgium_building_candidate.py"), + "--model", + str(candidate), + "--summary", + str(summary), + "--corpus-manifest", + str(args.corpus_manifest), + "--output", + str(report), + "--device", + "cuda:0", + ], + iteration_dir / f"{role}.log", + ) + assessment = iteration_dir / "assessment.json" + run( + [ + sys.executable, + str(scripts_dir / "assess_belgium_building_training_iteration.py"), + "--calibration", + str(reports["calibration"]), + "--test", + str(reports["test"]), + "--background", + str(reports["background"]), + "--output", + str(assessment), + ], + iteration_dir / "assessment.log", + allowed={0, 2}, + ) + decision = json.loads(assessment.read_text(encoding="utf-8")) + record = { + "iteration": index, + "candidate": str(candidate), + "candidate_sha256": sha256(candidate), + "assessment": str(assessment), + "status": decision["status"], + "failures": decision["failures"], + } + state["iterations"].append(record) + state["next_model"] = str(candidate) + if decision["status"] == "training_complete": + state["status"] = "training_complete" + state["completed_at"] = datetime.now(UTC).isoformat() + write_json(state_path, state) + print(json.dumps(state, indent=2)) + return 0 + model = candidate + write_json(state_path, state) + + state["status"] = "continue_training_loop" + state["yielded_at"] = datetime.now(UTC).isoformat() + write_json(state_path, state) + print(json.dumps(state, indent=2)) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main())