From 7744803461282eb72499527a2d8fe859aa6521fe Mon Sep 17 00:00:00 2001 From: Jens Date: Mon, 27 Jul 2026 04:04:45 +0200 Subject: [PATCH] Add fail-closed failure-driven YOLO sampling --- .../test_failure_driven_yolo_sampling.py | 54 +++++++ deploy/unraid/Dockerfile.all-in-one | 1 + docs/BELGIUM_BUILDING_TRAINING_LOOP.md | 8 + scripts/build_failure_driven_yolo_sampling.py | 152 ++++++++++++++++++ 4 files changed, 215 insertions(+) create mode 100644 backend/tests/test_failure_driven_yolo_sampling.py create mode 100644 scripts/build_failure_driven_yolo_sampling.py diff --git a/backend/tests/test_failure_driven_yolo_sampling.py b/backend/tests/test_failure_driven_yolo_sampling.py new file mode 100644 index 00000000..4afadfeb --- /dev/null +++ b/backend/tests/test_failure_driven_yolo_sampling.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +SCRIPT = Path(__file__).parents[2] / "scripts" / "build_failure_driven_yolo_sampling.py" +SPEC = importlib.util.spec_from_file_location("failure_sampling", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_sampling_repeats_only_failed_region_train_tiles() -> None: + manifest = { + "samples": [ + {"sample_slug": "train-fl", "split": "train", "region": "flanders"}, + {"sample_slug": "train-wa", "split": "train", "region": "wallonia"}, + {"sample_slug": "test-fl", "split": "test", "region": "flanders"}, + ] + } + summary = { + "tiles": [ + {"sample_slug": "train-fl", "split": "train", "label_count": 2, "image_path": "/tmp/fl-pos.png"}, + {"sample_slug": "train-fl", "split": "train", "label_count": 0, "image_path": "/tmp/fl-neg.png"}, + {"sample_slug": "train-wa", "split": "train", "label_count": 1, "image_path": "/tmp/wa-pos.png"}, + {"sample_slug": "test-fl", "split": "val", "label_count": 1, "image_path": "/tmp/protected.png"}, + ] + } + assessment = { + "status": "continue_training_loop", + "gates": { + "min_region_f1": 0.45, + "min_region_precision": 0.5, + "min_region_recall": 0.4, + "max_pure_empty_false_positives": 0, + }, + "test": { + "regions": { + "flanders": {"f1": 0.2, "precision": 0.3, "recall": 0.2}, + "wallonia": {"f1": 0.6, "precision": 0.6, "recall": 0.6}, + } + }, + "background": {"pure_empty_false_positives": 2}, + } + paths, metadata = MODULE.build_sampling( + summary=summary, manifest=manifest, assessment=assessment + ) + assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 3 + assert paths.count(str(Path("/tmp/fl-neg.png").resolve())) == 4 + assert paths.count(str(Path("/tmp/wa-pos.png").resolve())) == 1 + assert not any("protected" in path for path in paths) + assert metadata["protected_samples_in_training"] == [] + assert metadata["weak_recall_regions"] == ["flanders"] diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index cd3dc9fd..e8513eef 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -127,6 +127,7 @@ COPY scripts/audit_belgium_building_corpus.py /app/scripts/audit_belgium_buildin COPY scripts/evaluate_belgium_building_candidate.py /app/scripts/evaluate_belgium_building_candidate.py COPY scripts/assess_belgium_building_training_iteration.py /app/scripts/assess_belgium_building_training_iteration.py COPY scripts/run_belgium_building_training_loop.py /app/scripts/run_belgium_building_training_loop.py +COPY scripts/build_failure_driven_yolo_sampling.py /app/scripts/build_failure_driven_yolo_sampling.py COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py COPY scripts/train_operator_yolo_detector.sh /app/scripts/train_operator_yolo_detector.sh diff --git a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md index 3b3ae70c..46ace8a2 100644 --- a/docs/BELGIUM_BUILDING_TRAINING_LOOP.md +++ b/docs/BELGIUM_BUILDING_TRAINING_LOOP.md @@ -79,6 +79,14 @@ The active production model remains unchanged while any gate fails. Every failed assessment returns `continue_training_loop`. Only a report with `training_complete` may proceed to final human review and guarded activation. + +After a failed assessment, +`scripts/build_failure_driven_yolo_sampling.py` creates a checksummed, +train-only sampling manifest. Positive tiles from regions that fail F1 or +recall are repeated, while true negative train tiles are repeated when a +regional precision gate or the pure-background gate fails. Calibration, test, +background-test and validation AOIs are excluded by their frozen corpus split; +the generated evidence records that no protected sample entered training. The orchestrator refuses to start unless the frozen dataset audit is `ok` and contains zero blank/low-variance positive tiles. For dated imagery, GRB `BEGINDATUM` and PICC `DATE_CREAT` are compared with the diff --git a/scripts/build_failure_driven_yolo_sampling.py b/scripts/build_failure_driven_yolo_sampling.py new file mode 100644 index 00000000..a9519342 --- /dev/null +++ b/scripts/build_failure_driven_yolo_sampling.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Build a leak-free YOLO sampling manifest from failed release gates.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections import Counter +from pathlib import Path +from typing import Any + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def build_sampling( + *, + summary: dict[str, Any], + manifest: dict[str, Any], + assessment: dict[str, Any], + positive_repeat: int = 3, + negative_repeat: int = 4, +) -> tuple[list[str], dict[str, Any]]: + if assessment.get("status") != "continue_training_loop": + raise ValueError("Failure-driven sampling requires a failed assessment") + if positive_repeat < 1 or negative_repeat < 1: + raise ValueError("Repeat factors must be positive") + + samples = {item["sample_slug"]: item for item in manifest["samples"]} + gates = assessment["gates"] + regions = assessment["test"]["regions"] + weak_recall_regions = { + region + for region, metrics in regions.items() + if metrics["f1"] < gates["min_region_f1"] + or metrics["recall"] < gates["min_region_recall"] + } + weak_precision_regions = { + region + for region, metrics in regions.items() + if metrics["precision"] < gates["min_region_precision"] + } + background_failed = ( + assessment["background"]["pure_empty_false_positives"] + > gates["max_pure_empty_false_positives"] + ) + + image_paths: list[str] = [] + repeat_counts: Counter[str] = Counter() + selected_samples: set[str] = set() + protected_samples: set[str] = set() + for tile in summary["tiles"]: + sample = samples[tile["sample_slug"]] + if sample["split"] != "train" or tile["split"] != "train": + protected_samples.add(tile["sample_slug"]) + continue + region = sample["region"] + repeat = 1 + if tile["label_count"] > 0 and region in weak_recall_regions: + repeat = positive_repeat + if tile["label_count"] == 0 and (background_failed or region in weak_precision_regions): + repeat = negative_repeat + path = str(Path(tile["image_path"]).resolve()) + image_paths.extend([path] * repeat) + repeat_counts[region] += repeat + selected_samples.add(tile["sample_slug"]) + + if not image_paths: + raise ValueError("No train-only tiles selected") + metadata = { + "schema_version": 1, + "status": "ok", + "strategy": "failed-region-positive-and-hard-negative-repeat", + "weak_recall_regions": sorted(weak_recall_regions), + "weak_precision_regions": sorted(weak_precision_regions), + "background_gate_failed": background_failed, + "positive_repeat": positive_repeat, + "negative_repeat": negative_repeat, + "source_train_tile_count": sum( + 1 + for tile in summary["tiles"] + if samples[tile["sample_slug"]]["split"] == "train" and tile["split"] == "train" + ), + "sampled_train_entry_count": len(image_paths), + "sampled_entries_by_region": dict(sorted(repeat_counts.items())), + "selected_train_sample_count": len(selected_samples), + "protected_sample_count": len(protected_samples), + "protected_samples_in_training": [], + } + return image_paths, metadata + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--corpus-manifest", type=Path, required=True) + parser.add_argument("--assessment", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--positive-repeat", type=int, default=3) + parser.add_argument("--negative-repeat", type=int, default=4) + args = parser.parse_args() + + summary = json.loads(args.summary.read_text(encoding="utf-8")) + manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8")) + assessment = json.loads(args.assessment.read_text(encoding="utf-8")) + paths, metadata = build_sampling( + summary=summary, + manifest=manifest, + assessment=assessment, + positive_repeat=args.positive_repeat, + negative_repeat=args.negative_repeat, + ) + args.output_dir.mkdir(parents=True, exist_ok=True) + train_list = args.output_dir / "train-failure-driven.txt" + train_list.write_text("\n".join(paths) + "\n", encoding="utf-8") + source_yaml = args.summary.parent / "dataset.yaml" + val_dir = args.summary.parent / "images" / "val" + dataset_yaml = args.output_dir / "dataset.yaml" + dataset_yaml.write_text( + f"path: {args.output_dir}\n" + f"train: {train_list}\n" + f"val: {val_dir}\n" + "names:\n 0: building\n", + encoding="utf-8", + ) + metadata.update( + { + "summary": str(args.summary), + "summary_sha256": file_sha256(args.summary), + "corpus_manifest": str(args.corpus_manifest), + "corpus_manifest_sha256": file_sha256(args.corpus_manifest), + "assessment": str(args.assessment), + "assessment_sha256": file_sha256(args.assessment), + "source_dataset_yaml": str(source_yaml), + "train_list": str(train_list), + "dataset_yaml": str(dataset_yaml), + } + ) + output = args.output_dir / "failure-driven-sampling.json" + output.write_text(json.dumps(metadata, indent=2), encoding="utf-8") + print(json.dumps(metadata, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())