Add fail-closed failure-driven YOLO sampling

backend/tests/test_failure_driven_yolo_sampling.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).parents[2] / "scripts" / "build_failure_driven_yolo_sampling.py"
+SPEC = importlib.util.spec_from_file_location("failure_sampling", SCRIPT)
+assert SPEC and SPEC.loader
+MODULE = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MODULE)
+
+
+def test_sampling_repeats_only_failed_region_train_tiles() -> None:
+    manifest = {
+        "samples": [
+            {"sample_slug": "train-fl", "split": "train", "region": "flanders"},
+            {"sample_slug": "train-wa", "split": "train", "region": "wallonia"},
+            {"sample_slug": "test-fl", "split": "test", "region": "flanders"},
+        ]
+    }
+    summary = {
+        "tiles": [
+            {"sample_slug": "train-fl", "split": "train", "label_count": 2, "image_path": "/tmp/fl-pos.png"},
+            {"sample_slug": "train-fl", "split": "train", "label_count": 0, "image_path": "/tmp/fl-neg.png"},
+            {"sample_slug": "train-wa", "split": "train", "label_count": 1, "image_path": "/tmp/wa-pos.png"},
+            {"sample_slug": "test-fl", "split": "val", "label_count": 1, "image_path": "/tmp/protected.png"},
+        ]
+    }
+    assessment = {
+        "status": "continue_training_loop",
+        "gates": {
+            "min_region_f1": 0.45,
+            "min_region_precision": 0.5,
+            "min_region_recall": 0.4,
+            "max_pure_empty_false_positives": 0,
+        },
+        "test": {
+            "regions": {
+                "flanders": {"f1": 0.2, "precision": 0.3, "recall": 0.2},
+                "wallonia": {"f1": 0.6, "precision": 0.6, "recall": 0.6},
+            }
+        },
+        "background": {"pure_empty_false_positives": 2},
+    }
+    paths, metadata = MODULE.build_sampling(
+        summary=summary, manifest=manifest, assessment=assessment
+    )
+    assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 3
+    assert paths.count(str(Path("/tmp/fl-neg.png").resolve())) == 4
+    assert paths.count(str(Path("/tmp/wa-pos.png").resolve())) == 1
+    assert not any("protected" in path for path in paths)
+    assert metadata["protected_samples_in_training"] == []
+    assert metadata["weak_recall_regions"] == ["flanders"]

deploy/unraid/Dockerfile.all-in-one

@@ -127,6 +127,7 @@ COPY scripts/audit_belgium_building_corpus.py /app/scripts/audit_belgium_buildin
 COPY scripts/evaluate_belgium_building_candidate.py /app/scripts/evaluate_belgium_building_candidate.py
 COPY scripts/assess_belgium_building_training_iteration.py /app/scripts/assess_belgium_building_training_iteration.py
 COPY scripts/run_belgium_building_training_loop.py /app/scripts/run_belgium_building_training_loop.py
+COPY scripts/build_failure_driven_yolo_sampling.py /app/scripts/build_failure_driven_yolo_sampling.py
 COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
 COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
 COPY scripts/train_operator_yolo_detector.sh /app/scripts/train_operator_yolo_detector.sh

docs/BELGIUM_BUILDING_TRAINING_LOOP.md

@@ -79,6 +79,14 @@ The active production model remains unchanged while any gate fails.
 
 Every failed assessment returns `continue_training_loop`. Only a report with
 `training_complete` may proceed to final human review and guarded activation.
+
+After a failed assessment,
+`scripts/build_failure_driven_yolo_sampling.py` creates a checksummed,
+train-only sampling manifest. Positive tiles from regions that fail F1 or
+recall are repeated, while true negative train tiles are repeated when a
+regional precision gate or the pure-background gate fails. Calibration, test,
+background-test and validation AOIs are excluded by their frozen corpus split;
+the generated evidence records that no protected sample entered training.
 The orchestrator refuses to start unless the frozen dataset audit is `ok` and
 contains zero blank/low-variance positive tiles.
 For dated imagery, GRB `BEGINDATUM` and PICC `DATE_CREAT` are compared with the

scripts/build_failure_driven_yolo_sampling.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Build a leak-free YOLO sampling manifest from failed release gates."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+
+def file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def build_sampling(
+    *,
+    summary: dict[str, Any],
+    manifest: dict[str, Any],
+    assessment: dict[str, Any],
+    positive_repeat: int = 3,
+    negative_repeat: int = 4,
+) -> tuple[list[str], dict[str, Any]]:
+    if assessment.get("status") != "continue_training_loop":
+        raise ValueError("Failure-driven sampling requires a failed assessment")
+    if positive_repeat < 1 or negative_repeat < 1:
+        raise ValueError("Repeat factors must be positive")
+
+    samples = {item["sample_slug"]: item for item in manifest["samples"]}
+    gates = assessment["gates"]
+    regions = assessment["test"]["regions"]
+    weak_recall_regions = {
+        region
+        for region, metrics in regions.items()
+        if metrics["f1"] < gates["min_region_f1"]
+        or metrics["recall"] < gates["min_region_recall"]
+    }
+    weak_precision_regions = {
+        region
+        for region, metrics in regions.items()
+        if metrics["precision"] < gates["min_region_precision"]
+    }
+    background_failed = (
+        assessment["background"]["pure_empty_false_positives"]
+        > gates["max_pure_empty_false_positives"]
+    )
+
+    image_paths: list[str] = []
+    repeat_counts: Counter[str] = Counter()
+    selected_samples: set[str] = set()
+    protected_samples: set[str] = set()
+    for tile in summary["tiles"]:
+        sample = samples[tile["sample_slug"]]
+        if sample["split"] != "train" or tile["split"] != "train":
+            protected_samples.add(tile["sample_slug"])
+            continue
+        region = sample["region"]
+        repeat = 1
+        if tile["label_count"] > 0 and region in weak_recall_regions:
+            repeat = positive_repeat
+        if tile["label_count"] == 0 and (background_failed or region in weak_precision_regions):
+            repeat = negative_repeat
+        path = str(Path(tile["image_path"]).resolve())
+        image_paths.extend([path] * repeat)
+        repeat_counts[region] += repeat
+        selected_samples.add(tile["sample_slug"])
+
+    if not image_paths:
+        raise ValueError("No train-only tiles selected")
+    metadata = {
+        "schema_version": 1,
+        "status": "ok",
+        "strategy": "failed-region-positive-and-hard-negative-repeat",
+        "weak_recall_regions": sorted(weak_recall_regions),
+        "weak_precision_regions": sorted(weak_precision_regions),
+        "background_gate_failed": background_failed,
+        "positive_repeat": positive_repeat,
+        "negative_repeat": negative_repeat,
+        "source_train_tile_count": sum(
+            1
+            for tile in summary["tiles"]
+            if samples[tile["sample_slug"]]["split"] == "train" and tile["split"] == "train"
+        ),
+        "sampled_train_entry_count": len(image_paths),
+        "sampled_entries_by_region": dict(sorted(repeat_counts.items())),
+        "selected_train_sample_count": len(selected_samples),
+        "protected_sample_count": len(protected_samples),
+        "protected_samples_in_training": [],
+    }
+    return image_paths, metadata
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--summary", type=Path, required=True)
+    parser.add_argument("--corpus-manifest", type=Path, required=True)
+    parser.add_argument("--assessment", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--positive-repeat", type=int, default=3)
+    parser.add_argument("--negative-repeat", type=int, default=4)
+    args = parser.parse_args()
+
+    summary = json.loads(args.summary.read_text(encoding="utf-8"))
+    manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
+    assessment = json.loads(args.assessment.read_text(encoding="utf-8"))
+    paths, metadata = build_sampling(
+        summary=summary,
+        manifest=manifest,
+        assessment=assessment,
+        positive_repeat=args.positive_repeat,
+        negative_repeat=args.negative_repeat,
+    )
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    train_list = args.output_dir / "train-failure-driven.txt"
+    train_list.write_text("\n".join(paths) + "\n", encoding="utf-8")
+    source_yaml = args.summary.parent / "dataset.yaml"
+    val_dir = args.summary.parent / "images" / "val"
+    dataset_yaml = args.output_dir / "dataset.yaml"
+    dataset_yaml.write_text(
+        f"path: {args.output_dir}\n"
+        f"train: {train_list}\n"
+        f"val: {val_dir}\n"
+        "names:\n  0: building\n",
+        encoding="utf-8",
+    )
+    metadata.update(
+        {
+            "summary": str(args.summary),
+            "summary_sha256": file_sha256(args.summary),
+            "corpus_manifest": str(args.corpus_manifest),
+            "corpus_manifest_sha256": file_sha256(args.corpus_manifest),
+            "assessment": str(args.assessment),
+            "assessment_sha256": file_sha256(args.assessment),
+            "source_dataset_yaml": str(source_yaml),
+            "train_list": str(train_list),
+            "dataset_yaml": str(dataset_yaml),
+        }
+    )
+    output = args.output_dir / "failure-driven-sampling.json"
+    output.write_text(json.dumps(metadata, indent=2), encoding="utf-8")
+    print(json.dumps(metadata, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())