Add fail-closed Belgian training loop

scripts/assess_belgium_building_training_iteration.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""Select on calibration evidence and apply fixed, fail-closed release gates."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+
+def load(path: Path) -> dict[str, Any]:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def find_threshold(report: dict[str, Any], threshold: float) -> dict[str, Any]:
+    for item in report["sweeps"]:
+        if abs(float(item["threshold"]) - threshold) < 1e-9:
+            return item
+    raise ValueError(f"Threshold {threshold} is absent from {report.get('summary')}")
+
+
+def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
+    eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
+    if not eligible:
+        eligible = report["sweeps"]
+    return max(
+        eligible,
+        key=lambda item: (
+            min(region["f1"] for region in item["regions"].values()),
+            item["aggregate"]["f1"],
+            -item["pure_empty_false_positives"],
+        ),
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--calibration", type=Path, required=True)
+    parser.add_argument("--test", type=Path, required=True)
+    parser.add_argument("--background", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--min-aggregate-f1", type=float, default=0.55)
+    parser.add_argument("--min-region-f1", type=float, default=0.45)
+    parser.add_argument("--min-region-precision", type=float, default=0.5)
+    parser.add_argument("--min-region-recall", type=float, default=0.4)
+    parser.add_argument("--max-pure-empty-fp", type=int, default=0)
+    args = parser.parse_args()
+
+    calibration = load(args.calibration)
+    chosen = select_calibration_threshold(calibration)
+    threshold = float(chosen["threshold"])
+    test = find_threshold(load(args.test), threshold)
+    background = find_threshold(load(args.background), threshold)
+    failures: list[str] = []
+    if test["aggregate"]["f1"] < args.min_aggregate_f1:
+        failures.append("test_aggregate_f1_below_gate")
+    for region, values in test["regions"].items():
+        if values["f1"] < args.min_region_f1:
+            failures.append(f"test_{region}_f1_below_gate")
+        if values["precision"] < args.min_region_precision:
+            failures.append(f"test_{region}_precision_below_gate")
+        if values["recall"] < args.min_region_recall:
+            failures.append(f"test_{region}_recall_below_gate")
+    if background["pure_empty_false_positives"] > args.max_pure_empty_fp:
+        failures.append("pure_empty_false_positive_gate_failed")
+    payload = {
+        "schema_version": 1,
+        "status": "training_complete" if not failures else "continue_training_loop",
+        "threshold_selection_source": "calibration_only",
+        "selected_threshold": threshold,
+        "gates": {
+            "min_aggregate_f1": args.min_aggregate_f1,
+            "min_region_f1": args.min_region_f1,
+            "min_region_precision": args.min_region_precision,
+            "min_region_recall": args.min_region_recall,
+            "max_pure_empty_false_positives": args.max_pure_empty_fp,
+        },
+        "calibration": chosen,
+        "test": test,
+        "background": background,
+        "failures": failures,
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    print(json.dumps(payload, indent=2))
+    return 0 if not failures else 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/evaluate_belgium_building_candidate.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Evaluate one building detector without using protected data for tuning."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import defaultdict
+from pathlib import Path
+from typing import Any
+
+
+def iou(left: tuple[float, float, float, float], right: tuple[float, float, float, float]) -> float:
+    x1, y1 = max(left[0], right[0]), max(left[1], right[1])
+    x2, y2 = min(left[2], right[2]), min(left[3], right[3])
+    intersection = max(0.0, x2 - x1) * max(0.0, y2 - y1)
+    union = (left[2] - left[0]) * (left[3] - left[1]) + (right[2] - right[0]) * (
+        right[3] - right[1]
+    ) - intersection
+    return intersection / union if union > 0 else 0.0
+
+
+def match_boxes(
+    predictions: list[tuple[tuple[float, float, float, float], float]],
+    references: list[tuple[float, float, float, float]],
+    *,
+    confidence: float,
+    match_iou: float,
+) -> tuple[int, int, int]:
+    unmatched = set(range(len(references)))
+    true_positive = 0
+    considered = sorted((item for item in predictions if item[1] >= confidence), key=lambda item: -item[1])
+    for box, _score in considered:
+        candidates = [(iou(box, references[index]), index) for index in unmatched]
+        best_iou, best_index = max(candidates, default=(0.0, -1))
+        if best_iou >= match_iou:
+            unmatched.remove(best_index)
+            true_positive += 1
+    return true_positive, len(considered) - true_positive, len(unmatched)
+
+
+def metrics(tp: int, fp: int, fn: int) -> dict[str, float | int]:
+    precision = tp / (tp + fp) if tp + fp else 1.0
+    recall = tp / (tp + fn) if tp + fn else 1.0
+    return {
+        "true_positive": tp,
+        "false_positive": fp,
+        "false_negative": fn,
+        "precision": precision,
+        "recall": recall,
+        "f1": 2 * precision * recall / (precision + recall) if precision + recall else 0.0,
+    }
+
+
+def read_references(path: Path, width: int, height: int) -> list[tuple[float, float, float, float]]:
+    boxes = []
+    for line in path.read_text(encoding="utf-8").splitlines() if path.is_file() else []:
+        parts = line.split()
+        if len(parts) != 5:
+            raise ValueError(f"Invalid YOLO label row in {path}: {line}")
+        _class_id, cx, cy, box_width, box_height = map(float, parts)
+        boxes.append(
+            (
+                (cx - box_width / 2) * width,
+                (cy - box_height / 2) * height,
+                (cx + box_width / 2) * width,
+                (cy + box_height / 2) * height,
+            )
+        )
+    return boxes
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", type=Path, required=True)
+    parser.add_argument("--summary", type=Path, required=True)
+    parser.add_argument("--corpus-manifest", type=Path, required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    parser.add_argument("--thresholds", type=float, nargs="+", default=[0.1, 0.15, 0.2, 0.25, 0.3, 0.4])
+    parser.add_argument("--match-iou", type=float, default=0.25)
+    parser.add_argument("--device", default="cuda:0")
+    parser.add_argument("--split", default="val")
+    args = parser.parse_args()
+
+    from ultralytics import YOLO
+
+    summary = json.loads(args.summary.read_text(encoding="utf-8"))
+    manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
+    regions = {item["sample_slug"]: item["region"] for item in manifest["samples"]}
+    pure_empty_slugs = {
+        item["sample_slug"] for item in manifest["samples"] if bool(item.get("require_empty"))
+    }
+    tiles = [item for item in summary["tiles"] if item.get("kept", True) and item["split"] == args.split]
+    image_paths = [item["image_path"] for item in tiles]
+    results = YOLO(str(args.model)).predict(image_paths, conf=min(args.thresholds), device=args.device, verbose=False)
+    observations: list[dict[str, Any]] = []
+    for tile, result in zip(tiles, results, strict=True):
+        height, width = result.orig_shape
+        predictions = [
+            (tuple(map(float, box)), float(score))
+            for box, score in zip(result.boxes.xyxy.cpu().tolist(), result.boxes.conf.cpu().tolist(), strict=True)
+        ]
+        observations.append(
+            {
+                "sample_slug": tile["sample_slug"],
+                "region": regions[tile["sample_slug"]],
+                "references": read_references(Path(tile["label_path"]), width, height),
+                "predictions": predictions,
+            }
+        )
+
+    sweeps = []
+    for threshold in args.thresholds:
+        totals: defaultdict[str, list[int]] = defaultdict(lambda: [0, 0, 0])
+        pure_empty_fp = 0
+        for item in observations:
+            tp, fp, fn = match_boxes(
+                item["predictions"], item["references"], confidence=threshold, match_iou=args.match_iou
+            )
+            for key in ("all", item["region"], item["sample_slug"]):
+                totals[key][0] += tp
+                totals[key][1] += fp
+                totals[key][2] += fn
+            if item["sample_slug"] in pure_empty_slugs:
+                pure_empty_fp += fp
+        sweeps.append(
+            {
+                "threshold": threshold,
+                "aggregate": metrics(*totals["all"]),
+                "regions": {region: metrics(*totals[region]) for region in sorted(set(regions.values()))},
+                "samples": {slug: metrics(*counts) for slug, counts in sorted(totals.items()) if slug not in {"all", *regions.values()}},
+                "pure_empty_false_positives": pure_empty_fp,
+            }
+        )
+    payload = {
+        "schema_version": 1,
+        "model": str(args.model),
+        "summary": str(args.summary),
+        "split": args.split,
+        "match_iou": args.match_iou,
+        "tile_count": len(tiles),
+        "sweeps": sweeps,
+    }
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    print(json.dumps(payload, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/provision_belgium_building_training_portfolio.py

@@ -33,12 +33,18 @@ AOIS = (
     Aoi("flanders-farms-train", "flanders", "rural-farms", "train", 4.850, 50.900),
     Aoi("kalmthout-heath-train-bg", "flanders", "heath-negative", "train", 4.450, 51.390, "background_candidate"),
     Aoi("limburg-forest-train-bg", "flanders", "forest-negative", "train", 5.550, 51.050, "background_candidate"),
+    Aoi("ostend-coastal-train", "flanders", "coastal-urban", "train", 2.920, 51.225),
+    Aoi("roeselare-industry-train", "flanders", "industrial", "train", 3.120, 50.945),
+    Aoi("dendermonde-suburban-train", "flanders", "suburban", "train", 4.100, 51.030),
+    Aoi("houthalen-forest-train-bg", "flanders", "forest-negative", "train", 5.380, 51.030, "background_candidate"),
     Aoi("bruges-val", "flanders", "historic-urban", "val", 3.224, 51.209),
     Aoi("turnhout-val", "flanders", "suburban", "val", 4.944, 51.322),
     Aoi("hasselt-cal", "flanders", "suburban", "calibration", 5.340, 50.930),
     Aoi("kortrijk-cal", "flanders", "urban-industrial", "calibration", 3.265, 50.828),
+    Aoi("waregem-cal", "flanders", "ribbon-development", "calibration", 3.430, 50.890),
     Aoi("leuven-test", "flanders", "urban", "test", 4.700, 50.880),
     Aoi("sint-niklaas-test", "flanders", "ribbon-development", "test", 4.143, 51.165),
+    Aoi("mechelen-test", "flanders", "mixed-urban", "test", 4.480, 51.030),
     Aoi("kempen-forest-bg", "flanders", "forest-heath", "background-test", 5.180, 51.300, "background_candidate", True),
     Aoi("antwerp-port-bg", "flanders", "port-hard-negative", "background-test", 4.380, 51.280, "background_candidate", True),
     # Wallonia.
@@ -48,12 +54,18 @@ AOIS = (
     Aoi("namur-residential-train", "wallonia", "residential", "train", 4.870, 50.470),
     Aoi("ardennes-forest-train-bg", "wallonia", "forest-negative", "train", 5.700, 50.200, "background_candidate"),
     Aoi("wallonia-quarry-train-hard", "wallonia", "quarry-hard-negative", "train", 5.130, 50.530, "background_candidate"),
+    Aoi("wavre-suburban-train", "wallonia", "suburban", "train", 4.610, 50.720),
+    Aoi("marche-smallcity-train", "wallonia", "small-city", "train", 5.340, 50.230),
+    Aoi("ath-rural-train", "wallonia", "rural-town", "train", 3.780, 50.630),
+    Aoi("condroz-field-train-bg", "wallonia", "farmland-negative", "train", 4.700, 50.300, "background_candidate"),
     Aoi("tournai-val", "wallonia", "historic-urban", "val", 3.389, 50.606),
     Aoi("arlon-val", "wallonia", "small-city", "val", 5.817, 49.683),
     Aoi("verviers-cal", "wallonia", "suburban", "calibration", 5.860, 50.590),
     Aoi("dinant-cal", "wallonia", "valley-town", "calibration", 4.912, 50.260),
+    Aoi("mouscron-cal", "wallonia", "mixed-urban", "calibration", 3.210, 50.740),
     Aoi("mons-test", "wallonia", "urban", "test", 3.950, 50.450),
     Aoi("bastogne-test", "wallonia", "rural-town", "test", 5.720, 50.000),
+    Aoi("ottignies-test", "wallonia", "suburban", "test", 4.570, 50.670),
     Aoi("wallonia-rural-bg", "wallonia", "open-rural", "background-test", 5.000, 50.300, "background_candidate", True),
     Aoi("ardennes-forest-hard", "wallonia", "forest-hard-negative", "background-test", 5.600, 50.100, "background_candidate"),
     # Brussels.
@@ -63,12 +75,18 @@ AOIS = (
     Aoi("schaerbeek-train", "brussels", "dense-residential", "train", 4.380, 50.865),
     Aoi("brussels-rail-train-hard", "brussels", "rail-hard-negative", "train", 4.345, 50.875, "background_candidate"),
     Aoi("brussels-park-train-hard", "brussels", "park-hard-negative", "train", 4.400, 50.820, "background_candidate"),
+    Aoi("haren-mixed-train", "brussels", "mixed-urban", "train", 4.420, 50.890),
+    Aoi("ixelles-dense-train", "brussels", "dense-urban", "train", 4.370, 50.830),
+    Aoi("forest-residential-train", "brussels", "residential", "train", 4.320, 50.810),
+    Aoi("woluwe-park-train-hard", "brussels", "park-hard-negative", "train", 4.440, 50.840, "background_candidate"),
     Aoi("woluwe-val", "brussels", "suburban", "val", 4.430, 50.845),
     Aoi("molenbeek-val", "brussels", "mixed-urban", "val", 4.325, 50.855),
     Aoi("brussels-park-cal", "brussels", "park-edge", "calibration", 4.380, 50.820),
     Aoi("brussels-canal-cal", "brussels", "canal-industry", "calibration", 4.340, 50.870),
+    Aoi("saint-gilles-cal", "brussels", "dense-urban", "calibration", 4.345, 50.825),
     Aoi("brussels-rail-test", "brussels", "rail-context", "test", 4.330, 50.840),
     Aoi("jette-test", "brussels", "residential-park", "test", 4.325, 50.880),
+    Aoi("auderghem-test", "brussels", "residential-forest-edge", "test", 4.430, 50.815),
     Aoi("sonian-forest-hard", "brussels", "forest-hard-negative", "background-test", 4.420, 50.790, "background_candidate"),
     Aoi("bois-cambre-hard", "brussels", "park-hard-negative", "background-test", 4.375, 50.795, "background_candidate"),
 )

scripts/render_operator_yolo_label_qa_contact_sheets.py

@@ -212,6 +212,8 @@ def draw_tile_card(
         top = max(header_height, y_center - height / 2)
         right = min(thumb_size - 1, x_center + width / 2)
         bottom = min(thumb_size + header_height - 1, y_center + height / 2)
+        if right < left or bottom < top:
+            continue
         draw.rectangle((left, top, right, bottom), outline=(255, 214, 10), width=3)
 
     return card
@@ -370,7 +372,7 @@ def write_markdown(report: dict[str, Any], output_dir: Path) -> None:
     if report["contact_sheets"]:
         for sheet in report["contact_sheets"]:
             lines.append(f"- `{sheet['path']}` ({sheet['tile_count']} tiles)")
-            lines.append(f"")
+            lines.append("")
             lines.append(f"![{sheet['path']}]({sheet['path']})")
             lines.append("")
     else: