Audit and suppress nested detector proposals

backend/tests/test_belgium_candidate_evaluation.py

@@ -26,3 +26,15 @@ def test_empty_reference_counts_false_positives() -> None:
 
 def test_box_scaling_preserves_center() -> None:
     assert MODULE.scale_box((10.0, 20.0, 30.0, 40.0), 1.5) == (5.0, 15.0, 35.0, 45.0)
+
+
+def test_containment_suppression_removes_nested_lower_score_box() -> None:
+    predictions = [
+        ((0.0, 0.0, 20.0, 20.0), 0.9),
+        ((5.0, 5.0, 15.0, 15.0), 0.8),
+        ((25.0, 0.0, 35.0, 10.0), 0.7),
+    ]
+    assert MODULE.suppress_contained_predictions(predictions, 0.8) == [
+        predictions[0],
+        predictions[2],
+    ]

backend/tests/test_retile_yolo_dataset.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+
+SCRIPT = Path(__file__).parents[2] / "scripts" / "retile_yolo_dataset.py"
+SPEC = importlib.util.spec_from_file_location("retile_yolo", SCRIPT)
+assert SPEC and SPEC.loader
+MODULE = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(MODULE)
+
+
+def test_tile_starts_cover_edges_with_overlap() -> None:
+    assert MODULE.tile_starts(640, 384, 128) == [0, 256]
+    assert MODULE.tile_starts(700, 384, 128) == [0, 256, 316]
+
+
+def test_tile_starts_reject_image_smaller_than_tile() -> None:
+    assert MODULE.tile_starts(320, 384, 128) == []

scripts/audit_operator_yolo_dataset_quality.py

@@ -113,6 +113,13 @@ def safe_median(values: list[float]) -> float | None:
     return round(float(statistics.median(values)), 12) if values else None
 
 
+def safe_percentile(values: list[float], fraction: float) -> float | None:
+    if not values:
+        return None
+    ordered = sorted(values)
+    return round(float(ordered[round((len(ordered) - 1) * fraction)]), 12)
+
+
 def summarize_label_files(
     label_file_paths: set[Path],
     small_box_area_threshold: float,
@@ -131,6 +138,7 @@ def summarize_label_files(
     areas = [box["area"] for box in boxes]
     widths = [box["width"] for box in boxes]
     heights = [box["height"] for box in boxes]
+    aspect_ratios = [max(box["width"] / box["height"], box["height"] / box["width"]) for box in boxes]
     small_box_count = sum(1 for area in areas if area < small_box_area_threshold)
 
     return {
@@ -143,6 +151,11 @@ def summarize_label_files(
         "mean_box_area": safe_mean(areas),
         "median_box_width": safe_median(widths),
         "median_box_height": safe_median(heights),
+        "box_area_p99": safe_percentile(areas, 0.99),
+        "box_area_max": max(areas) if areas else None,
+        "aspect_ratio_median": safe_median(aspect_ratios),
+        "aspect_ratio_p99": safe_percentile(aspect_ratios, 0.99),
+        "aspect_ratio_max": max(aspect_ratios) if aspect_ratios else None,
         "small_box_area_threshold": small_box_area_threshold,
         "small_box_count": small_box_count,
         "small_box_share": small_box_count / len(areas) if areas else 0.0,

scripts/evaluate_belgium_building_candidate.py

@@ -52,6 +52,29 @@ def metrics(tp: int, fp: int, fn: int) -> dict[str, float | int]:
     }
 
 
+def containment_overlap(
+    left: tuple[float, float, float, float], right: tuple[float, float, float, float]
+) -> float:
+    x1, y1 = max(left[0], right[0]), max(left[1], right[1])
+    x2, y2 = min(left[2], right[2]), min(left[3], right[3])
+    intersection = max(0.0, x2 - x1) * max(0.0, y2 - y1)
+    left_area = max(0.0, left[2] - left[0]) * max(0.0, left[3] - left[1])
+    right_area = max(0.0, right[2] - right[0]) * max(0.0, right[3] - right[1])
+    denominator = min(left_area, right_area)
+    return intersection / denominator if denominator > 0 else 0.0
+
+
+def suppress_contained_predictions(
+    predictions: list[tuple[tuple[float, float, float, float], float]], threshold: float
+) -> list[tuple[tuple[float, float, float, float], float]]:
+    kept: list[tuple[tuple[float, float, float, float], float]] = []
+    for candidate in sorted(predictions, key=lambda item: -item[1]):
+        if any(containment_overlap(candidate[0], existing[0]) >= threshold for existing in kept):
+            continue
+        kept.append(candidate)
+    return kept
+
+
 def ensemble_predictions(
     primary: list[tuple[tuple[float, float, float, float], float]],
     secondary: list[tuple[tuple[float, float, float, float], float]],
@@ -128,6 +151,18 @@ def main() -> int:
         default=1000,
         help="Maximum detections retained per tile; dense Belgian urban tiles exceed YOLO's default 300.",
     )
+    parser.add_argument(
+        "--nms-iou",
+        type=float,
+        default=0.7,
+        help="Inference NMS IoU; freeze calibration-selected values before protected test evaluation.",
+    )
+    parser.add_argument(
+        "--containment-nms",
+        type=float,
+        default=1.0,
+        help="Suppress lower-score nested boxes at this intersection-over-minimum-area threshold.",
+    )
     parser.add_argument("--box-scale", type=float, default=1.0)
     parser.add_argument("--box-offset-x", type=float, default=0.0)
     parser.add_argument("--box-offset-y", type=float, default=0.0)
@@ -141,6 +176,10 @@ def main() -> int:
     args = parser.parse_args()
     if args.proposal_classifier_batch < 1:
         parser.error("--proposal-classifier-batch must be positive")
+    if not 0.0 < args.nms_iou < 1.0:
+        parser.error("--nms-iou must be between zero and one")
+    if not 0.0 < args.containment_nms <= 1.0:
+        parser.error("--containment-nms must be above zero and at most one")
 
     from ultralytics import YOLO
     proposal_classifier = None
@@ -173,13 +212,14 @@ def main() -> int:
         augment=args.augment,
         imgsz=args.imgsz,
         max_det=args.max_det,
+        iou=args.nms_iou,
         verbose=False,
     )
     additional_results = None
     if args.additional_model:
         additional_results = YOLO(str(args.additional_model)).predict(
             image_paths, conf=min(args.thresholds), device=args.device, augment=args.augment,
-            imgsz=args.imgsz, max_det=args.max_det, verbose=False,
+            imgsz=args.imgsz, max_det=args.max_det, iou=args.nms_iou, verbose=False,
         )
     observations: list[dict[str, Any]] = []
     for result_index, (tile, result) in enumerate(zip(tiles, results, strict=True)):
@@ -193,6 +233,7 @@ def main() -> int:
             )
             for box, score in zip(result.boxes.xyxy.cpu().tolist(), result.boxes.conf.cpu().tolist(), strict=True)
         ]
+        predictions = suppress_contained_predictions(predictions, args.containment_nms)
         if additional_results is not None:
             secondary = [
                 (
@@ -204,6 +245,7 @@ def main() -> int:
                     additional_results[result_index].boxes.conf.cpu().tolist(), strict=True,
                 )
             ]
+            secondary = suppress_contained_predictions(secondary, args.containment_nms)
             predictions = ensemble_predictions(
                 predictions, secondary, match_iou=args.ensemble_match_iou, mode=args.ensemble_mode
             )
@@ -274,6 +316,8 @@ def main() -> int:
         "test_time_augmentation": args.augment,
         "inference_imgsz": args.imgsz,
         "max_detections_per_tile": args.max_det,
+        "nms_iou": args.nms_iou,
+        "containment_nms": args.containment_nms,
         "box_scale": args.box_scale,
         "box_offset_x": args.box_offset_x,
         "box_offset_y": args.box_offset_y,

scripts/render_building_candidate_error_contact_sheets.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""Render deterministic reference/prediction error overlays for protected calibration AOIs."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import statistics
+from pathlib import Path
+
+from PIL import Image, ImageDraw, ImageFont
+
+from evaluate_belgium_building_candidate import iou, read_references
+
+
+def match_details(predictions, references, match_iou: float):
+    candidates = sorted(predictions, key=lambda item: -item[1])
+    unmatched = set(range(len(references)))
+    matched_predictions: list[tuple[tuple[float, float, float, float], float]] = []
+    false_predictions: list[tuple[tuple[float, float, float, float], float]] = []
+    for prediction in candidates:
+        best = max(unmatched, key=lambda index: iou(prediction[0], references[index]), default=None)
+        if best is not None and iou(prediction[0], references[best]) >= match_iou:
+            unmatched.remove(best)
+            matched_predictions.append(prediction)
+        else:
+            false_predictions.append(prediction)
+    return matched_predictions, false_predictions, [references[index] for index in sorted(unmatched)]
+
+
+def draw_boxes(draw, boxes, *, color, width=3, scores=False):
+    font = ImageFont.load_default()
+    for item in boxes:
+        box, score = item if scores else (item, None)
+        draw.rectangle(box, outline=color, width=width)
+        if score is not None:
+            draw.text((box[0] + 2, box[1] + 2), f"{score:.2f}", fill=color, font=font)
+
+
+def geometry(box):
+    width = max(0.0, box[2] - box[0])
+    height = max(0.0, box[3] - box[1])
+    return width * height, max(width / height, height / width) if width and height else float("inf")
+
+
+def distribution(values):
+    ordered = sorted(value for value in values if math.isfinite(value))
+    if not ordered:
+        return {"count": 0}
+    def percentile(fraction):
+        return ordered[round((len(ordered) - 1) * fraction)]
+    return {
+        "count": len(ordered), "median": statistics.median(ordered),
+        "p90": percentile(0.9), "p95": percentile(0.95), "p99": percentile(0.99),
+        "max": ordered[-1],
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", type=Path, required=True)
+    parser.add_argument("--summary", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--sample", action="append", required=True)
+    parser.add_argument("--threshold", type=float, required=True)
+    parser.add_argument("--match-iou", type=float, default=0.25)
+    parser.add_argument("--device", default="cuda:0")
+    parser.add_argument("--imgsz", type=int, default=640)
+    parser.add_argument("--max-det", type=int, default=1000)
+    parser.add_argument("--nms-iou", type=float, default=0.7)
+    parser.add_argument("--max-tiles", type=int, default=24)
+    parser.add_argument("--columns", type=int, default=4)
+    args = parser.parse_args()
+
+    from ultralytics import YOLO
+
+    summary = json.loads(args.summary.read_text(encoding="utf-8"))
+    selected = [tile for tile in summary["tiles"] if tile["sample_slug"] in set(args.sample)]
+    image_paths = [Path(tile["image_path"]) for tile in selected]
+    results = YOLO(str(args.model)).predict(
+        image_paths, conf=args.threshold, device=args.device, imgsz=args.imgsz,
+        max_det=args.max_det, iou=args.nms_iou, verbose=False, stream=False,
+    )
+    evidence = []
+    cards = []
+    geometry_evidence = {role: {"area_share": [], "aspect_ratio": []} for role in ("reference", "matched", "false_positive", "false_negative")}
+    for tile, image_path, result in zip(selected, image_paths, results, strict=True):
+        image = Image.open(image_path).convert("RGB")
+        predictions = [
+            ((float(box[0]), float(box[1]), float(box[2]), float(box[3])), float(score))
+            for box, score in zip(result.boxes.xyxy.cpu().tolist(), result.boxes.conf.cpu().tolist(), strict=True)
+        ]
+        references = read_references(Path(tile["label_path"]), image.width, image.height)
+        matched, false_positive, false_negative = match_details(predictions, references, args.match_iou)
+        for role, boxes in (("reference", references), ("false_negative", false_negative)):
+            for box in boxes:
+                area, aspect = geometry(box)
+                geometry_evidence[role]["area_share"].append(area / (image.width * image.height))
+                geometry_evidence[role]["aspect_ratio"].append(aspect)
+        for role, boxes in (("matched", matched), ("false_positive", false_positive)):
+            for box, _score in boxes:
+                area, aspect = geometry(box)
+                geometry_evidence[role]["area_share"].append(area / (image.width * image.height))
+                geometry_evidence[role]["aspect_ratio"].append(aspect)
+        evidence.append({
+            "sample_slug": tile["sample_slug"], "image_path": str(image_path),
+            "label_count": len(references), "true_positive": len(matched),
+            "false_positive": len(false_positive), "false_negative": len(false_negative),
+        })
+        draw = ImageDraw.Draw(image)
+        draw_boxes(draw, references, color=(255, 215, 0), width=2)
+        draw_boxes(draw, matched, color=(0, 220, 80), scores=True)
+        draw_boxes(draw, false_positive, color=(255, 40, 40), scores=True)
+        draw_boxes(draw, false_negative, color=(255, 0, 220), width=3)
+        cards.append((len(false_positive) + len(false_negative), tile["sample_slug"], image_path.name, image))
+
+    cards.sort(key=lambda item: (-item[0], item[1], item[2]))
+    cards = cards[: args.max_tiles]
+    header = 34
+    rendered = []
+    font = ImageFont.load_default()
+    for errors, slug, name, image in cards:
+        card = Image.new("RGB", (image.width, image.height + header), (20, 31, 44))
+        card.paste(image, (0, header))
+        ImageDraw.Draw(card).text((6, 7), f"{slug} | {name} | errors={errors}", fill="white", font=font)
+        rendered.append(card)
+    gap = 10
+    rows = math.ceil(len(rendered) / args.columns) if rendered else 0
+    if rendered:
+        width = args.columns * rendered[0].width + (args.columns + 1) * gap
+        height = rows * rendered[0].height + (rows + 1) * gap
+        sheet = Image.new("RGB", (width, height), (226, 232, 240))
+        for index, card in enumerate(rendered):
+            x = gap + (index % args.columns) * (card.width + gap)
+            y = gap + (index // args.columns) * (card.height + gap)
+            sheet.paste(card, (x, y))
+        args.output_dir.mkdir(parents=True, exist_ok=True)
+        sheet.save(args.output_dir / "candidate_error_contact_sheet.png")
+    report = {
+        "schema_version": 1, "model": str(args.model), "summary": str(args.summary),
+        "threshold": args.threshold, "match_iou": args.match_iou, "samples": args.sample,
+        "legend": {"reference": "yellow", "matched_prediction": "green", "false_positive": "red", "false_negative": "magenta"},
+        "geometry": {
+            role: {name: distribution(values) for name, values in metrics.items()}
+            for role, metrics in geometry_evidence.items()
+        },
+        "tiles": evidence,
+    }
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    (args.output_dir / "candidate_error_contact_sheet.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
+    print(json.dumps({"status": "ok", "tile_count": len(evidence), "rendered": len(rendered)}, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/retile_yolo_dataset.py

@@ -31,15 +31,27 @@ def read_boxes(path: Path, width: int, height: int) -> list[tuple[float, float,
     return boxes
 
 
+def tile_starts(length: int, tile_size: int, overlap: int) -> list[int]:
+    if length < tile_size:
+        return []
+    stride = tile_size - overlap
+    starts = list(range(0, length - tile_size + 1, stride))
+    final = length - tile_size
+    if starts[-1] != final:
+        starts.append(final)
+    return starts
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--summary", type=Path, required=True)
     parser.add_argument("--output-dir", type=Path, required=True)
     parser.add_argument("--tile-size", type=int, default=320)
+    parser.add_argument("--overlap", type=int, default=0)
     parser.add_argument("--min-visible-ratio", type=float, default=0.25)
     parser.add_argument("--force", action="store_true")
     args = parser.parse_args()
-    if args.tile_size < 32 or not 0 <= args.min_visible_ratio <= 1:
+    if args.tile_size < 32 or not 0 <= args.overlap < args.tile_size or not 0 <= args.min_visible_ratio <= 1:
         raise SystemExit("invalid tile size or visible ratio")
     if args.output_dir.exists():
         if not args.force:
@@ -56,8 +68,8 @@ def main() -> int:
             source = opened.convert("RGB")
         width, height = source.size
         boxes = read_boxes(Path(source_tile["label_path"]), width, height)
-        for top in range(0, height, args.tile_size):
-            for left in range(0, width, args.tile_size):
+        for top in tile_starts(height, args.tile_size, args.overlap):
+            for left in tile_starts(width, args.tile_size, args.overlap):
                 right, bottom = min(width, left+args.tile_size), min(height, top+args.tile_size)
                 if right-left < args.tile_size or bottom-top < args.tile_size:
                     continue
@@ -95,6 +107,7 @@ def main() -> int:
     output_summary = dict(summary)
     output_summary.update({"output_dir":str(args.output_dir), "dataset_yaml":str(args.output_dir/"dataset.yaml"),
                            "tiles":output_tiles, "retile_size":args.tile_size,
+                           "retile_overlap":args.overlap,
                            "retile_min_visible_ratio":args.min_visible_ratio})
     summary_path = args.output_dir / "yolo_tile_dataset_summary.json"
     summary_path.write_text(json.dumps(output_summary, indent=2), encoding="utf-8")
@@ -102,6 +115,7 @@ def main() -> int:
         f"path: {args.output_dir}\ntrain: images/train\nval: images/val\nnames:\n  0: building\n", encoding="utf-8")
     evidence = {"schema_version":1, "status":"ok", "source_summary":str(args.summary),
                 "source_summary_sha256":sha256(args.summary), "tile_size":args.tile_size,
+                "overlap":args.overlap,
                 "min_visible_ratio":args.min_visible_ratio, "output_tile_count":len(output_tiles),
                 "kept_label_count":kept_labels, "dropped_label_count":dropped_labels,
                 "summary":str(summary_path), "summary_sha256":sha256(summary_path)}