diff --git a/backend/tests/test_belgium_candidate_evaluation.py b/backend/tests/test_belgium_candidate_evaluation.py index 0d80e809..da477ddd 100644 --- a/backend/tests/test_belgium_candidate_evaluation.py +++ b/backend/tests/test_belgium_candidate_evaluation.py @@ -26,3 +26,15 @@ def test_empty_reference_counts_false_positives() -> None: def test_box_scaling_preserves_center() -> None: assert MODULE.scale_box((10.0, 20.0, 30.0, 40.0), 1.5) == (5.0, 15.0, 35.0, 45.0) + + +def test_containment_suppression_removes_nested_lower_score_box() -> None: + predictions = [ + ((0.0, 0.0, 20.0, 20.0), 0.9), + ((5.0, 5.0, 15.0, 15.0), 0.8), + ((25.0, 0.0, 35.0, 10.0), 0.7), + ] + assert MODULE.suppress_contained_predictions(predictions, 0.8) == [ + predictions[0], + predictions[2], + ] diff --git a/backend/tests/test_retile_yolo_dataset.py b/backend/tests/test_retile_yolo_dataset.py new file mode 100644 index 00000000..f51c2980 --- /dev/null +++ b/backend/tests/test_retile_yolo_dataset.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +SCRIPT = Path(__file__).parents[2] / "scripts" / "retile_yolo_dataset.py" +SPEC = importlib.util.spec_from_file_location("retile_yolo", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_tile_starts_cover_edges_with_overlap() -> None: + assert MODULE.tile_starts(640, 384, 128) == [0, 256] + assert MODULE.tile_starts(700, 384, 128) == [0, 256, 316] + + +def test_tile_starts_reject_image_smaller_than_tile() -> None: + assert MODULE.tile_starts(320, 384, 128) == [] diff --git a/scripts/audit_operator_yolo_dataset_quality.py b/scripts/audit_operator_yolo_dataset_quality.py index b8b0db5e..6142d679 100644 --- a/scripts/audit_operator_yolo_dataset_quality.py +++ b/scripts/audit_operator_yolo_dataset_quality.py @@ -113,6 +113,13 @@ def safe_median(values: list[float]) -> float | None: return round(float(statistics.median(values)), 12) if values else None +def safe_percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + return round(float(ordered[round((len(ordered) - 1) * fraction)]), 12) + + def summarize_label_files( label_file_paths: set[Path], small_box_area_threshold: float, @@ -131,6 +138,7 @@ def summarize_label_files( areas = [box["area"] for box in boxes] widths = [box["width"] for box in boxes] heights = [box["height"] for box in boxes] + aspect_ratios = [max(box["width"] / box["height"], box["height"] / box["width"]) for box in boxes] small_box_count = sum(1 for area in areas if area < small_box_area_threshold) return { @@ -143,6 +151,11 @@ def summarize_label_files( "mean_box_area": safe_mean(areas), "median_box_width": safe_median(widths), "median_box_height": safe_median(heights), + "box_area_p99": safe_percentile(areas, 0.99), + "box_area_max": max(areas) if areas else None, + "aspect_ratio_median": safe_median(aspect_ratios), + "aspect_ratio_p99": safe_percentile(aspect_ratios, 0.99), + "aspect_ratio_max": max(aspect_ratios) if aspect_ratios else None, "small_box_area_threshold": small_box_area_threshold, "small_box_count": small_box_count, "small_box_share": small_box_count / len(areas) if areas else 0.0, diff --git a/scripts/evaluate_belgium_building_candidate.py b/scripts/evaluate_belgium_building_candidate.py index d4a91790..05cdff83 100644 --- a/scripts/evaluate_belgium_building_candidate.py +++ b/scripts/evaluate_belgium_building_candidate.py @@ -52,6 +52,29 @@ def metrics(tp: int, fp: int, fn: int) -> dict[str, float | int]: } +def containment_overlap( + left: tuple[float, float, float, float], right: tuple[float, float, float, float] +) -> float: + x1, y1 = max(left[0], right[0]), max(left[1], right[1]) + x2, y2 = min(left[2], right[2]), min(left[3], right[3]) + intersection = max(0.0, x2 - x1) * max(0.0, y2 - y1) + left_area = max(0.0, left[2] - left[0]) * max(0.0, left[3] - left[1]) + right_area = max(0.0, right[2] - right[0]) * max(0.0, right[3] - right[1]) + denominator = min(left_area, right_area) + return intersection / denominator if denominator > 0 else 0.0 + + +def suppress_contained_predictions( + predictions: list[tuple[tuple[float, float, float, float], float]], threshold: float +) -> list[tuple[tuple[float, float, float, float], float]]: + kept: list[tuple[tuple[float, float, float, float], float]] = [] + for candidate in sorted(predictions, key=lambda item: -item[1]): + if any(containment_overlap(candidate[0], existing[0]) >= threshold for existing in kept): + continue + kept.append(candidate) + return kept + + def ensemble_predictions( primary: list[tuple[tuple[float, float, float, float], float]], secondary: list[tuple[tuple[float, float, float, float], float]], @@ -128,6 +151,18 @@ def main() -> int: default=1000, help="Maximum detections retained per tile; dense Belgian urban tiles exceed YOLO's default 300.", ) + parser.add_argument( + "--nms-iou", + type=float, + default=0.7, + help="Inference NMS IoU; freeze calibration-selected values before protected test evaluation.", + ) + parser.add_argument( + "--containment-nms", + type=float, + default=1.0, + help="Suppress lower-score nested boxes at this intersection-over-minimum-area threshold.", + ) parser.add_argument("--box-scale", type=float, default=1.0) parser.add_argument("--box-offset-x", type=float, default=0.0) parser.add_argument("--box-offset-y", type=float, default=0.0) @@ -141,6 +176,10 @@ def main() -> int: args = parser.parse_args() if args.proposal_classifier_batch < 1: parser.error("--proposal-classifier-batch must be positive") + if not 0.0 < args.nms_iou < 1.0: + parser.error("--nms-iou must be between zero and one") + if not 0.0 < args.containment_nms <= 1.0: + parser.error("--containment-nms must be above zero and at most one") from ultralytics import YOLO proposal_classifier = None @@ -173,13 +212,14 @@ def main() -> int: augment=args.augment, imgsz=args.imgsz, max_det=args.max_det, + iou=args.nms_iou, verbose=False, ) additional_results = None if args.additional_model: additional_results = YOLO(str(args.additional_model)).predict( image_paths, conf=min(args.thresholds), device=args.device, augment=args.augment, - imgsz=args.imgsz, max_det=args.max_det, verbose=False, + imgsz=args.imgsz, max_det=args.max_det, iou=args.nms_iou, verbose=False, ) observations: list[dict[str, Any]] = [] for result_index, (tile, result) in enumerate(zip(tiles, results, strict=True)): @@ -193,6 +233,7 @@ def main() -> int: ) for box, score in zip(result.boxes.xyxy.cpu().tolist(), result.boxes.conf.cpu().tolist(), strict=True) ] + predictions = suppress_contained_predictions(predictions, args.containment_nms) if additional_results is not None: secondary = [ ( @@ -204,6 +245,7 @@ def main() -> int: additional_results[result_index].boxes.conf.cpu().tolist(), strict=True, ) ] + secondary = suppress_contained_predictions(secondary, args.containment_nms) predictions = ensemble_predictions( predictions, secondary, match_iou=args.ensemble_match_iou, mode=args.ensemble_mode ) @@ -274,6 +316,8 @@ def main() -> int: "test_time_augmentation": args.augment, "inference_imgsz": args.imgsz, "max_detections_per_tile": args.max_det, + "nms_iou": args.nms_iou, + "containment_nms": args.containment_nms, "box_scale": args.box_scale, "box_offset_x": args.box_offset_x, "box_offset_y": args.box_offset_y, diff --git a/scripts/render_building_candidate_error_contact_sheets.py b/scripts/render_building_candidate_error_contact_sheets.py new file mode 100644 index 00000000..b8683e6b --- /dev/null +++ b/scripts/render_building_candidate_error_contact_sheets.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Render deterministic reference/prediction error overlays for protected calibration AOIs.""" + +from __future__ import annotations + +import argparse +import json +import math +import statistics +from pathlib import Path + +from PIL import Image, ImageDraw, ImageFont + +from evaluate_belgium_building_candidate import iou, read_references + + +def match_details(predictions, references, match_iou: float): + candidates = sorted(predictions, key=lambda item: -item[1]) + unmatched = set(range(len(references))) + matched_predictions: list[tuple[tuple[float, float, float, float], float]] = [] + false_predictions: list[tuple[tuple[float, float, float, float], float]] = [] + for prediction in candidates: + best = max(unmatched, key=lambda index: iou(prediction[0], references[index]), default=None) + if best is not None and iou(prediction[0], references[best]) >= match_iou: + unmatched.remove(best) + matched_predictions.append(prediction) + else: + false_predictions.append(prediction) + return matched_predictions, false_predictions, [references[index] for index in sorted(unmatched)] + + +def draw_boxes(draw, boxes, *, color, width=3, scores=False): + font = ImageFont.load_default() + for item in boxes: + box, score = item if scores else (item, None) + draw.rectangle(box, outline=color, width=width) + if score is not None: + draw.text((box[0] + 2, box[1] + 2), f"{score:.2f}", fill=color, font=font) + + +def geometry(box): + width = max(0.0, box[2] - box[0]) + height = max(0.0, box[3] - box[1]) + return width * height, max(width / height, height / width) if width and height else float("inf") + + +def distribution(values): + ordered = sorted(value for value in values if math.isfinite(value)) + if not ordered: + return {"count": 0} + def percentile(fraction): + return ordered[round((len(ordered) - 1) * fraction)] + return { + "count": len(ordered), "median": statistics.median(ordered), + "p90": percentile(0.9), "p95": percentile(0.95), "p99": percentile(0.99), + "max": ordered[-1], + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--sample", action="append", required=True) + parser.add_argument("--threshold", type=float, required=True) + parser.add_argument("--match-iou", type=float, default=0.25) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--imgsz", type=int, default=640) + parser.add_argument("--max-det", type=int, default=1000) + parser.add_argument("--nms-iou", type=float, default=0.7) + parser.add_argument("--max-tiles", type=int, default=24) + parser.add_argument("--columns", type=int, default=4) + args = parser.parse_args() + + from ultralytics import YOLO + + summary = json.loads(args.summary.read_text(encoding="utf-8")) + selected = [tile for tile in summary["tiles"] if tile["sample_slug"] in set(args.sample)] + image_paths = [Path(tile["image_path"]) for tile in selected] + results = YOLO(str(args.model)).predict( + image_paths, conf=args.threshold, device=args.device, imgsz=args.imgsz, + max_det=args.max_det, iou=args.nms_iou, verbose=False, stream=False, + ) + evidence = [] + cards = [] + geometry_evidence = {role: {"area_share": [], "aspect_ratio": []} for role in ("reference", "matched", "false_positive", "false_negative")} + for tile, image_path, result in zip(selected, image_paths, results, strict=True): + image = Image.open(image_path).convert("RGB") + predictions = [ + ((float(box[0]), float(box[1]), float(box[2]), float(box[3])), float(score)) + for box, score in zip(result.boxes.xyxy.cpu().tolist(), result.boxes.conf.cpu().tolist(), strict=True) + ] + references = read_references(Path(tile["label_path"]), image.width, image.height) + matched, false_positive, false_negative = match_details(predictions, references, args.match_iou) + for role, boxes in (("reference", references), ("false_negative", false_negative)): + for box in boxes: + area, aspect = geometry(box) + geometry_evidence[role]["area_share"].append(area / (image.width * image.height)) + geometry_evidence[role]["aspect_ratio"].append(aspect) + for role, boxes in (("matched", matched), ("false_positive", false_positive)): + for box, _score in boxes: + area, aspect = geometry(box) + geometry_evidence[role]["area_share"].append(area / (image.width * image.height)) + geometry_evidence[role]["aspect_ratio"].append(aspect) + evidence.append({ + "sample_slug": tile["sample_slug"], "image_path": str(image_path), + "label_count": len(references), "true_positive": len(matched), + "false_positive": len(false_positive), "false_negative": len(false_negative), + }) + draw = ImageDraw.Draw(image) + draw_boxes(draw, references, color=(255, 215, 0), width=2) + draw_boxes(draw, matched, color=(0, 220, 80), scores=True) + draw_boxes(draw, false_positive, color=(255, 40, 40), scores=True) + draw_boxes(draw, false_negative, color=(255, 0, 220), width=3) + cards.append((len(false_positive) + len(false_negative), tile["sample_slug"], image_path.name, image)) + + cards.sort(key=lambda item: (-item[0], item[1], item[2])) + cards = cards[: args.max_tiles] + header = 34 + rendered = [] + font = ImageFont.load_default() + for errors, slug, name, image in cards: + card = Image.new("RGB", (image.width, image.height + header), (20, 31, 44)) + card.paste(image, (0, header)) + ImageDraw.Draw(card).text((6, 7), f"{slug} | {name} | errors={errors}", fill="white", font=font) + rendered.append(card) + gap = 10 + rows = math.ceil(len(rendered) / args.columns) if rendered else 0 + if rendered: + width = args.columns * rendered[0].width + (args.columns + 1) * gap + height = rows * rendered[0].height + (rows + 1) * gap + sheet = Image.new("RGB", (width, height), (226, 232, 240)) + for index, card in enumerate(rendered): + x = gap + (index % args.columns) * (card.width + gap) + y = gap + (index // args.columns) * (card.height + gap) + sheet.paste(card, (x, y)) + args.output_dir.mkdir(parents=True, exist_ok=True) + sheet.save(args.output_dir / "candidate_error_contact_sheet.png") + report = { + "schema_version": 1, "model": str(args.model), "summary": str(args.summary), + "threshold": args.threshold, "match_iou": args.match_iou, "samples": args.sample, + "legend": {"reference": "yellow", "matched_prediction": "green", "false_positive": "red", "false_negative": "magenta"}, + "geometry": { + role: {name: distribution(values) for name, values in metrics.items()} + for role, metrics in geometry_evidence.items() + }, + "tiles": evidence, + } + args.output_dir.mkdir(parents=True, exist_ok=True) + (args.output_dir / "candidate_error_contact_sheet.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + print(json.dumps({"status": "ok", "tile_count": len(evidence), "rendered": len(rendered)}, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/retile_yolo_dataset.py b/scripts/retile_yolo_dataset.py index 60a8e985..23fa7f71 100644 --- a/scripts/retile_yolo_dataset.py +++ b/scripts/retile_yolo_dataset.py @@ -31,15 +31,27 @@ def read_boxes(path: Path, width: int, height: int) -> list[tuple[float, float, return boxes +def tile_starts(length: int, tile_size: int, overlap: int) -> list[int]: + if length < tile_size: + return [] + stride = tile_size - overlap + starts = list(range(0, length - tile_size + 1, stride)) + final = length - tile_size + if starts[-1] != final: + starts.append(final) + return starts + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--summary", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--tile-size", type=int, default=320) + parser.add_argument("--overlap", type=int, default=0) parser.add_argument("--min-visible-ratio", type=float, default=0.25) parser.add_argument("--force", action="store_true") args = parser.parse_args() - if args.tile_size < 32 or not 0 <= args.min_visible_ratio <= 1: + if args.tile_size < 32 or not 0 <= args.overlap < args.tile_size or not 0 <= args.min_visible_ratio <= 1: raise SystemExit("invalid tile size or visible ratio") if args.output_dir.exists(): if not args.force: @@ -56,8 +68,8 @@ def main() -> int: source = opened.convert("RGB") width, height = source.size boxes = read_boxes(Path(source_tile["label_path"]), width, height) - for top in range(0, height, args.tile_size): - for left in range(0, width, args.tile_size): + for top in tile_starts(height, args.tile_size, args.overlap): + for left in tile_starts(width, args.tile_size, args.overlap): right, bottom = min(width, left+args.tile_size), min(height, top+args.tile_size) if right-left < args.tile_size or bottom-top < args.tile_size: continue @@ -95,6 +107,7 @@ def main() -> int: output_summary = dict(summary) output_summary.update({"output_dir":str(args.output_dir), "dataset_yaml":str(args.output_dir/"dataset.yaml"), "tiles":output_tiles, "retile_size":args.tile_size, + "retile_overlap":args.overlap, "retile_min_visible_ratio":args.min_visible_ratio}) summary_path = args.output_dir / "yolo_tile_dataset_summary.json" summary_path.write_text(json.dumps(output_summary, indent=2), encoding="utf-8") @@ -102,6 +115,7 @@ def main() -> int: f"path: {args.output_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n", encoding="utf-8") evidence = {"schema_version":1, "status":"ok", "source_summary":str(args.summary), "source_summary_sha256":sha256(args.summary), "tile_size":args.tile_size, + "overlap":args.overlap, "min_visible_ratio":args.min_visible_ratio, "output_tile_count":len(output_tiles), "kept_label_count":kept_labels, "dropped_label_count":dropped_labels, "summary":str(summary_path), "summary_sha256":sha256(summary_path)}