evaluate models on fresh regional calibration AOIs

scripts/evaluate_yolo_checkpoint_matrix.py

@@ -270,12 +270,39 @@ def background_detection_count(
     return len(selected), count
 
 
+def validate_pure_background_prefixes(
+    images: list[Path], prefixes: tuple[str, ...]
+) -> list[Path]:
+    selected = [path for path in images if path.stem.casefold().startswith(prefixes)]
+    if not selected:
+        raise ValueError(
+            "no validation images match the declared pure-background prefixes"
+        )
+    nonempty: list[str] = []
+    for image in selected:
+        try:
+            relative = image.relative_to(image.parents[1])
+        except ValueError as exc:
+            raise ValueError(f"cannot resolve label path for {image}") from exc
+        label = image.parents[1].parent / "labels" / relative.with_suffix(".txt")
+        if not label.is_file():
+            raise ValueError(f"pure-background image has no label file: {image}")
+        if label.read_text(encoding="utf-8").strip():
+            nonempty.append(str(label))
+    if nonempty:
+        raise ValueError(
+            "pure-background prefixes include non-empty labels: "
+            + ", ".join(nonempty[:10])
+        )
+    return selected
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--dataset-yaml", type=Path, required=True)
     parser.add_argument("--model", type=Path, action="append", required=True)
     parser.add_argument("--output", type=Path, required=True)
-    parser.add_argument("--background-prefix", action="append", required=True)
+    parser.add_argument("--background-prefix", action="append", default=[])
     parser.add_argument("--background-confidence", type=float, default=0.15)
     parser.add_argument(
         "--lineage-summary",
@@ -321,6 +348,8 @@ def main() -> int:
         write_blocked_manifest(args.output, dataset_yaml, independence_evidence)
         return 3
     prefixes = tuple(value.casefold() for value in args.background_prefix)
+    if prefixes:
+        validate_pure_background_prefixes(images, prefixes)
 
     import torch
     from ultralytics import YOLO
@@ -359,14 +388,6 @@ def main() -> int:
                 save_json=False,
                 verbose=False,
             )
-            background_images, background_detections = background_detection_count(
-                model,
-                images,
-                prefixes=prefixes,
-                confidence=args.background_confidence,
-                image_size=args.imgsz,
-                device=args.device,
-            )
             row.update(
                 {
                     "status": "ok",
@@ -374,10 +395,23 @@ def main() -> int:
                     "recall": metric_value(metrics, "mr"),
                     "map50": metric_value(metrics, "map50"),
                     "map50_95": metric_value(metrics, "map"),
-                    "pure_background_image_count": background_images,
-                    "pure_background_detection_count": background_detections,
                 }
             )
+            if prefixes:
+                background_images, background_detections = background_detection_count(
+                    model,
+                    images,
+                    prefixes=prefixes,
+                    confidence=args.background_confidence,
+                    image_size=args.imgsz,
+                    device=args.device,
+                )
+                row.update(
+                    {
+                        "pure_background_image_count": background_images,
+                        "pure_background_detection_count": background_detections,
+                    }
+                )
         except Exception as exc:  # preserve the complete attempted matrix
             row.update(
                 {
@@ -396,7 +430,7 @@ def main() -> int:
     successful = [row for row in rows if row["status"] == "ok"]
     successful.sort(
         key=lambda row: (
-            int(row["pure_background_detection_count"] == 0),
+            int(row.get("pure_background_detection_count") == 0 and bool(prefixes)),
             row["map50_95"],
             row["map50"],
             row["precision"],