derive cleaner min-4px YOLO corpus

scripts/audit_operator_yolo_dataset_quality.py

@@ -20,8 +20,12 @@ def parse_args() -> argparse.Namespace:
             "negative balance and label-quality risks."
         )
     )
-    parser.add_argument("--summary-path", required=True, help="Path to yolo_tile_dataset_summary.json")
-    parser.add_argument("--output-dir", required=True, help="Directory for JSON and Markdown reports")
+    parser.add_argument(
+        "--summary-path", required=True, help="Path to yolo_tile_dataset_summary.json"
+    )
+    parser.add_argument(
+        "--output-dir", required=True, help="Directory for JSON and Markdown reports"
+    )
     parser.add_argument("--min-positive-samples", type=int, default=6)
     parser.add_argument("--min-val-positive-samples", type=int, default=2)
     parser.add_argument("--max-repeated-negative-share", type=float, default=0.65)
@@ -86,7 +90,12 @@ def parse_yolo_label_file(path: Path | None) -> tuple[list[dict[str, float]], in
         except ValueError:
             invalid_count += 1
             continue
-        if not (0 <= center_x <= 1 and 0 <= center_y <= 1 and 0 < width <= 1 and 0 < height <= 1):
+        if not (
+            0 <= center_x <= 1
+            and 0 <= center_y <= 1
+            and 0 < width <= 1
+            and 0 < height <= 1
+        ):
             invalid_count += 1
             continue
         boxes.append(
@@ -138,7 +147,9 @@ def summarize_label_files(
     areas = [box["area"] for box in boxes]
     widths = [box["width"] for box in boxes]
     heights = [box["height"] for box in boxes]
-    aspect_ratios = [max(box["width"] / box["height"], box["height"] / box["width"]) for box in boxes]
+    aspect_ratios = [
+        max(box["width"] / box["height"], box["height"] / box["width"]) for box in boxes
+    ]
     small_box_count = sum(1 for area in areas if area < small_box_area_threshold)
 
     return {
@@ -175,7 +186,10 @@ def build_label_quality_warning_codes(
     median_box_area = label_stats["median_box_area"]
     if median_box_area is not None and median_box_area < args.min_median_box_area:
         warning_codes.append("median_box_area_below_gate")
-    if label_stats["parsed_label_count"] and label_stats["small_box_share"] > args.max_small_box_share:
+    if (
+        label_stats["parsed_label_count"]
+        and label_stats["small_box_share"] > args.max_small_box_share
+    ):
         warning_codes.append("small_box_share_above_gate")
     return warning_codes
 
@@ -233,7 +247,9 @@ def build_sample_summaries(
 
     result: list[dict[str, Any]] = []
     for sample in samples.values():
-        label_stats = summarize_label_files(sample.pop("_label_file_paths"), args.small_box_area_threshold)
+        label_stats = summarize_label_files(
+            sample.pop("_label_file_paths"), args.small_box_area_threshold
+        )
         quality_warnings = build_label_quality_warning_codes(label_stats, args)
         result.append(
             {
@@ -246,8 +262,14 @@ def build_sample_summaries(
     return sorted(result, key=lambda item: str(item["sample_slug"]))
 
 
-def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Namespace) -> dict[str, Any]:
-    tiles = [tile for tile in summary.get("tiles", []) if isinstance(tile, dict) and tile.get("kept", True)]
+def build_audit(
+    summary: dict[str, Any], summary_path: Path, args: argparse.Namespace
+) -> dict[str, Any]:
+    tiles = [
+        tile
+        for tile in summary.get("tiles", [])
+        if isinstance(tile, dict) and tile.get("kept", True)
+    ]
     sample_summaries = build_sample_summaries(tiles, summary_path, args)
 
     split_counts = Counter(str(tile.get("split") or "unknown") for tile in tiles)
@@ -258,8 +280,12 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
         if bool(tile.get("is_negative")) or int(tile.get("label_count") or 0) == 0
     ]
     positive_tiles = [tile for tile in tiles if tile not in negative_tiles]
-    train_negative_tiles = [tile for tile in negative_tiles if tile.get("split") == "train"]
-    low_variance_positive_tiles = [tile for tile in positive_tiles if tile.get("low_visual_variance")]
+    train_negative_tiles = [
+        tile for tile in negative_tiles if tile.get("split") == "train"
+    ]
+    low_variance_positive_tiles = [
+        tile for tile in positive_tiles if tile.get("low_visual_variance")
+    ]
     val_positive_samples = {
         str(tile.get("sample_slug"))
         for tile in positive_tiles
@@ -275,8 +301,12 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
         for sample in sample_summaries
         if sample["sample_role"] == "background_candidate"
     }
-    repeated_negative_count = sum(1 for tile in negative_tiles if tile.get("is_repeated_background_negative"))
-    repeated_negative_share = repeated_negative_count / len(negative_tiles) if negative_tiles else 0.0
+    repeated_negative_count = sum(
+        1 for tile in negative_tiles if tile.get("is_repeated_background_negative")
+    )
+    repeated_negative_share = (
+        repeated_negative_count / len(negative_tiles) if negative_tiles else 0.0
+    )
 
     label_stats = summarize_labels(tiles, summary_path, args.small_box_area_threshold)
     warnings: list[dict[str, str]] = []
@@ -381,7 +411,10 @@ def build_audit(summary: dict[str, Any], summary_path: Path, args: argparse.Name
 def build_recommendations(warnings: list[dict[str, str]]) -> list[str]:
     codes = {warning["code"] for warning in warnings}
     recommendations: list[str] = []
-    if "positive_sample_count_below_gate" in codes or "val_positive_sample_count_below_gate" in codes:
+    if (
+        "positive_sample_count_below_gate" in codes
+        or "val_positive_sample_count_below_gate" in codes
+    ):
         recommendations.append(
             "Add more labeled positive AOIs before extending training duration or increasing model size."
         )
@@ -394,13 +427,17 @@ def build_recommendations(warnings: list[dict[str, str]]) -> list[str]:
             "Inspect clipped building labels visually; very small boxes may indicate tile size or label clipping issues."
         )
     if "label_files_missing" in codes or "invalid_label_rows" in codes:
-        recommendations.append("Regenerate the YOLO tile dataset and review exporter path/label integrity.")
+        recommendations.append(
+            "Regenerate the YOLO tile dataset and review exporter path/label integrity."
+        )
     if "positive_tiles_have_low_visual_variance" in codes:
         recommendations.append(
             "Reject the raster product for affected AOIs or replace it with an officially complete imagery edition before training."
         )
     if not recommendations:
-        recommendations.append("Dataset audit passed the configured gates; continue with benchmarked training.")
+        recommendations.append(
+            "Dataset audit passed the configured gates; continue with benchmarked training."
+        )
     return recommendations
 
 
@@ -475,7 +512,7 @@ def main() -> int:
     json_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
     write_markdown(report, markdown_path)
 
-    print("Operator YOLO dataset quality audit passed")
+    print("Operator YOLO dataset quality audit completed")
     print(f"Status: {report['status']}")
     print(f"JSON: {json_path}")
     print(f"Markdown: {markdown_path}")