From 31c18792d1cd67f97d262befef55cf5ffd9535f7 Mon Sep 17 00:00:00 2001 From: Jens Date: Mon, 27 Jul 2026 05:21:53 +0200 Subject: [PATCH] Add reproducible grayscale detector corpus --- backend/tests/test_grayscale_yolo_dataset.py | 47 ++++++++++ deploy/unraid/Dockerfile.all-in-one | 1 + scripts/build_grayscale_yolo_dataset.py | 87 +++++++++++++++++++ .../evaluate_belgium_building_candidate.py | 3 + 4 files changed, 138 insertions(+) create mode 100644 backend/tests/test_grayscale_yolo_dataset.py create mode 100644 scripts/build_grayscale_yolo_dataset.py diff --git a/backend/tests/test_grayscale_yolo_dataset.py b/backend/tests/test_grayscale_yolo_dataset.py new file mode 100644 index 00000000..9d3291d6 --- /dev/null +++ b/backend/tests/test_grayscale_yolo_dataset.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +from PIL import Image + + +SCRIPT = Path(__file__).parents[2] / "scripts" / "build_grayscale_yolo_dataset.py" + + +def test_grayscale_builder_preserves_labels_and_split(tmp_path: Path) -> None: + source = tmp_path / "source" + (source / "images" / "train").mkdir(parents=True) + (source / "labels" / "train").mkdir(parents=True) + image = source / "images" / "train" / "tile.png" + label = source / "labels" / "train" / "tile.txt" + Image.new("RGB", (8, 8), (255, 0, 0)).save(image) + label.write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8") + summary = source / "yolo_tile_dataset_summary.json" + summary.write_text( + json.dumps( + { + "tiles": [ + { + "split": "train", + "image_path": str(image), + "label_path": str(label), + } + ] + } + ), + encoding="utf-8", + ) + output = tmp_path / "gray" + subprocess.run( + [sys.executable, str(SCRIPT), "--summary", str(summary), "--output-dir", str(output)], + check=True, + ) + converted = Image.open(output / "images" / "train" / "tile.png") + r, g, b = converted.getpixel((0, 0)) + assert r == g == b + assert (output / "labels" / "train" / "tile.txt").read_text() == label.read_text() + evidence = json.loads((output / "grayscale-dataset-evidence.json").read_text()) + assert evidence["converted_tile_count"] == 1 diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index e8513eef..7ba3bda6 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -128,6 +128,7 @@ COPY scripts/evaluate_belgium_building_candidate.py /app/scripts/evaluate_belgiu COPY scripts/assess_belgium_building_training_iteration.py /app/scripts/assess_belgium_building_training_iteration.py COPY scripts/run_belgium_building_training_loop.py /app/scripts/run_belgium_building_training_loop.py COPY scripts/build_failure_driven_yolo_sampling.py /app/scripts/build_failure_driven_yolo_sampling.py +COPY scripts/build_grayscale_yolo_dataset.py /app/scripts/build_grayscale_yolo_dataset.py COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py COPY scripts/train_operator_yolo_detector.sh /app/scripts/train_operator_yolo_detector.sh diff --git a/scripts/build_grayscale_yolo_dataset.py b/scripts/build_grayscale_yolo_dataset.py new file mode 100644 index 00000000..01ef3f18 --- /dev/null +++ b/scripts/build_grayscale_yolo_dataset.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Create a checksummed luminance-normalized YOLO dataset without changing labels/splits.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +from pathlib import Path + +from PIL import Image + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--force", action="store_true") + args = parser.parse_args() + if args.output_dir.exists(): + if not args.force: + raise SystemExit(f"Output exists: {args.output_dir}") + shutil.rmtree(args.output_dir) + args.output_dir.mkdir(parents=True) + + summary = json.loads(args.summary.read_text(encoding="utf-8")) + converted = 0 + for tile in summary["tiles"]: + split = tile["split"] + source_image = Path(tile["image_path"]) + source_label = Path(tile["label_path"]) + image_target = args.output_dir / "images" / split / source_image.name + label_target = args.output_dir / "labels" / split / source_label.name + image_target.parent.mkdir(parents=True, exist_ok=True) + label_target.parent.mkdir(parents=True, exist_ok=True) + with Image.open(source_image) as image: + image.convert("L").convert("RGB").save(image_target) + shutil.copyfile(source_label, label_target) + tile["image_path"] = str(image_target) + tile["label_path"] = str(label_target) + converted += 1 + + dataset_yaml = args.output_dir / "dataset.yaml" + dataset_yaml.write_text( + f"path: {args.output_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n", + encoding="utf-8", + ) + summary.update( + { + "output_dir": str(args.output_dir), + "dataset_yaml": str(dataset_yaml), + "preprocessing": "luminance_rgb_replicated", + "source_summary": str(args.summary), + "source_summary_sha256": sha256(args.summary), + } + ) + output_summary = args.output_dir / "yolo_tile_dataset_summary.json" + output_summary.write_text(json.dumps(summary, indent=2), encoding="utf-8") + evidence = { + "schema_version": 1, + "status": "ok", + "preprocessing": "luminance_rgb_replicated", + "source_summary": str(args.summary), + "source_summary_sha256": sha256(args.summary), + "converted_tile_count": converted, + "output_summary": str(output_summary), + "output_summary_sha256": sha256(output_summary), + "dataset_yaml": str(dataset_yaml), + } + (args.output_dir / "grayscale-dataset-evidence.json").write_text( + json.dumps(evidence, indent=2), encoding="utf-8" + ) + print(json.dumps(evidence, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluate_belgium_building_candidate.py b/scripts/evaluate_belgium_building_candidate.py index d619d5a5..33729fc5 100644 --- a/scripts/evaluate_belgium_building_candidate.py +++ b/scripts/evaluate_belgium_building_candidate.py @@ -81,6 +81,7 @@ def main() -> int: parser.add_argument("--device", default="cuda:0") parser.add_argument("--split", default="val") parser.add_argument("--augment", action="store_true", help="Enable deterministic YOLO test-time augmentation.") + parser.add_argument("--imgsz", type=int, default=640) args = parser.parse_args() from ultralytics import YOLO @@ -104,6 +105,7 @@ def main() -> int: conf=min(args.thresholds), device=args.device, augment=args.augment, + imgsz=args.imgsz, verbose=False, ) observations: list[dict[str, Any]] = [] @@ -152,6 +154,7 @@ def main() -> int: "split": args.split, "match_iou": args.match_iou, "test_time_augmentation": args.augment, + "inference_imgsz": args.imgsz, "tile_count": len(tiles), "sweeps": sweeps, }