diff --git a/CHANGELOG.md b/CHANGELOG.md index ee9e1c28..0294c848 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ # Changelog +## Sprint 130 Operator YOLO tile-level dataset tooling (2026-07-07) + +- Added `scripts/export_operator_yolo_tile_dataset.py` to convert prepared operator samples into overlapping YOLO tile datasets with clipped building labels and deterministic negative tile retention. +- Added readiness coverage for the tile exporter Python compile check. +- Added regression coverage in `backend/tests/test_sprint130_operator_yolo_tile_dataset.py` for script contract, help behavior without GIS imports, edge-covering tile windows and deterministic negative-tile selection. +- Updated operator documentation for tile-level dataset export and reuse of the existing local training wrapper. +- No Training Studio UI, API contract change, provider fetching, model auto-provisioning or app-side model training behavior was introduced. + ## Sprint 129 Operator YOLO training dataset tooling (2026-07-07) - Added `scripts/export_operator_yolo_dataset.py` to convert prepared operator orthophoto/GRB sample pairs into a standard local YOLO detection dataset with `dataset.yaml`, train/validation image folders, label folders and `yolo_dataset_summary.json`. diff --git a/backend/README.md b/backend/README.md index 072dca3d..61443ec9 100644 --- a/backend/README.md +++ b/backend/README.md @@ -311,6 +311,25 @@ explicit operator sample manifest. The training wrapper writes `training_summary.json` and a local `.pt` artifact, which still must be validated through model preflight and the real-data QA matrix before use. +For a larger tile-level training set, use overlapping windows instead of one +image per AOI: + +```bash +docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \ + --manifest-path /app/storage/operator-data/operator_samples_manifest.json \ + --output-dir /app/storage/operator-data/yolo-building-tile-dataset \ + --tile-size 192 \ + --stride 96 \ + --negative-keep-ratio 0.5 \ + --val-samples turnhout \ + --force +``` + +Then point `OPERATOR_YOLO_DATASET_DIR` at +`/app/storage/operator-data/yolo-building-tile-dataset` and keep the same +training wrapper. Tile-level output remains operator tooling outside the V1 +browser product. + The backend also exposes a read-only model asset catalog for the mounted model directory: diff --git a/backend/tests/test_sprint130_operator_yolo_tile_dataset.py b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py new file mode 100644 index 00000000..ee58a955 --- /dev/null +++ b/backend/tests/test_sprint130_operator_yolo_tile_dataset.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parents[2] + + +def load_tile_exporter(): + script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py" + spec = importlib.util.spec_from_file_location("operator_tile_exporter", script_path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_operator_yolo_tile_dataset_export_script_contract() -> None: + script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py" + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert script_path.exists() + script = script_path.read_text(encoding="utf-8") + + assert "py_compile scripts/export_operator_yolo_tile_dataset.py" in readiness + assert "operator_samples_manifest.json" in script + assert "yolo-building-tile-dataset" in script + assert "dataset.yaml" in script + assert "images/train" in script + assert "labels/train" in script + assert "images/val" in script + assert "labels/val" in script + assert "tile_size" in script + assert "stride" in script + assert "negative_keep_ratio" in script + assert "positive_tile_count" in script + assert "negative_tile_count" in script + assert "skipped_negative_tile_count" in script + assert "source_name" in script + assert "reference_layer_name" in script + assert "Window" in script + assert "Transformer" in script + assert "fixture_mode" not in script + assert "will_download_models" not in script + + +def test_operator_yolo_tile_dataset_export_help_does_not_require_gis_dependencies() -> None: + script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py" + + result = subprocess.run( + [sys.executable, str(script_path), "--help"], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0 + assert "Export operator real-data samples to a tile-level YOLO detection dataset" in result.stdout + assert "--tile-size" in result.stdout + assert "--stride" in result.stdout + assert "--negative-keep-ratio" in result.stdout + + +def test_iter_tile_windows_covers_edges_without_duplicates() -> None: + module = load_tile_exporter() + + windows = list(module.iter_tile_windows(width=512, height=512, tile_size=192, stride=96)) + + assert len(windows) == 25 + assert windows[0].row_off == 0 + assert windows[0].col_off == 0 + assert windows[-1].row_off == 320 + assert windows[-1].col_off == 320 + assert len({(window.row_off, window.col_off) for window in windows}) == len(windows) + assert all(window.width == 192 for window in windows) + assert all(window.height == 192 for window in windows) + + +def test_negative_tile_keep_is_deterministic_and_ratio_bound() -> None: + module = load_tile_exporter() + + first = [module.keep_negative_tile("geel", index, 0.25) for index in range(50)] + second = [module.keep_negative_tile("geel", index, 0.25) for index in range(50)] + all_kept = [module.keep_negative_tile("geel", index, 1.0) for index in range(10)] + none_kept = [module.keep_negative_tile("geel", index, 0.0) for index in range(10)] + + assert first == second + assert 1 <= sum(first) <= 25 + assert all(all_kept) + assert not any(none_kept) diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index 212ba7aa..b7a29dda 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -266,6 +266,25 @@ This remains operator tooling only. GeoIntel does not expose Training Studio in V1, does not generate labels from predictions and does not treat the trained artifact as useful until it passes the same real-data Detection + QA matrix. +If the whole-image dataset underfits or produces unusable detections, export +overlapping tile-level samples: + +```bash +docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \ + --manifest-path /app/storage/operator-data/operator_samples_manifest.json \ + --output-dir /app/storage/operator-data/yolo-building-tile-dataset \ + --tile-size 192 \ + --stride 96 \ + --negative-keep-ratio 0.5 \ + --val-samples turnhout \ + --force +``` + +The tile exporter clips reference building boxes into tile-local YOLO labels +and records the positive/negative tile counts. This gives the training smoke +more image samples while preserving the same explicit operator-data and QA/QC +validation boundary. + For visual error inspection, export the persisted QA evidence from a calibration summary: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index bbaef0b4..cd1d3435 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,3 +1,28 @@ +## Sprint 130 Operator YOLO tile-level dataset tooling (2026-07-07) + +Changed: +- Added `scripts/export_operator_yolo_tile_dataset.py`. +- The exporter reads `operator_samples_manifest.json`, opens each raster/reference pair, creates overlapping tile windows, clips GRB building bounding boxes into tile-local YOLO labels, writes `dataset.yaml`, and reports `yolo_tile_dataset_summary.json`. +- Added deterministic negative tile retention through `negative_keep_ratio`. +- Added readiness compile coverage for the tile exporter. +- Added regression coverage in `backend/tests/test_sprint130_operator_yolo_tile_dataset.py`. +- Updated `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`. + +Tested: +- RED: `python -m pytest backend\tests\test_sprint130_operator_yolo_tile_dataset.py -q` failed while `scripts/export_operator_yolo_tile_dataset.py` did not exist. +- `python -m pytest backend\tests\test_sprint130_operator_yolo_tile_dataset.py -q` passed. +- `python scripts\export_operator_yolo_tile_dataset.py --help` passed without requiring local GIS dependencies. +- `python -m py_compile scripts\export_operator_yolo_tile_dataset.py` passed. + +Open: +- Run the tile exporter inside the AI-enabled Tower runtime, train a local tile-level model, and benchmark it through the existing multi-sample Detection + QA matrix. + +Limitations: +- This remains operator tooling only. It does not add Training Studio, browser training controls, provider fetching, fake detections, model auto-provisioning or API contract changes. + +Next recommended pass: +- Export the tile-level dataset on Tower with `tile-size=192`, `stride=96`, train a longer local model, and compare it against the current `yolov8s-building-segmentation-pt` baseline. + ## Sprint 129 Operator YOLO training dataset tooling (2026-07-07) Changed: diff --git a/docs/TODO.md b/docs/TODO.md index 068b36f5..3c4fbc88 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -406,4 +406,6 @@ This file now starts with the current implementation status. Older preparation/b - [x] Harden configured YOLO inference for single-band raster tiles and wrapped runtime errors. - [x] Add operator-only YOLO dataset export and local training-smoke wrapper for real sample calibration. - [x] Train/evaluate a small local GeoIntel building-detector smoke from the current operator samples and reject it because QA/QC did not improve. -- [ ] Build a larger tile-level training dataset with more AOIs, positive/negative tiles and validation splits before the next local model attempt. +- [x] Add operator-only tile-level YOLO dataset export with overlapping windows and deterministic negative tile retention. +- [ ] Run tile-level training on Tower and accept/reject the resulting local model through the persisted QA/QC matrix. +- [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout. diff --git a/scripts/README.md b/scripts/README.md index 62c15fab..b4432dac 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -302,6 +302,44 @@ the resulting `.pt` file like any other local model asset: verify preflight, run the real-data matrix and compare persisted QA/QC metrics before activating it as a useful default. +When whole-image training does not improve QA/QC, export a tile-level dataset +with overlapping raster windows: + +```bash +docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \ + --manifest-path /app/storage/operator-data/operator_samples_manifest.json \ + --output-dir /app/storage/operator-data/yolo-building-tile-dataset \ + --tile-size 192 \ + --stride 96 \ + --negative-keep-ratio 0.5 \ + --val-samples turnhout \ + --force +``` + +The tile exporter clips GRB building bounding boxes into each tile, writes +YOLO labels beside each tile image, keeps a deterministic ratio of empty +negative tiles, and records `yolo_tile_dataset_summary.json` with +`positive_tile_count`, `negative_tile_count` and skipped negative tile counts. +It remains operator tooling only: no provider fetch, no API mutation and no +automatic model training. + +Train against the tile dataset by pointing the existing wrapper at the tile +output directory: + +```bash +docker exec \ + -e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-tile-dataset \ + -e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \ + -e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-tile-detector.pt \ + -e TRAIN_EPOCHS=30 \ + -e TRAIN_IMGSZ=256 \ + -e TRAIN_BATCH=4 \ + -e TRAIN_WORKERS=0 \ + -e TRAIN_DEVICE=cpu \ + -e PYTHON_BIN=python3 \ + geointel bash /app/scripts/train_operator_yolo_detector.sh +``` + Export calibration QA evidence for visual review: ```bash diff --git a/scripts/export_operator_yolo_tile_dataset.py b/scripts/export_operator_yolo_tile_dataset.py new file mode 100644 index 00000000..5b7b28b4 --- /dev/null +++ b/scripts/export_operator_yolo_tile_dataset.py @@ -0,0 +1,396 @@ +"""Export operator real-data samples to a tile-level YOLO detection dataset. + +This operator/runtime helper turns the prepared orthophoto + GRB reference +sample manifest into overlapping raster tiles with YOLO labels. It does not +call GeoIntel APIs, does not train automatically and does not fetch provider +data. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +import hashlib +import json +import os +import shutil +import sys +from pathlib import Path +from typing import Any, Iterable + + +DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json") +DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset") +rasterio: Any = None +Window: Any = None +Transformer: Any = None +Image: Any = None + + +@dataclass(frozen=True) +class TileWindow: + row_off: int + col_off: int + height: int + width: int + + +@dataclass(frozen=True) +class PixelBox: + min_col: float + min_row: float + max_col: float + max_row: float + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Export operator real-data samples to a tile-level YOLO detection dataset.", + ) + parser.add_argument( + "--manifest-path", + type=Path, + default=Path(os.environ.get("OPERATOR_SAMPLE_MANIFEST_PATH", DEFAULT_MANIFEST_PATH)), + help="operator_samples_manifest.json created by prepare_operator_real_data_samples.py.", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path(os.environ.get("OPERATOR_YOLO_TILE_DATASET_DIR", DEFAULT_OUTPUT_DIR)), + help="Output directory for tile images, labels, dataset.yaml and summary JSON.", + ) + parser.add_argument("--tile-size", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_SIZE", "256"))) + parser.add_argument("--stride", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_STRIDE", "128"))) + parser.add_argument( + "--val-samples", + default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"), + help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.", + ) + parser.add_argument( + "--negative-keep-ratio", + type=float, + default=float(os.environ.get("OPERATOR_YOLO_NEGATIVE_KEEP_RATIO", "0.35")), + help="Deterministic ratio of empty tiles to keep as negative examples.", + ) + parser.add_argument( + "--min-label-px", + type=float, + default=float(os.environ.get("OPERATOR_YOLO_MIN_LABEL_PX", "4")), + help="Minimum clipped box width/height in pixels before a tile label is kept.", + ) + parser.add_argument("--force", action="store_true", help="Remove and recreate output-dir before exporting.") + return parser.parse_args() + + +def ensure_dependencies() -> None: + global Image, Transformer, Window, rasterio + try: + import rasterio as rasterio_module + from PIL import Image as image_module + from pyproj import Transformer as transformer_class + from rasterio.windows import Window as window_class + except Exception as exc: # pragma: no cover - runtime environment only. + raise SystemExit( + "export_operator_yolo_tile_dataset.py requires rasterio, pyproj and Pillow. " + "Run it inside the GeoIntel all-in-one container or an equivalent GIS Python environment." + ) from exc + rasterio = rasterio_module + Window = window_class + Transformer = transformer_class + Image = image_module + + +def split_slugs(raw: str) -> set[str]: + return {value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()} + + +def edge_starts(length: int, tile_size: int, stride: int) -> list[int]: + if tile_size <= 0: + raise ValueError("tile_size must be positive") + if stride <= 0: + raise ValueError("stride must be positive") + if length <= tile_size: + return [0] + starts = list(range(0, max(length - tile_size, 0) + 1, stride)) + final_start = length - tile_size + if starts[-1] != final_start: + starts.append(final_start) + return starts + + +def iter_tile_windows(width: int, height: int, tile_size: int, stride: int) -> Iterable[TileWindow]: + for row_off in edge_starts(height, tile_size, stride): + for col_off in edge_starts(width, tile_size, stride): + yield TileWindow(row_off=row_off, col_off=col_off, height=min(tile_size, height), width=min(tile_size, width)) + + +def keep_negative_tile(sample_slug: str, tile_index: int, negative_keep_ratio: float) -> bool: + if negative_keep_ratio <= 0: + return False + if negative_keep_ratio >= 1: + return True + digest = hashlib.sha256(f"{sample_slug}:{tile_index}".encode("utf-8")).hexdigest() + bucket = int(digest[:8], 16) / 0xFFFFFFFF + return bucket < negative_keep_ratio + + +def resolve_manifest_path(raw: str, manifest_path: Path) -> Path: + path = Path(raw) + if path.exists(): + return path + if raw.startswith("/app/"): + relative = Path(raw.removeprefix("/app/")) + candidates = [ + Path.cwd() / relative, + manifest_path.resolve().parent.parent.parent / relative, + ] + for candidate in candidates: + if candidate.exists(): + return candidate + return path + + +def iter_geometry_coords(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]: + geometry_type = geometry.get("type") + coordinates = geometry.get("coordinates") + if geometry_type == "Polygon": + for ring in coordinates or []: + for point in ring: + if len(point) >= 2: + yield float(point[0]), float(point[1]) + elif geometry_type == "MultiPolygon": + for polygon in coordinates or []: + for ring in polygon: + for point in ring: + if len(point) >= 2: + yield float(point[0]), float(point[1]) + + +def load_reference_pixel_boxes(reference_path: Path, dataset: Any, min_label_px: float) -> list[PixelBox]: + reference = json.loads(reference_path.read_text(encoding="utf-8-sig")) + features = reference.get("features") or [] + transformer = Transformer.from_crs("EPSG:4326", dataset.crs, always_xy=True) + boxes: list[PixelBox] = [] + + for feature in features: + properties = feature.get("properties") or {} + if properties.get("source_name") != "grb": + continue + if properties.get("reference_layer_name") != "buildings": + continue + coords = list(iter_geometry_coords(feature.get("geometry") or {})) + if not coords: + continue + xs, ys = zip(*(transformer.transform(lon, lat) for lon, lat in coords), strict=False) + rows_cols = [dataset.index(x, y) for x, y in zip(xs, ys, strict=False)] + rows = [row for row, _ in rows_cols] + cols = [col for _, col in rows_cols] + min_col = max(0, min(cols)) + max_col = min(dataset.width - 1, max(cols)) + min_row = max(0, min(rows)) + max_row = min(dataset.height - 1, max(rows)) + if max_col - min_col < min_label_px or max_row - min_row < min_label_px: + continue + boxes.append(PixelBox(min_col=min_col, min_row=min_row, max_col=max_col, max_row=max_row)) + return boxes + + +def labels_for_tile(tile_window: TileWindow, boxes: list[PixelBox], min_label_px: float) -> list[str]: + labels: list[str] = [] + tile_min_col = tile_window.col_off + tile_min_row = tile_window.row_off + tile_max_col = tile_window.col_off + tile_window.width + tile_max_row = tile_window.row_off + tile_window.height + + for box in boxes: + min_col = max(box.min_col, tile_min_col) + max_col = min(box.max_col, tile_max_col) + min_row = max(box.min_row, tile_min_row) + max_row = min(box.max_row, tile_max_row) + box_width = max_col - min_col + box_height = max_row - min_row + if box_width < min_label_px or box_height < min_label_px: + continue + local_min_col = min_col - tile_min_col + local_max_col = max_col - tile_min_col + local_min_row = min_row - tile_min_row + local_max_row = max_row - tile_min_row + x_center = (local_min_col + local_max_col) / 2.0 / tile_window.width + y_center = (local_min_row + local_max_row) / 2.0 / tile_window.height + norm_width = box_width / tile_window.width + norm_height = box_height / tile_window.height + labels.append(f"0 {x_center:.8f} {y_center:.8f} {norm_width:.8f} {norm_height:.8f}") + return labels + + +def image_array_from_raster_window(dataset: Any, tile_window: TileWindow) -> Any: + import numpy as np + + data = dataset.read(window=Window(tile_window.col_off, tile_window.row_off, tile_window.width, tile_window.height)) + if data.shape[0] == 1: + rgb = np.repeat(data[:1], 3, axis=0) + else: + rgb = data[:3] + rgb = np.moveaxis(rgb, 0, -1) + if rgb.dtype != np.uint8: + rgb_min = float(np.nanmin(rgb)) + rgb_max = float(np.nanmax(rgb)) + if rgb_max > rgb_min: + rgb = ((rgb - rgb_min) / (rgb_max - rgb_min) * 255.0).clip(0, 255).astype("uint8") + else: + rgb = np.zeros(rgb.shape, dtype="uint8") + return rgb + + +def ensure_yolo_directories(output_dir: Path) -> None: + for relative_path in ("images/train", "labels/train", "images/val", "labels/val"): + (output_dir / relative_path).mkdir(parents=True, exist_ok=True) + + +def write_dataset_yaml(output_dir: Path) -> Path: + yaml_path = output_dir / "dataset.yaml" + yaml_path.write_text( + "\n".join( + [ + f"path: {output_dir}", + "train: images/train", + "val: images/val", + "names:", + " 0: building", + "", + ] + ), + encoding="utf-8", + ) + return yaml_path + + +def export_sample_tiles( + sample: dict[str, Any], + manifest_path: Path, + output_dir: Path, + val_slugs: set[str], + tile_size: int, + stride: int, + negative_keep_ratio: float, + min_label_px: float, +) -> list[dict[str, Any]]: + sample_slug = str(sample["sample_slug"]) + split = "val" if sample_slug.lower() in val_slugs else "train" + raster_path = resolve_manifest_path(str(sample["raster_path"]), manifest_path) + reference_path = resolve_manifest_path(str(sample["reference_path"]), manifest_path) + if not raster_path.exists(): + raise SystemExit(f"Raster path is not readable for sample {sample_slug}: {raster_path}") + if not reference_path.exists(): + raise SystemExit(f"Reference path is not readable for sample {sample_slug}: {reference_path}") + + exported: list[dict[str, Any]] = [] + with rasterio.open(raster_path) as dataset: + boxes = load_reference_pixel_boxes(reference_path, dataset, min_label_px=min_label_px) + for tile_index, tile_window in enumerate(iter_tile_windows(dataset.width, dataset.height, tile_size, stride)): + labels = labels_for_tile(tile_window, boxes, min_label_px=min_label_px) + is_negative = not labels + if is_negative and not keep_negative_tile(sample_slug, tile_index, negative_keep_ratio): + exported.append( + { + "sample_slug": sample_slug, + "split": split, + "tile_index": tile_index, + "kept": False, + "label_count": 0, + "is_negative": True, + } + ) + continue + tile_name = f"{sample_slug}_{tile_index:04d}_r{tile_window.row_off}_c{tile_window.col_off}" + image_path = output_dir / "images" / split / f"{tile_name}.png" + label_path = output_dir / "labels" / split / f"{tile_name}.txt" + image_path.parent.mkdir(parents=True, exist_ok=True) + label_path.parent.mkdir(parents=True, exist_ok=True) + Image.fromarray(image_array_from_raster_window(dataset, tile_window)).save(image_path) + label_path.write_text("\n".join(labels) + ("\n" if labels else ""), encoding="utf-8") + exported.append( + { + "sample_slug": sample_slug, + "split": split, + "tile_index": tile_index, + "kept": True, + "image_path": str(image_path), + "label_path": str(label_path), + "label_count": len(labels), + "is_negative": is_negative, + "window": { + "row_off": tile_window.row_off, + "col_off": tile_window.col_off, + "height": tile_window.height, + "width": tile_window.width, + }, + } + ) + return exported + + +def main() -> int: + args = parse_args() + ensure_dependencies() + if args.force and args.output_dir.exists(): + shutil.rmtree(args.output_dir) + args.output_dir.mkdir(parents=True, exist_ok=True) + ensure_yolo_directories(args.output_dir) + + manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig")) + samples = manifest.get("samples") or [] + if not samples: + raise SystemExit("Operator sample manifest contains no samples") + val_slugs = split_slugs(args.val_samples) + exported_tiles: list[dict[str, Any]] = [] + for sample in samples: + exported_tiles.extend( + export_sample_tiles( + sample=sample, + manifest_path=args.manifest_path, + output_dir=args.output_dir, + val_slugs=val_slugs, + tile_size=args.tile_size, + stride=args.stride, + negative_keep_ratio=args.negative_keep_ratio, + min_label_px=args.min_label_px, + ) + ) + + kept_tiles = [tile for tile in exported_tiles if tile["kept"]] + if not any(tile["split"] == "train" for tile in kept_tiles): + raise SystemExit("YOLO tile dataset export produced no training tiles") + if not any(tile["split"] == "val" for tile in kept_tiles): + raise SystemExit("YOLO tile dataset export produced no validation tiles") + dataset_yaml = write_dataset_yaml(args.output_dir) + positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]] + negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]] + skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]] + summary = { + "status": "ok", + "dataset_yaml": str(dataset_yaml), + "output_dir": str(args.output_dir), + "class_names": ["building"], + "tile_size": args.tile_size, + "stride": args.stride, + "negative_keep_ratio": args.negative_keep_ratio, + "min_label_px": args.min_label_px, + "source_sample_count": len(samples), + "tile_count": len(kept_tiles), + "positive_tile_count": len(positive_tiles), + "negative_tile_count": len(negative_tiles), + "skipped_negative_tile_count": len(skipped_negative_tiles), + "label_count": sum(tile["label_count"] for tile in kept_tiles), + "train_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "train"), + "val_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "val"), + "tiles": kept_tiles, + } + summary_path = args.output_dir / "yolo_tile_dataset_summary.json" + summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 1d7615c0..29c601dc 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -43,6 +43,7 @@ ${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py ${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py ${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py +${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m compileall backend/app