Add operator YOLO tile dataset exporter
GeoIntel CI / docs-smoke (push) Has been cancelled
GeoIntel CI / contract-smoke (push) Has been cancelled

This commit is contained in:
Codex
2026-07-07 21:02:09 +02:00
parent 404b37fa07
commit 1d27e4a059
9 changed files with 604 additions and 1 deletions
+8
View File
@@ -7,6 +7,14 @@
# Changelog # Changelog
## Sprint 130 Operator YOLO tile-level dataset tooling (2026-07-07)
- Added `scripts/export_operator_yolo_tile_dataset.py` to convert prepared operator samples into overlapping YOLO tile datasets with clipped building labels and deterministic negative tile retention.
- Added readiness coverage for the tile exporter Python compile check.
- Added regression coverage in `backend/tests/test_sprint130_operator_yolo_tile_dataset.py` for script contract, help behavior without GIS imports, edge-covering tile windows and deterministic negative-tile selection.
- Updated operator documentation for tile-level dataset export and reuse of the existing local training wrapper.
- No Training Studio UI, API contract change, provider fetching, model auto-provisioning or app-side model training behavior was introduced.
## Sprint 129 Operator YOLO training dataset tooling (2026-07-07) ## Sprint 129 Operator YOLO training dataset tooling (2026-07-07)
- Added `scripts/export_operator_yolo_dataset.py` to convert prepared operator orthophoto/GRB sample pairs into a standard local YOLO detection dataset with `dataset.yaml`, train/validation image folders, label folders and `yolo_dataset_summary.json`. - Added `scripts/export_operator_yolo_dataset.py` to convert prepared operator orthophoto/GRB sample pairs into a standard local YOLO detection dataset with `dataset.yaml`, train/validation image folders, label folders and `yolo_dataset_summary.json`.
+19
View File
@@ -311,6 +311,25 @@ explicit operator sample manifest. The training wrapper writes
`training_summary.json` and a local `.pt` artifact, which still must be `training_summary.json` and a local `.pt` artifact, which still must be
validated through model preflight and the real-data QA matrix before use. validated through model preflight and the real-data QA matrix before use.
For a larger tile-level training set, use overlapping windows instead of one
image per AOI:
```bash
docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
--output-dir /app/storage/operator-data/yolo-building-tile-dataset \
--tile-size 192 \
--stride 96 \
--negative-keep-ratio 0.5 \
--val-samples turnhout \
--force
```
Then point `OPERATOR_YOLO_DATASET_DIR` at
`/app/storage/operator-data/yolo-building-tile-dataset` and keep the same
training wrapper. Tile-level output remains operator tooling outside the V1
browser product.
The backend also exposes a read-only model asset catalog for the mounted model The backend also exposes a read-only model asset catalog for the mounted model
directory: directory:
@@ -0,0 +1,95 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[2]
def load_tile_exporter():
script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py"
spec = importlib.util.spec_from_file_location("operator_tile_exporter", script_path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def test_operator_yolo_tile_dataset_export_script_contract() -> None:
script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py"
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
assert script_path.exists()
script = script_path.read_text(encoding="utf-8")
assert "py_compile scripts/export_operator_yolo_tile_dataset.py" in readiness
assert "operator_samples_manifest.json" in script
assert "yolo-building-tile-dataset" in script
assert "dataset.yaml" in script
assert "images/train" in script
assert "labels/train" in script
assert "images/val" in script
assert "labels/val" in script
assert "tile_size" in script
assert "stride" in script
assert "negative_keep_ratio" in script
assert "positive_tile_count" in script
assert "negative_tile_count" in script
assert "skipped_negative_tile_count" in script
assert "source_name" in script
assert "reference_layer_name" in script
assert "Window" in script
assert "Transformer" in script
assert "fixture_mode" not in script
assert "will_download_models" not in script
def test_operator_yolo_tile_dataset_export_help_does_not_require_gis_dependencies() -> None:
script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py"
result = subprocess.run(
[sys.executable, str(script_path), "--help"],
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0
assert "Export operator real-data samples to a tile-level YOLO detection dataset" in result.stdout
assert "--tile-size" in result.stdout
assert "--stride" in result.stdout
assert "--negative-keep-ratio" in result.stdout
def test_iter_tile_windows_covers_edges_without_duplicates() -> None:
module = load_tile_exporter()
windows = list(module.iter_tile_windows(width=512, height=512, tile_size=192, stride=96))
assert len(windows) == 25
assert windows[0].row_off == 0
assert windows[0].col_off == 0
assert windows[-1].row_off == 320
assert windows[-1].col_off == 320
assert len({(window.row_off, window.col_off) for window in windows}) == len(windows)
assert all(window.width == 192 for window in windows)
assert all(window.height == 192 for window in windows)
def test_negative_tile_keep_is_deterministic_and_ratio_bound() -> None:
module = load_tile_exporter()
first = [module.keep_negative_tile("geel", index, 0.25) for index in range(50)]
second = [module.keep_negative_tile("geel", index, 0.25) for index in range(50)]
all_kept = [module.keep_negative_tile("geel", index, 1.0) for index in range(10)]
none_kept = [module.keep_negative_tile("geel", index, 0.0) for index in range(10)]
assert first == second
assert 1 <= sum(first) <= 25
assert all(all_kept)
assert not any(none_kept)
+19
View File
@@ -266,6 +266,25 @@ This remains operator tooling only. GeoIntel does not expose Training Studio in
V1, does not generate labels from predictions and does not treat the trained V1, does not generate labels from predictions and does not treat the trained
artifact as useful until it passes the same real-data Detection + QA matrix. artifact as useful until it passes the same real-data Detection + QA matrix.
If the whole-image dataset underfits or produces unusable detections, export
overlapping tile-level samples:
```bash
docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
--output-dir /app/storage/operator-data/yolo-building-tile-dataset \
--tile-size 192 \
--stride 96 \
--negative-keep-ratio 0.5 \
--val-samples turnhout \
--force
```
The tile exporter clips reference building boxes into tile-local YOLO labels
and records the positive/negative tile counts. This gives the training smoke
more image samples while preserving the same explicit operator-data and QA/QC
validation boundary.
For visual error inspection, export the persisted QA evidence from a calibration For visual error inspection, export the persisted QA evidence from a calibration
summary: summary:
+25
View File
@@ -1,3 +1,28 @@
## Sprint 130 Operator YOLO tile-level dataset tooling (2026-07-07)
Changed:
- Added `scripts/export_operator_yolo_tile_dataset.py`.
- The exporter reads `operator_samples_manifest.json`, opens each raster/reference pair, creates overlapping tile windows, clips GRB building bounding boxes into tile-local YOLO labels, writes `dataset.yaml`, and reports `yolo_tile_dataset_summary.json`.
- Added deterministic negative tile retention through `negative_keep_ratio`.
- Added readiness compile coverage for the tile exporter.
- Added regression coverage in `backend/tests/test_sprint130_operator_yolo_tile_dataset.py`.
- Updated `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`.
Tested:
- RED: `python -m pytest backend\tests\test_sprint130_operator_yolo_tile_dataset.py -q` failed while `scripts/export_operator_yolo_tile_dataset.py` did not exist.
- `python -m pytest backend\tests\test_sprint130_operator_yolo_tile_dataset.py -q` passed.
- `python scripts\export_operator_yolo_tile_dataset.py --help` passed without requiring local GIS dependencies.
- `python -m py_compile scripts\export_operator_yolo_tile_dataset.py` passed.
Open:
- Run the tile exporter inside the AI-enabled Tower runtime, train a local tile-level model, and benchmark it through the existing multi-sample Detection + QA matrix.
Limitations:
- This remains operator tooling only. It does not add Training Studio, browser training controls, provider fetching, fake detections, model auto-provisioning or API contract changes.
Next recommended pass:
- Export the tile-level dataset on Tower with `tile-size=192`, `stride=96`, train a longer local model, and compare it against the current `yolov8s-building-segmentation-pt` baseline.
## Sprint 129 Operator YOLO training dataset tooling (2026-07-07) ## Sprint 129 Operator YOLO training dataset tooling (2026-07-07)
Changed: Changed:
+3 -1
View File
@@ -406,4 +406,6 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Harden configured YOLO inference for single-band raster tiles and wrapped runtime errors. - [x] Harden configured YOLO inference for single-band raster tiles and wrapped runtime errors.
- [x] Add operator-only YOLO dataset export and local training-smoke wrapper for real sample calibration. - [x] Add operator-only YOLO dataset export and local training-smoke wrapper for real sample calibration.
- [x] Train/evaluate a small local GeoIntel building-detector smoke from the current operator samples and reject it because QA/QC did not improve. - [x] Train/evaluate a small local GeoIntel building-detector smoke from the current operator samples and reject it because QA/QC did not improve.
- [ ] Build a larger tile-level training dataset with more AOIs, positive/negative tiles and validation splits before the next local model attempt. - [x] Add operator-only tile-level YOLO dataset export with overlapping windows and deterministic negative tile retention.
- [ ] Run tile-level training on Tower and accept/reject the resulting local model through the persisted QA/QC matrix.
- [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout.
+38
View File
@@ -302,6 +302,44 @@ the resulting `.pt` file like any other local model asset: verify preflight,
run the real-data matrix and compare persisted QA/QC metrics before activating run the real-data matrix and compare persisted QA/QC metrics before activating
it as a useful default. it as a useful default.
When whole-image training does not improve QA/QC, export a tile-level dataset
with overlapping raster windows:
```bash
docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
--output-dir /app/storage/operator-data/yolo-building-tile-dataset \
--tile-size 192 \
--stride 96 \
--negative-keep-ratio 0.5 \
--val-samples turnhout \
--force
```
The tile exporter clips GRB building bounding boxes into each tile, writes
YOLO labels beside each tile image, keeps a deterministic ratio of empty
negative tiles, and records `yolo_tile_dataset_summary.json` with
`positive_tile_count`, `negative_tile_count` and skipped negative tile counts.
It remains operator tooling only: no provider fetch, no API mutation and no
automatic model training.
Train against the tile dataset by pointing the existing wrapper at the tile
output directory:
```bash
docker exec \
-e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-tile-dataset \
-e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \
-e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-tile-detector.pt \
-e TRAIN_EPOCHS=30 \
-e TRAIN_IMGSZ=256 \
-e TRAIN_BATCH=4 \
-e TRAIN_WORKERS=0 \
-e TRAIN_DEVICE=cpu \
-e PYTHON_BIN=python3 \
geointel bash /app/scripts/train_operator_yolo_detector.sh
```
Export calibration QA evidence for visual review: Export calibration QA evidence for visual review:
```bash ```bash
@@ -0,0 +1,396 @@
"""Export operator real-data samples to a tile-level YOLO detection dataset.
This operator/runtime helper turns the prepared orthophoto + GRB reference
sample manifest into overlapping raster tiles with YOLO labels. It does not
call GeoIntel APIs, does not train automatically and does not fetch provider
data.
"""
from __future__ import annotations
import argparse
from dataclasses import dataclass
import hashlib
import json
import os
import shutil
import sys
from pathlib import Path
from typing import Any, Iterable
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset")
rasterio: Any = None
Window: Any = None
Transformer: Any = None
Image: Any = None
@dataclass(frozen=True)
class TileWindow:
row_off: int
col_off: int
height: int
width: int
@dataclass(frozen=True)
class PixelBox:
min_col: float
min_row: float
max_col: float
max_row: float
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Export operator real-data samples to a tile-level YOLO detection dataset.",
)
parser.add_argument(
"--manifest-path",
type=Path,
default=Path(os.environ.get("OPERATOR_SAMPLE_MANIFEST_PATH", DEFAULT_MANIFEST_PATH)),
help="operator_samples_manifest.json created by prepare_operator_real_data_samples.py.",
)
parser.add_argument(
"--output-dir",
type=Path,
default=Path(os.environ.get("OPERATOR_YOLO_TILE_DATASET_DIR", DEFAULT_OUTPUT_DIR)),
help="Output directory for tile images, labels, dataset.yaml and summary JSON.",
)
parser.add_argument("--tile-size", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_SIZE", "256")))
parser.add_argument("--stride", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_STRIDE", "128")))
parser.add_argument(
"--val-samples",
default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"),
help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.",
)
parser.add_argument(
"--negative-keep-ratio",
type=float,
default=float(os.environ.get("OPERATOR_YOLO_NEGATIVE_KEEP_RATIO", "0.35")),
help="Deterministic ratio of empty tiles to keep as negative examples.",
)
parser.add_argument(
"--min-label-px",
type=float,
default=float(os.environ.get("OPERATOR_YOLO_MIN_LABEL_PX", "4")),
help="Minimum clipped box width/height in pixels before a tile label is kept.",
)
parser.add_argument("--force", action="store_true", help="Remove and recreate output-dir before exporting.")
return parser.parse_args()
def ensure_dependencies() -> None:
global Image, Transformer, Window, rasterio
try:
import rasterio as rasterio_module
from PIL import Image as image_module
from pyproj import Transformer as transformer_class
from rasterio.windows import Window as window_class
except Exception as exc: # pragma: no cover - runtime environment only.
raise SystemExit(
"export_operator_yolo_tile_dataset.py requires rasterio, pyproj and Pillow. "
"Run it inside the GeoIntel all-in-one container or an equivalent GIS Python environment."
) from exc
rasterio = rasterio_module
Window = window_class
Transformer = transformer_class
Image = image_module
def split_slugs(raw: str) -> set[str]:
return {value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()}
def edge_starts(length: int, tile_size: int, stride: int) -> list[int]:
if tile_size <= 0:
raise ValueError("tile_size must be positive")
if stride <= 0:
raise ValueError("stride must be positive")
if length <= tile_size:
return [0]
starts = list(range(0, max(length - tile_size, 0) + 1, stride))
final_start = length - tile_size
if starts[-1] != final_start:
starts.append(final_start)
return starts
def iter_tile_windows(width: int, height: int, tile_size: int, stride: int) -> Iterable[TileWindow]:
for row_off in edge_starts(height, tile_size, stride):
for col_off in edge_starts(width, tile_size, stride):
yield TileWindow(row_off=row_off, col_off=col_off, height=min(tile_size, height), width=min(tile_size, width))
def keep_negative_tile(sample_slug: str, tile_index: int, negative_keep_ratio: float) -> bool:
if negative_keep_ratio <= 0:
return False
if negative_keep_ratio >= 1:
return True
digest = hashlib.sha256(f"{sample_slug}:{tile_index}".encode("utf-8")).hexdigest()
bucket = int(digest[:8], 16) / 0xFFFFFFFF
return bucket < negative_keep_ratio
def resolve_manifest_path(raw: str, manifest_path: Path) -> Path:
path = Path(raw)
if path.exists():
return path
if raw.startswith("/app/"):
relative = Path(raw.removeprefix("/app/"))
candidates = [
Path.cwd() / relative,
manifest_path.resolve().parent.parent.parent / relative,
]
for candidate in candidates:
if candidate.exists():
return candidate
return path
def iter_geometry_coords(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
geometry_type = geometry.get("type")
coordinates = geometry.get("coordinates")
if geometry_type == "Polygon":
for ring in coordinates or []:
for point in ring:
if len(point) >= 2:
yield float(point[0]), float(point[1])
elif geometry_type == "MultiPolygon":
for polygon in coordinates or []:
for ring in polygon:
for point in ring:
if len(point) >= 2:
yield float(point[0]), float(point[1])
def load_reference_pixel_boxes(reference_path: Path, dataset: Any, min_label_px: float) -> list[PixelBox]:
reference = json.loads(reference_path.read_text(encoding="utf-8-sig"))
features = reference.get("features") or []
transformer = Transformer.from_crs("EPSG:4326", dataset.crs, always_xy=True)
boxes: list[PixelBox] = []
for feature in features:
properties = feature.get("properties") or {}
if properties.get("source_name") != "grb":
continue
if properties.get("reference_layer_name") != "buildings":
continue
coords = list(iter_geometry_coords(feature.get("geometry") or {}))
if not coords:
continue
xs, ys = zip(*(transformer.transform(lon, lat) for lon, lat in coords), strict=False)
rows_cols = [dataset.index(x, y) for x, y in zip(xs, ys, strict=False)]
rows = [row for row, _ in rows_cols]
cols = [col for _, col in rows_cols]
min_col = max(0, min(cols))
max_col = min(dataset.width - 1, max(cols))
min_row = max(0, min(rows))
max_row = min(dataset.height - 1, max(rows))
if max_col - min_col < min_label_px or max_row - min_row < min_label_px:
continue
boxes.append(PixelBox(min_col=min_col, min_row=min_row, max_col=max_col, max_row=max_row))
return boxes
def labels_for_tile(tile_window: TileWindow, boxes: list[PixelBox], min_label_px: float) -> list[str]:
labels: list[str] = []
tile_min_col = tile_window.col_off
tile_min_row = tile_window.row_off
tile_max_col = tile_window.col_off + tile_window.width
tile_max_row = tile_window.row_off + tile_window.height
for box in boxes:
min_col = max(box.min_col, tile_min_col)
max_col = min(box.max_col, tile_max_col)
min_row = max(box.min_row, tile_min_row)
max_row = min(box.max_row, tile_max_row)
box_width = max_col - min_col
box_height = max_row - min_row
if box_width < min_label_px or box_height < min_label_px:
continue
local_min_col = min_col - tile_min_col
local_max_col = max_col - tile_min_col
local_min_row = min_row - tile_min_row
local_max_row = max_row - tile_min_row
x_center = (local_min_col + local_max_col) / 2.0 / tile_window.width
y_center = (local_min_row + local_max_row) / 2.0 / tile_window.height
norm_width = box_width / tile_window.width
norm_height = box_height / tile_window.height
labels.append(f"0 {x_center:.8f} {y_center:.8f} {norm_width:.8f} {norm_height:.8f}")
return labels
def image_array_from_raster_window(dataset: Any, tile_window: TileWindow) -> Any:
import numpy as np
data = dataset.read(window=Window(tile_window.col_off, tile_window.row_off, tile_window.width, tile_window.height))
if data.shape[0] == 1:
rgb = np.repeat(data[:1], 3, axis=0)
else:
rgb = data[:3]
rgb = np.moveaxis(rgb, 0, -1)
if rgb.dtype != np.uint8:
rgb_min = float(np.nanmin(rgb))
rgb_max = float(np.nanmax(rgb))
if rgb_max > rgb_min:
rgb = ((rgb - rgb_min) / (rgb_max - rgb_min) * 255.0).clip(0, 255).astype("uint8")
else:
rgb = np.zeros(rgb.shape, dtype="uint8")
return rgb
def ensure_yolo_directories(output_dir: Path) -> None:
for relative_path in ("images/train", "labels/train", "images/val", "labels/val"):
(output_dir / relative_path).mkdir(parents=True, exist_ok=True)
def write_dataset_yaml(output_dir: Path) -> Path:
yaml_path = output_dir / "dataset.yaml"
yaml_path.write_text(
"\n".join(
[
f"path: {output_dir}",
"train: images/train",
"val: images/val",
"names:",
" 0: building",
"",
]
),
encoding="utf-8",
)
return yaml_path
def export_sample_tiles(
sample: dict[str, Any],
manifest_path: Path,
output_dir: Path,
val_slugs: set[str],
tile_size: int,
stride: int,
negative_keep_ratio: float,
min_label_px: float,
) -> list[dict[str, Any]]:
sample_slug = str(sample["sample_slug"])
split = "val" if sample_slug.lower() in val_slugs else "train"
raster_path = resolve_manifest_path(str(sample["raster_path"]), manifest_path)
reference_path = resolve_manifest_path(str(sample["reference_path"]), manifest_path)
if not raster_path.exists():
raise SystemExit(f"Raster path is not readable for sample {sample_slug}: {raster_path}")
if not reference_path.exists():
raise SystemExit(f"Reference path is not readable for sample {sample_slug}: {reference_path}")
exported: list[dict[str, Any]] = []
with rasterio.open(raster_path) as dataset:
boxes = load_reference_pixel_boxes(reference_path, dataset, min_label_px=min_label_px)
for tile_index, tile_window in enumerate(iter_tile_windows(dataset.width, dataset.height, tile_size, stride)):
labels = labels_for_tile(tile_window, boxes, min_label_px=min_label_px)
is_negative = not labels
if is_negative and not keep_negative_tile(sample_slug, tile_index, negative_keep_ratio):
exported.append(
{
"sample_slug": sample_slug,
"split": split,
"tile_index": tile_index,
"kept": False,
"label_count": 0,
"is_negative": True,
}
)
continue
tile_name = f"{sample_slug}_{tile_index:04d}_r{tile_window.row_off}_c{tile_window.col_off}"
image_path = output_dir / "images" / split / f"{tile_name}.png"
label_path = output_dir / "labels" / split / f"{tile_name}.txt"
image_path.parent.mkdir(parents=True, exist_ok=True)
label_path.parent.mkdir(parents=True, exist_ok=True)
Image.fromarray(image_array_from_raster_window(dataset, tile_window)).save(image_path)
label_path.write_text("\n".join(labels) + ("\n" if labels else ""), encoding="utf-8")
exported.append(
{
"sample_slug": sample_slug,
"split": split,
"tile_index": tile_index,
"kept": True,
"image_path": str(image_path),
"label_path": str(label_path),
"label_count": len(labels),
"is_negative": is_negative,
"window": {
"row_off": tile_window.row_off,
"col_off": tile_window.col_off,
"height": tile_window.height,
"width": tile_window.width,
},
}
)
return exported
def main() -> int:
args = parse_args()
ensure_dependencies()
if args.force and args.output_dir.exists():
shutil.rmtree(args.output_dir)
args.output_dir.mkdir(parents=True, exist_ok=True)
ensure_yolo_directories(args.output_dir)
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
samples = manifest.get("samples") or []
if not samples:
raise SystemExit("Operator sample manifest contains no samples")
val_slugs = split_slugs(args.val_samples)
exported_tiles: list[dict[str, Any]] = []
for sample in samples:
exported_tiles.extend(
export_sample_tiles(
sample=sample,
manifest_path=args.manifest_path,
output_dir=args.output_dir,
val_slugs=val_slugs,
tile_size=args.tile_size,
stride=args.stride,
negative_keep_ratio=args.negative_keep_ratio,
min_label_px=args.min_label_px,
)
)
kept_tiles = [tile for tile in exported_tiles if tile["kept"]]
if not any(tile["split"] == "train" for tile in kept_tiles):
raise SystemExit("YOLO tile dataset export produced no training tiles")
if not any(tile["split"] == "val" for tile in kept_tiles):
raise SystemExit("YOLO tile dataset export produced no validation tiles")
dataset_yaml = write_dataset_yaml(args.output_dir)
positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]]
summary = {
"status": "ok",
"dataset_yaml": str(dataset_yaml),
"output_dir": str(args.output_dir),
"class_names": ["building"],
"tile_size": args.tile_size,
"stride": args.stride,
"negative_keep_ratio": args.negative_keep_ratio,
"min_label_px": args.min_label_px,
"source_sample_count": len(samples),
"tile_count": len(kept_tiles),
"positive_tile_count": len(positive_tiles),
"negative_tile_count": len(negative_tiles),
"skipped_negative_tile_count": len(skipped_negative_tiles),
"label_count": sum(tile["label_count"] for tile in kept_tiles),
"train_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "train"),
"val_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "val"),
"tiles": kept_tiles,
}
summary_path = args.output_dir / "yolo_tile_dataset_summary.json"
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
print(json.dumps(summary, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
sys.exit(main())
+1
View File
@@ -43,6 +43,7 @@ ${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py
${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py ${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py
${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py ${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
${PYTHON_BIN} -m compileall backend/app ${PYTHON_BIN} -m compileall backend/app