Add operator YOLO tile dataset exporter
This commit is contained in:
@@ -7,6 +7,14 @@
|
|||||||
|
|
||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## Sprint 130 Operator YOLO tile-level dataset tooling (2026-07-07)
|
||||||
|
|
||||||
|
- Added `scripts/export_operator_yolo_tile_dataset.py` to convert prepared operator samples into overlapping YOLO tile datasets with clipped building labels and deterministic negative tile retention.
|
||||||
|
- Added readiness coverage for the tile exporter Python compile check.
|
||||||
|
- Added regression coverage in `backend/tests/test_sprint130_operator_yolo_tile_dataset.py` for script contract, help behavior without GIS imports, edge-covering tile windows and deterministic negative-tile selection.
|
||||||
|
- Updated operator documentation for tile-level dataset export and reuse of the existing local training wrapper.
|
||||||
|
- No Training Studio UI, API contract change, provider fetching, model auto-provisioning or app-side model training behavior was introduced.
|
||||||
|
|
||||||
## Sprint 129 Operator YOLO training dataset tooling (2026-07-07)
|
## Sprint 129 Operator YOLO training dataset tooling (2026-07-07)
|
||||||
|
|
||||||
- Added `scripts/export_operator_yolo_dataset.py` to convert prepared operator orthophoto/GRB sample pairs into a standard local YOLO detection dataset with `dataset.yaml`, train/validation image folders, label folders and `yolo_dataset_summary.json`.
|
- Added `scripts/export_operator_yolo_dataset.py` to convert prepared operator orthophoto/GRB sample pairs into a standard local YOLO detection dataset with `dataset.yaml`, train/validation image folders, label folders and `yolo_dataset_summary.json`.
|
||||||
|
|||||||
@@ -311,6 +311,25 @@ explicit operator sample manifest. The training wrapper writes
|
|||||||
`training_summary.json` and a local `.pt` artifact, which still must be
|
`training_summary.json` and a local `.pt` artifact, which still must be
|
||||||
validated through model preflight and the real-data QA matrix before use.
|
validated through model preflight and the real-data QA matrix before use.
|
||||||
|
|
||||||
|
For a larger tile-level training set, use overlapping windows instead of one
|
||||||
|
image per AOI:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \
|
||||||
|
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
|
||||||
|
--output-dir /app/storage/operator-data/yolo-building-tile-dataset \
|
||||||
|
--tile-size 192 \
|
||||||
|
--stride 96 \
|
||||||
|
--negative-keep-ratio 0.5 \
|
||||||
|
--val-samples turnhout \
|
||||||
|
--force
|
||||||
|
```
|
||||||
|
|
||||||
|
Then point `OPERATOR_YOLO_DATASET_DIR` at
|
||||||
|
`/app/storage/operator-data/yolo-building-tile-dataset` and keep the same
|
||||||
|
training wrapper. Tile-level output remains operator tooling outside the V1
|
||||||
|
browser product.
|
||||||
|
|
||||||
The backend also exposes a read-only model asset catalog for the mounted model
|
The backend also exposes a read-only model asset catalog for the mounted model
|
||||||
directory:
|
directory:
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,95 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def load_tile_exporter():
|
||||||
|
script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py"
|
||||||
|
spec = importlib.util.spec_from_file_location("operator_tile_exporter", script_path)
|
||||||
|
assert spec is not None
|
||||||
|
assert spec.loader is not None
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def test_operator_yolo_tile_dataset_export_script_contract() -> None:
|
||||||
|
script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py"
|
||||||
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert script_path.exists()
|
||||||
|
script = script_path.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "py_compile scripts/export_operator_yolo_tile_dataset.py" in readiness
|
||||||
|
assert "operator_samples_manifest.json" in script
|
||||||
|
assert "yolo-building-tile-dataset" in script
|
||||||
|
assert "dataset.yaml" in script
|
||||||
|
assert "images/train" in script
|
||||||
|
assert "labels/train" in script
|
||||||
|
assert "images/val" in script
|
||||||
|
assert "labels/val" in script
|
||||||
|
assert "tile_size" in script
|
||||||
|
assert "stride" in script
|
||||||
|
assert "negative_keep_ratio" in script
|
||||||
|
assert "positive_tile_count" in script
|
||||||
|
assert "negative_tile_count" in script
|
||||||
|
assert "skipped_negative_tile_count" in script
|
||||||
|
assert "source_name" in script
|
||||||
|
assert "reference_layer_name" in script
|
||||||
|
assert "Window" in script
|
||||||
|
assert "Transformer" in script
|
||||||
|
assert "fixture_mode" not in script
|
||||||
|
assert "will_download_models" not in script
|
||||||
|
|
||||||
|
|
||||||
|
def test_operator_yolo_tile_dataset_export_help_does_not_require_gis_dependencies() -> None:
|
||||||
|
script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py"
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, str(script_path), "--help"],
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.returncode == 0
|
||||||
|
assert "Export operator real-data samples to a tile-level YOLO detection dataset" in result.stdout
|
||||||
|
assert "--tile-size" in result.stdout
|
||||||
|
assert "--stride" in result.stdout
|
||||||
|
assert "--negative-keep-ratio" in result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def test_iter_tile_windows_covers_edges_without_duplicates() -> None:
|
||||||
|
module = load_tile_exporter()
|
||||||
|
|
||||||
|
windows = list(module.iter_tile_windows(width=512, height=512, tile_size=192, stride=96))
|
||||||
|
|
||||||
|
assert len(windows) == 25
|
||||||
|
assert windows[0].row_off == 0
|
||||||
|
assert windows[0].col_off == 0
|
||||||
|
assert windows[-1].row_off == 320
|
||||||
|
assert windows[-1].col_off == 320
|
||||||
|
assert len({(window.row_off, window.col_off) for window in windows}) == len(windows)
|
||||||
|
assert all(window.width == 192 for window in windows)
|
||||||
|
assert all(window.height == 192 for window in windows)
|
||||||
|
|
||||||
|
|
||||||
|
def test_negative_tile_keep_is_deterministic_and_ratio_bound() -> None:
|
||||||
|
module = load_tile_exporter()
|
||||||
|
|
||||||
|
first = [module.keep_negative_tile("geel", index, 0.25) for index in range(50)]
|
||||||
|
second = [module.keep_negative_tile("geel", index, 0.25) for index in range(50)]
|
||||||
|
all_kept = [module.keep_negative_tile("geel", index, 1.0) for index in range(10)]
|
||||||
|
none_kept = [module.keep_negative_tile("geel", index, 0.0) for index in range(10)]
|
||||||
|
|
||||||
|
assert first == second
|
||||||
|
assert 1 <= sum(first) <= 25
|
||||||
|
assert all(all_kept)
|
||||||
|
assert not any(none_kept)
|
||||||
@@ -266,6 +266,25 @@ This remains operator tooling only. GeoIntel does not expose Training Studio in
|
|||||||
V1, does not generate labels from predictions and does not treat the trained
|
V1, does not generate labels from predictions and does not treat the trained
|
||||||
artifact as useful until it passes the same real-data Detection + QA matrix.
|
artifact as useful until it passes the same real-data Detection + QA matrix.
|
||||||
|
|
||||||
|
If the whole-image dataset underfits or produces unusable detections, export
|
||||||
|
overlapping tile-level samples:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \
|
||||||
|
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
|
||||||
|
--output-dir /app/storage/operator-data/yolo-building-tile-dataset \
|
||||||
|
--tile-size 192 \
|
||||||
|
--stride 96 \
|
||||||
|
--negative-keep-ratio 0.5 \
|
||||||
|
--val-samples turnhout \
|
||||||
|
--force
|
||||||
|
```
|
||||||
|
|
||||||
|
The tile exporter clips reference building boxes into tile-local YOLO labels
|
||||||
|
and records the positive/negative tile counts. This gives the training smoke
|
||||||
|
more image samples while preserving the same explicit operator-data and QA/QC
|
||||||
|
validation boundary.
|
||||||
|
|
||||||
For visual error inspection, export the persisted QA evidence from a calibration
|
For visual error inspection, export the persisted QA evidence from a calibration
|
||||||
summary:
|
summary:
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,28 @@
|
|||||||
|
## Sprint 130 Operator YOLO tile-level dataset tooling (2026-07-07)
|
||||||
|
|
||||||
|
Changed:
|
||||||
|
- Added `scripts/export_operator_yolo_tile_dataset.py`.
|
||||||
|
- The exporter reads `operator_samples_manifest.json`, opens each raster/reference pair, creates overlapping tile windows, clips GRB building bounding boxes into tile-local YOLO labels, writes `dataset.yaml`, and reports `yolo_tile_dataset_summary.json`.
|
||||||
|
- Added deterministic negative tile retention through `negative_keep_ratio`.
|
||||||
|
- Added readiness compile coverage for the tile exporter.
|
||||||
|
- Added regression coverage in `backend/tests/test_sprint130_operator_yolo_tile_dataset.py`.
|
||||||
|
- Updated `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`.
|
||||||
|
|
||||||
|
Tested:
|
||||||
|
- RED: `python -m pytest backend\tests\test_sprint130_operator_yolo_tile_dataset.py -q` failed while `scripts/export_operator_yolo_tile_dataset.py` did not exist.
|
||||||
|
- `python -m pytest backend\tests\test_sprint130_operator_yolo_tile_dataset.py -q` passed.
|
||||||
|
- `python scripts\export_operator_yolo_tile_dataset.py --help` passed without requiring local GIS dependencies.
|
||||||
|
- `python -m py_compile scripts\export_operator_yolo_tile_dataset.py` passed.
|
||||||
|
|
||||||
|
Open:
|
||||||
|
- Run the tile exporter inside the AI-enabled Tower runtime, train a local tile-level model, and benchmark it through the existing multi-sample Detection + QA matrix.
|
||||||
|
|
||||||
|
Limitations:
|
||||||
|
- This remains operator tooling only. It does not add Training Studio, browser training controls, provider fetching, fake detections, model auto-provisioning or API contract changes.
|
||||||
|
|
||||||
|
Next recommended pass:
|
||||||
|
- Export the tile-level dataset on Tower with `tile-size=192`, `stride=96`, train a longer local model, and compare it against the current `yolov8s-building-segmentation-pt` baseline.
|
||||||
|
|
||||||
## Sprint 129 Operator YOLO training dataset tooling (2026-07-07)
|
## Sprint 129 Operator YOLO training dataset tooling (2026-07-07)
|
||||||
|
|
||||||
Changed:
|
Changed:
|
||||||
|
|||||||
+3
-1
@@ -406,4 +406,6 @@ This file now starts with the current implementation status. Older preparation/b
|
|||||||
- [x] Harden configured YOLO inference for single-band raster tiles and wrapped runtime errors.
|
- [x] Harden configured YOLO inference for single-band raster tiles and wrapped runtime errors.
|
||||||
- [x] Add operator-only YOLO dataset export and local training-smoke wrapper for real sample calibration.
|
- [x] Add operator-only YOLO dataset export and local training-smoke wrapper for real sample calibration.
|
||||||
- [x] Train/evaluate a small local GeoIntel building-detector smoke from the current operator samples and reject it because QA/QC did not improve.
|
- [x] Train/evaluate a small local GeoIntel building-detector smoke from the current operator samples and reject it because QA/QC did not improve.
|
||||||
- [ ] Build a larger tile-level training dataset with more AOIs, positive/negative tiles and validation splits before the next local model attempt.
|
- [x] Add operator-only tile-level YOLO dataset export with overlapping windows and deterministic negative tile retention.
|
||||||
|
- [ ] Run tile-level training on Tower and accept/reject the resulting local model through the persisted QA/QC matrix.
|
||||||
|
- [ ] Add more AOIs after the tile-level baseline so the next local model attempt is not limited to Geel/Mol/Turnhout.
|
||||||
|
|||||||
@@ -302,6 +302,44 @@ the resulting `.pt` file like any other local model asset: verify preflight,
|
|||||||
run the real-data matrix and compare persisted QA/QC metrics before activating
|
run the real-data matrix and compare persisted QA/QC metrics before activating
|
||||||
it as a useful default.
|
it as a useful default.
|
||||||
|
|
||||||
|
When whole-image training does not improve QA/QC, export a tile-level dataset
|
||||||
|
with overlapping raster windows:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \
|
||||||
|
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
|
||||||
|
--output-dir /app/storage/operator-data/yolo-building-tile-dataset \
|
||||||
|
--tile-size 192 \
|
||||||
|
--stride 96 \
|
||||||
|
--negative-keep-ratio 0.5 \
|
||||||
|
--val-samples turnhout \
|
||||||
|
--force
|
||||||
|
```
|
||||||
|
|
||||||
|
The tile exporter clips GRB building bounding boxes into each tile, writes
|
||||||
|
YOLO labels beside each tile image, keeps a deterministic ratio of empty
|
||||||
|
negative tiles, and records `yolo_tile_dataset_summary.json` with
|
||||||
|
`positive_tile_count`, `negative_tile_count` and skipped negative tile counts.
|
||||||
|
It remains operator tooling only: no provider fetch, no API mutation and no
|
||||||
|
automatic model training.
|
||||||
|
|
||||||
|
Train against the tile dataset by pointing the existing wrapper at the tile
|
||||||
|
output directory:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec \
|
||||||
|
-e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-tile-dataset \
|
||||||
|
-e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \
|
||||||
|
-e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-tile-detector.pt \
|
||||||
|
-e TRAIN_EPOCHS=30 \
|
||||||
|
-e TRAIN_IMGSZ=256 \
|
||||||
|
-e TRAIN_BATCH=4 \
|
||||||
|
-e TRAIN_WORKERS=0 \
|
||||||
|
-e TRAIN_DEVICE=cpu \
|
||||||
|
-e PYTHON_BIN=python3 \
|
||||||
|
geointel bash /app/scripts/train_operator_yolo_detector.sh
|
||||||
|
```
|
||||||
|
|
||||||
Export calibration QA evidence for visual review:
|
Export calibration QA evidence for visual review:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -0,0 +1,396 @@
|
|||||||
|
"""Export operator real-data samples to a tile-level YOLO detection dataset.
|
||||||
|
|
||||||
|
This operator/runtime helper turns the prepared orthophoto + GRB reference
|
||||||
|
sample manifest into overlapping raster tiles with YOLO labels. It does not
|
||||||
|
call GeoIntel APIs, does not train automatically and does not fetch provider
|
||||||
|
data.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
|
||||||
|
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset")
|
||||||
|
rasterio: Any = None
|
||||||
|
Window: Any = None
|
||||||
|
Transformer: Any = None
|
||||||
|
Image: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class TileWindow:
|
||||||
|
row_off: int
|
||||||
|
col_off: int
|
||||||
|
height: int
|
||||||
|
width: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PixelBox:
|
||||||
|
min_col: float
|
||||||
|
min_row: float
|
||||||
|
max_col: float
|
||||||
|
max_row: float
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Export operator real-data samples to a tile-level YOLO detection dataset.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--manifest-path",
|
||||||
|
type=Path,
|
||||||
|
default=Path(os.environ.get("OPERATOR_SAMPLE_MANIFEST_PATH", DEFAULT_MANIFEST_PATH)),
|
||||||
|
help="operator_samples_manifest.json created by prepare_operator_real_data_samples.py.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output-dir",
|
||||||
|
type=Path,
|
||||||
|
default=Path(os.environ.get("OPERATOR_YOLO_TILE_DATASET_DIR", DEFAULT_OUTPUT_DIR)),
|
||||||
|
help="Output directory for tile images, labels, dataset.yaml and summary JSON.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--tile-size", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_SIZE", "256")))
|
||||||
|
parser.add_argument("--stride", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_STRIDE", "128")))
|
||||||
|
parser.add_argument(
|
||||||
|
"--val-samples",
|
||||||
|
default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"),
|
||||||
|
help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--negative-keep-ratio",
|
||||||
|
type=float,
|
||||||
|
default=float(os.environ.get("OPERATOR_YOLO_NEGATIVE_KEEP_RATIO", "0.35")),
|
||||||
|
help="Deterministic ratio of empty tiles to keep as negative examples.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--min-label-px",
|
||||||
|
type=float,
|
||||||
|
default=float(os.environ.get("OPERATOR_YOLO_MIN_LABEL_PX", "4")),
|
||||||
|
help="Minimum clipped box width/height in pixels before a tile label is kept.",
|
||||||
|
)
|
||||||
|
parser.add_argument("--force", action="store_true", help="Remove and recreate output-dir before exporting.")
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_dependencies() -> None:
|
||||||
|
global Image, Transformer, Window, rasterio
|
||||||
|
try:
|
||||||
|
import rasterio as rasterio_module
|
||||||
|
from PIL import Image as image_module
|
||||||
|
from pyproj import Transformer as transformer_class
|
||||||
|
from rasterio.windows import Window as window_class
|
||||||
|
except Exception as exc: # pragma: no cover - runtime environment only.
|
||||||
|
raise SystemExit(
|
||||||
|
"export_operator_yolo_tile_dataset.py requires rasterio, pyproj and Pillow. "
|
||||||
|
"Run it inside the GeoIntel all-in-one container or an equivalent GIS Python environment."
|
||||||
|
) from exc
|
||||||
|
rasterio = rasterio_module
|
||||||
|
Window = window_class
|
||||||
|
Transformer = transformer_class
|
||||||
|
Image = image_module
|
||||||
|
|
||||||
|
|
||||||
|
def split_slugs(raw: str) -> set[str]:
|
||||||
|
return {value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()}
|
||||||
|
|
||||||
|
|
||||||
|
def edge_starts(length: int, tile_size: int, stride: int) -> list[int]:
|
||||||
|
if tile_size <= 0:
|
||||||
|
raise ValueError("tile_size must be positive")
|
||||||
|
if stride <= 0:
|
||||||
|
raise ValueError("stride must be positive")
|
||||||
|
if length <= tile_size:
|
||||||
|
return [0]
|
||||||
|
starts = list(range(0, max(length - tile_size, 0) + 1, stride))
|
||||||
|
final_start = length - tile_size
|
||||||
|
if starts[-1] != final_start:
|
||||||
|
starts.append(final_start)
|
||||||
|
return starts
|
||||||
|
|
||||||
|
|
||||||
|
def iter_tile_windows(width: int, height: int, tile_size: int, stride: int) -> Iterable[TileWindow]:
|
||||||
|
for row_off in edge_starts(height, tile_size, stride):
|
||||||
|
for col_off in edge_starts(width, tile_size, stride):
|
||||||
|
yield TileWindow(row_off=row_off, col_off=col_off, height=min(tile_size, height), width=min(tile_size, width))
|
||||||
|
|
||||||
|
|
||||||
|
def keep_negative_tile(sample_slug: str, tile_index: int, negative_keep_ratio: float) -> bool:
|
||||||
|
if negative_keep_ratio <= 0:
|
||||||
|
return False
|
||||||
|
if negative_keep_ratio >= 1:
|
||||||
|
return True
|
||||||
|
digest = hashlib.sha256(f"{sample_slug}:{tile_index}".encode("utf-8")).hexdigest()
|
||||||
|
bucket = int(digest[:8], 16) / 0xFFFFFFFF
|
||||||
|
return bucket < negative_keep_ratio
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_manifest_path(raw: str, manifest_path: Path) -> Path:
|
||||||
|
path = Path(raw)
|
||||||
|
if path.exists():
|
||||||
|
return path
|
||||||
|
if raw.startswith("/app/"):
|
||||||
|
relative = Path(raw.removeprefix("/app/"))
|
||||||
|
candidates = [
|
||||||
|
Path.cwd() / relative,
|
||||||
|
manifest_path.resolve().parent.parent.parent / relative,
|
||||||
|
]
|
||||||
|
for candidate in candidates:
|
||||||
|
if candidate.exists():
|
||||||
|
return candidate
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def iter_geometry_coords(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
|
||||||
|
geometry_type = geometry.get("type")
|
||||||
|
coordinates = geometry.get("coordinates")
|
||||||
|
if geometry_type == "Polygon":
|
||||||
|
for ring in coordinates or []:
|
||||||
|
for point in ring:
|
||||||
|
if len(point) >= 2:
|
||||||
|
yield float(point[0]), float(point[1])
|
||||||
|
elif geometry_type == "MultiPolygon":
|
||||||
|
for polygon in coordinates or []:
|
||||||
|
for ring in polygon:
|
||||||
|
for point in ring:
|
||||||
|
if len(point) >= 2:
|
||||||
|
yield float(point[0]), float(point[1])
|
||||||
|
|
||||||
|
|
||||||
|
def load_reference_pixel_boxes(reference_path: Path, dataset: Any, min_label_px: float) -> list[PixelBox]:
|
||||||
|
reference = json.loads(reference_path.read_text(encoding="utf-8-sig"))
|
||||||
|
features = reference.get("features") or []
|
||||||
|
transformer = Transformer.from_crs("EPSG:4326", dataset.crs, always_xy=True)
|
||||||
|
boxes: list[PixelBox] = []
|
||||||
|
|
||||||
|
for feature in features:
|
||||||
|
properties = feature.get("properties") or {}
|
||||||
|
if properties.get("source_name") != "grb":
|
||||||
|
continue
|
||||||
|
if properties.get("reference_layer_name") != "buildings":
|
||||||
|
continue
|
||||||
|
coords = list(iter_geometry_coords(feature.get("geometry") or {}))
|
||||||
|
if not coords:
|
||||||
|
continue
|
||||||
|
xs, ys = zip(*(transformer.transform(lon, lat) for lon, lat in coords), strict=False)
|
||||||
|
rows_cols = [dataset.index(x, y) for x, y in zip(xs, ys, strict=False)]
|
||||||
|
rows = [row for row, _ in rows_cols]
|
||||||
|
cols = [col for _, col in rows_cols]
|
||||||
|
min_col = max(0, min(cols))
|
||||||
|
max_col = min(dataset.width - 1, max(cols))
|
||||||
|
min_row = max(0, min(rows))
|
||||||
|
max_row = min(dataset.height - 1, max(rows))
|
||||||
|
if max_col - min_col < min_label_px or max_row - min_row < min_label_px:
|
||||||
|
continue
|
||||||
|
boxes.append(PixelBox(min_col=min_col, min_row=min_row, max_col=max_col, max_row=max_row))
|
||||||
|
return boxes
|
||||||
|
|
||||||
|
|
||||||
|
def labels_for_tile(tile_window: TileWindow, boxes: list[PixelBox], min_label_px: float) -> list[str]:
|
||||||
|
labels: list[str] = []
|
||||||
|
tile_min_col = tile_window.col_off
|
||||||
|
tile_min_row = tile_window.row_off
|
||||||
|
tile_max_col = tile_window.col_off + tile_window.width
|
||||||
|
tile_max_row = tile_window.row_off + tile_window.height
|
||||||
|
|
||||||
|
for box in boxes:
|
||||||
|
min_col = max(box.min_col, tile_min_col)
|
||||||
|
max_col = min(box.max_col, tile_max_col)
|
||||||
|
min_row = max(box.min_row, tile_min_row)
|
||||||
|
max_row = min(box.max_row, tile_max_row)
|
||||||
|
box_width = max_col - min_col
|
||||||
|
box_height = max_row - min_row
|
||||||
|
if box_width < min_label_px or box_height < min_label_px:
|
||||||
|
continue
|
||||||
|
local_min_col = min_col - tile_min_col
|
||||||
|
local_max_col = max_col - tile_min_col
|
||||||
|
local_min_row = min_row - tile_min_row
|
||||||
|
local_max_row = max_row - tile_min_row
|
||||||
|
x_center = (local_min_col + local_max_col) / 2.0 / tile_window.width
|
||||||
|
y_center = (local_min_row + local_max_row) / 2.0 / tile_window.height
|
||||||
|
norm_width = box_width / tile_window.width
|
||||||
|
norm_height = box_height / tile_window.height
|
||||||
|
labels.append(f"0 {x_center:.8f} {y_center:.8f} {norm_width:.8f} {norm_height:.8f}")
|
||||||
|
return labels
|
||||||
|
|
||||||
|
|
||||||
|
def image_array_from_raster_window(dataset: Any, tile_window: TileWindow) -> Any:
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
data = dataset.read(window=Window(tile_window.col_off, tile_window.row_off, tile_window.width, tile_window.height))
|
||||||
|
if data.shape[0] == 1:
|
||||||
|
rgb = np.repeat(data[:1], 3, axis=0)
|
||||||
|
else:
|
||||||
|
rgb = data[:3]
|
||||||
|
rgb = np.moveaxis(rgb, 0, -1)
|
||||||
|
if rgb.dtype != np.uint8:
|
||||||
|
rgb_min = float(np.nanmin(rgb))
|
||||||
|
rgb_max = float(np.nanmax(rgb))
|
||||||
|
if rgb_max > rgb_min:
|
||||||
|
rgb = ((rgb - rgb_min) / (rgb_max - rgb_min) * 255.0).clip(0, 255).astype("uint8")
|
||||||
|
else:
|
||||||
|
rgb = np.zeros(rgb.shape, dtype="uint8")
|
||||||
|
return rgb
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_yolo_directories(output_dir: Path) -> None:
|
||||||
|
for relative_path in ("images/train", "labels/train", "images/val", "labels/val"):
|
||||||
|
(output_dir / relative_path).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def write_dataset_yaml(output_dir: Path) -> Path:
|
||||||
|
yaml_path = output_dir / "dataset.yaml"
|
||||||
|
yaml_path.write_text(
|
||||||
|
"\n".join(
|
||||||
|
[
|
||||||
|
f"path: {output_dir}",
|
||||||
|
"train: images/train",
|
||||||
|
"val: images/val",
|
||||||
|
"names:",
|
||||||
|
" 0: building",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return yaml_path
|
||||||
|
|
||||||
|
|
||||||
|
def export_sample_tiles(
|
||||||
|
sample: dict[str, Any],
|
||||||
|
manifest_path: Path,
|
||||||
|
output_dir: Path,
|
||||||
|
val_slugs: set[str],
|
||||||
|
tile_size: int,
|
||||||
|
stride: int,
|
||||||
|
negative_keep_ratio: float,
|
||||||
|
min_label_px: float,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
sample_slug = str(sample["sample_slug"])
|
||||||
|
split = "val" if sample_slug.lower() in val_slugs else "train"
|
||||||
|
raster_path = resolve_manifest_path(str(sample["raster_path"]), manifest_path)
|
||||||
|
reference_path = resolve_manifest_path(str(sample["reference_path"]), manifest_path)
|
||||||
|
if not raster_path.exists():
|
||||||
|
raise SystemExit(f"Raster path is not readable for sample {sample_slug}: {raster_path}")
|
||||||
|
if not reference_path.exists():
|
||||||
|
raise SystemExit(f"Reference path is not readable for sample {sample_slug}: {reference_path}")
|
||||||
|
|
||||||
|
exported: list[dict[str, Any]] = []
|
||||||
|
with rasterio.open(raster_path) as dataset:
|
||||||
|
boxes = load_reference_pixel_boxes(reference_path, dataset, min_label_px=min_label_px)
|
||||||
|
for tile_index, tile_window in enumerate(iter_tile_windows(dataset.width, dataset.height, tile_size, stride)):
|
||||||
|
labels = labels_for_tile(tile_window, boxes, min_label_px=min_label_px)
|
||||||
|
is_negative = not labels
|
||||||
|
if is_negative and not keep_negative_tile(sample_slug, tile_index, negative_keep_ratio):
|
||||||
|
exported.append(
|
||||||
|
{
|
||||||
|
"sample_slug": sample_slug,
|
||||||
|
"split": split,
|
||||||
|
"tile_index": tile_index,
|
||||||
|
"kept": False,
|
||||||
|
"label_count": 0,
|
||||||
|
"is_negative": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
tile_name = f"{sample_slug}_{tile_index:04d}_r{tile_window.row_off}_c{tile_window.col_off}"
|
||||||
|
image_path = output_dir / "images" / split / f"{tile_name}.png"
|
||||||
|
label_path = output_dir / "labels" / split / f"{tile_name}.txt"
|
||||||
|
image_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
label_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
Image.fromarray(image_array_from_raster_window(dataset, tile_window)).save(image_path)
|
||||||
|
label_path.write_text("\n".join(labels) + ("\n" if labels else ""), encoding="utf-8")
|
||||||
|
exported.append(
|
||||||
|
{
|
||||||
|
"sample_slug": sample_slug,
|
||||||
|
"split": split,
|
||||||
|
"tile_index": tile_index,
|
||||||
|
"kept": True,
|
||||||
|
"image_path": str(image_path),
|
||||||
|
"label_path": str(label_path),
|
||||||
|
"label_count": len(labels),
|
||||||
|
"is_negative": is_negative,
|
||||||
|
"window": {
|
||||||
|
"row_off": tile_window.row_off,
|
||||||
|
"col_off": tile_window.col_off,
|
||||||
|
"height": tile_window.height,
|
||||||
|
"width": tile_window.width,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return exported
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = parse_args()
|
||||||
|
ensure_dependencies()
|
||||||
|
if args.force and args.output_dir.exists():
|
||||||
|
shutil.rmtree(args.output_dir)
|
||||||
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
ensure_yolo_directories(args.output_dir)
|
||||||
|
|
||||||
|
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
|
||||||
|
samples = manifest.get("samples") or []
|
||||||
|
if not samples:
|
||||||
|
raise SystemExit("Operator sample manifest contains no samples")
|
||||||
|
val_slugs = split_slugs(args.val_samples)
|
||||||
|
exported_tiles: list[dict[str, Any]] = []
|
||||||
|
for sample in samples:
|
||||||
|
exported_tiles.extend(
|
||||||
|
export_sample_tiles(
|
||||||
|
sample=sample,
|
||||||
|
manifest_path=args.manifest_path,
|
||||||
|
output_dir=args.output_dir,
|
||||||
|
val_slugs=val_slugs,
|
||||||
|
tile_size=args.tile_size,
|
||||||
|
stride=args.stride,
|
||||||
|
negative_keep_ratio=args.negative_keep_ratio,
|
||||||
|
min_label_px=args.min_label_px,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
kept_tiles = [tile for tile in exported_tiles if tile["kept"]]
|
||||||
|
if not any(tile["split"] == "train" for tile in kept_tiles):
|
||||||
|
raise SystemExit("YOLO tile dataset export produced no training tiles")
|
||||||
|
if not any(tile["split"] == "val" for tile in kept_tiles):
|
||||||
|
raise SystemExit("YOLO tile dataset export produced no validation tiles")
|
||||||
|
dataset_yaml = write_dataset_yaml(args.output_dir)
|
||||||
|
positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
|
||||||
|
negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
|
||||||
|
skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]]
|
||||||
|
summary = {
|
||||||
|
"status": "ok",
|
||||||
|
"dataset_yaml": str(dataset_yaml),
|
||||||
|
"output_dir": str(args.output_dir),
|
||||||
|
"class_names": ["building"],
|
||||||
|
"tile_size": args.tile_size,
|
||||||
|
"stride": args.stride,
|
||||||
|
"negative_keep_ratio": args.negative_keep_ratio,
|
||||||
|
"min_label_px": args.min_label_px,
|
||||||
|
"source_sample_count": len(samples),
|
||||||
|
"tile_count": len(kept_tiles),
|
||||||
|
"positive_tile_count": len(positive_tiles),
|
||||||
|
"negative_tile_count": len(negative_tiles),
|
||||||
|
"skipped_negative_tile_count": len(skipped_negative_tiles),
|
||||||
|
"label_count": sum(tile["label_count"] for tile in kept_tiles),
|
||||||
|
"train_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "train"),
|
||||||
|
"val_tile_count": sum(1 for tile in kept_tiles if tile["split"] == "val"),
|
||||||
|
"tiles": kept_tiles,
|
||||||
|
}
|
||||||
|
summary_path = args.output_dir / "yolo_tile_dataset_summary.json"
|
||||||
|
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
|
||||||
|
print(json.dumps(summary, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -43,6 +43,7 @@ ${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py
|
|||||||
${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py
|
${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
|
${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
||||||
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
||||||
${PYTHON_BIN} -m compileall backend/app
|
${PYTHON_BIN} -m compileall backend/app
|
||||||
|
|||||||
Reference in New Issue
Block a user