Add operator YOLO tile dataset exporter
This commit is contained in:
@@ -311,6 +311,25 @@ explicit operator sample manifest. The training wrapper writes
|
||||
`training_summary.json` and a local `.pt` artifact, which still must be
|
||||
validated through model preflight and the real-data QA matrix before use.
|
||||
|
||||
For a larger tile-level training set, use overlapping windows instead of one
|
||||
image per AOI:
|
||||
|
||||
```bash
|
||||
docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \
|
||||
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
|
||||
--output-dir /app/storage/operator-data/yolo-building-tile-dataset \
|
||||
--tile-size 192 \
|
||||
--stride 96 \
|
||||
--negative-keep-ratio 0.5 \
|
||||
--val-samples turnhout \
|
||||
--force
|
||||
```
|
||||
|
||||
Then point `OPERATOR_YOLO_DATASET_DIR` at
|
||||
`/app/storage/operator-data/yolo-building-tile-dataset` and keep the same
|
||||
training wrapper. Tile-level output remains operator tooling outside the V1
|
||||
browser product.
|
||||
|
||||
The backend also exposes a read-only model asset catalog for the mounted model
|
||||
directory:
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
def load_tile_exporter():
|
||||
script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py"
|
||||
spec = importlib.util.spec_from_file_location("operator_tile_exporter", script_path)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
def test_operator_yolo_tile_dataset_export_script_contract() -> None:
|
||||
script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py"
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert script_path.exists()
|
||||
script = script_path.read_text(encoding="utf-8")
|
||||
|
||||
assert "py_compile scripts/export_operator_yolo_tile_dataset.py" in readiness
|
||||
assert "operator_samples_manifest.json" in script
|
||||
assert "yolo-building-tile-dataset" in script
|
||||
assert "dataset.yaml" in script
|
||||
assert "images/train" in script
|
||||
assert "labels/train" in script
|
||||
assert "images/val" in script
|
||||
assert "labels/val" in script
|
||||
assert "tile_size" in script
|
||||
assert "stride" in script
|
||||
assert "negative_keep_ratio" in script
|
||||
assert "positive_tile_count" in script
|
||||
assert "negative_tile_count" in script
|
||||
assert "skipped_negative_tile_count" in script
|
||||
assert "source_name" in script
|
||||
assert "reference_layer_name" in script
|
||||
assert "Window" in script
|
||||
assert "Transformer" in script
|
||||
assert "fixture_mode" not in script
|
||||
assert "will_download_models" not in script
|
||||
|
||||
|
||||
def test_operator_yolo_tile_dataset_export_help_does_not_require_gis_dependencies() -> None:
|
||||
script_path = ROOT / "scripts" / "export_operator_yolo_tile_dataset.py"
|
||||
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(script_path), "--help"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0
|
||||
assert "Export operator real-data samples to a tile-level YOLO detection dataset" in result.stdout
|
||||
assert "--tile-size" in result.stdout
|
||||
assert "--stride" in result.stdout
|
||||
assert "--negative-keep-ratio" in result.stdout
|
||||
|
||||
|
||||
def test_iter_tile_windows_covers_edges_without_duplicates() -> None:
|
||||
module = load_tile_exporter()
|
||||
|
||||
windows = list(module.iter_tile_windows(width=512, height=512, tile_size=192, stride=96))
|
||||
|
||||
assert len(windows) == 25
|
||||
assert windows[0].row_off == 0
|
||||
assert windows[0].col_off == 0
|
||||
assert windows[-1].row_off == 320
|
||||
assert windows[-1].col_off == 320
|
||||
assert len({(window.row_off, window.col_off) for window in windows}) == len(windows)
|
||||
assert all(window.width == 192 for window in windows)
|
||||
assert all(window.height == 192 for window in windows)
|
||||
|
||||
|
||||
def test_negative_tile_keep_is_deterministic_and_ratio_bound() -> None:
|
||||
module = load_tile_exporter()
|
||||
|
||||
first = [module.keep_negative_tile("geel", index, 0.25) for index in range(50)]
|
||||
second = [module.keep_negative_tile("geel", index, 0.25) for index in range(50)]
|
||||
all_kept = [module.keep_negative_tile("geel", index, 1.0) for index in range(10)]
|
||||
none_kept = [module.keep_negative_tile("geel", index, 0.0) for index in range(10)]
|
||||
|
||||
assert first == second
|
||||
assert 1 <= sum(first) <= 25
|
||||
assert all(all_kept)
|
||||
assert not any(none_kept)
|
||||
Reference in New Issue
Block a user