Add operator YOLO training dataset tooling
This commit is contained in:
@@ -7,6 +7,15 @@
|
|||||||
|
|
||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## Sprint 129 Operator YOLO training dataset tooling (2026-07-07)
|
||||||
|
|
||||||
|
- Added `scripts/export_operator_yolo_dataset.py` to convert prepared operator orthophoto/GRB sample pairs into a standard local YOLO detection dataset with `dataset.yaml`, train/validation image folders, label folders and `yolo_dataset_summary.json`.
|
||||||
|
- Added `scripts/train_operator_yolo_detector.sh` as an operator-only training smoke wrapper that uses an existing local base `.pt` model and writes a trained local `.pt` artifact plus `training_summary.json`.
|
||||||
|
- Added readiness coverage for the exporter Python compile check and training wrapper shell syntax.
|
||||||
|
- Added regression coverage in `backend/tests/test_sprint129_operator_yolo_training_dataset.py`.
|
||||||
|
- Updated operator documentation for dataset export, training smoke usage and the requirement to benchmark any trained model through the existing real-data Detection + QA matrix before treating it as useful.
|
||||||
|
- No Training Studio UI, API contract change, provider fetching, model auto-provisioning or app-side model training behavior was introduced.
|
||||||
|
|
||||||
## Sprint 128 Stronger building model runtime benchmark (2026-07-07)
|
## Sprint 128 Stronger building model runtime benchmark (2026-07-07)
|
||||||
|
|
||||||
- Added `keremberke/yolov8s-building-segmentation` as an explicit Tower runtime model asset at `/mnt/user/appdata/geointel/models/yolov8s-building-segmentation.pt`; the file is not committed to Git.
|
- Added `keremberke/yolov8s-building-segmentation` as an explicit Tower runtime model asset at `/mnt/user/appdata/geointel/models/yolov8s-building-segmentation.pt`; the file is not committed to Git.
|
||||||
|
|||||||
@@ -278,6 +278,38 @@ python scripts/configure_yolo_model.py \
|
|||||||
The smoke loads only the supplied local model file, does not run inference and
|
The smoke loads only the supplied local model file, does not run inference and
|
||||||
does not download weights.
|
does not download weights.
|
||||||
|
|
||||||
|
Operator-only local training preparation is available when real public model
|
||||||
|
candidates are too weak for the target imagery. It is not a browser feature and
|
||||||
|
does not change API contracts:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it geointel python /app/scripts/export_operator_yolo_dataset.py \
|
||||||
|
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
|
||||||
|
--output-dir /app/storage/operator-data/yolo-building-dataset \
|
||||||
|
--val-samples turnhout \
|
||||||
|
--force
|
||||||
|
```
|
||||||
|
|
||||||
|
In an AI-enabled runtime with an existing local base model:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec \
|
||||||
|
-e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-dataset \
|
||||||
|
-e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \
|
||||||
|
-e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-detector.pt \
|
||||||
|
-e TRAIN_EPOCHS=8 \
|
||||||
|
-e TRAIN_IMGSZ=512 \
|
||||||
|
-e TRAIN_BATCH=2 \
|
||||||
|
-e TRAIN_WORKERS=0 \
|
||||||
|
-e TRAIN_DEVICE=cpu \
|
||||||
|
geointel bash /app/scripts/train_operator_yolo_detector.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The exporter creates a YOLO `dataset.yaml` plus image/label folders from the
|
||||||
|
explicit operator sample manifest. The training wrapper writes
|
||||||
|
`training_summary.json` and a local `.pt` artifact, which still must be
|
||||||
|
validated through model preflight and the real-data QA matrix before use.
|
||||||
|
|
||||||
The backend also exposes a read-only model asset catalog for the mounted model
|
The backend also exposes a read-only model asset catalog for the mounted model
|
||||||
directory:
|
directory:
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,68 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
|
|
||||||
|
def test_operator_yolo_dataset_export_script_contract() -> None:
|
||||||
|
script_path = ROOT / "scripts" / "export_operator_yolo_dataset.py"
|
||||||
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert script_path.exists()
|
||||||
|
script = script_path.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "py_compile scripts/export_operator_yolo_dataset.py" in readiness
|
||||||
|
assert "operator_samples_manifest.json" in script
|
||||||
|
assert "dataset.yaml" in script
|
||||||
|
assert "images/train" in script
|
||||||
|
assert "labels/train" in script
|
||||||
|
assert "images/val" in script
|
||||||
|
assert "labels/val" in script
|
||||||
|
assert "building" in script
|
||||||
|
assert "reference_feature_count" in script
|
||||||
|
assert "source_name" in script
|
||||||
|
assert "reference_layer_name" in script
|
||||||
|
assert "rasterio" in script
|
||||||
|
assert "Transformer" in script
|
||||||
|
assert "demo/workflow" not in script
|
||||||
|
assert "fixture_mode" not in script
|
||||||
|
assert "will_download_models" not in script
|
||||||
|
|
||||||
|
|
||||||
|
def test_operator_yolo_dataset_export_help_does_not_require_gis_dependencies() -> None:
|
||||||
|
script_path = ROOT / "scripts" / "export_operator_yolo_dataset.py"
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, str(script_path), "--help"],
|
||||||
|
check=False,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.returncode == 0
|
||||||
|
assert "Export operator real-data samples to a YOLO detection dataset" in result.stdout
|
||||||
|
assert "--manifest-path" in result.stdout
|
||||||
|
assert "--val-samples" in result.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def test_operator_yolo_train_smoke_script_contract() -> None:
|
||||||
|
script_path = ROOT / "scripts" / "train_operator_yolo_detector.sh"
|
||||||
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert script_path.exists()
|
||||||
|
script = script_path.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "bash -n scripts/train_operator_yolo_detector.sh" in readiness
|
||||||
|
assert "OPERATOR_YOLO_DATASET_DIR" in script
|
||||||
|
assert "YOLO_BASE_MODEL_PATH" in script
|
||||||
|
assert "TRAIN_MODEL_OUTPUT_PATH" in script
|
||||||
|
assert "TRAIN_EPOCHS" in script
|
||||||
|
assert "TRAIN_IMGSZ" in script
|
||||||
|
assert "dataset.yaml" in script
|
||||||
|
assert "from ultralytics import YOLO" in script
|
||||||
|
assert "model.train" in script
|
||||||
|
assert "training_summary.json" in script
|
||||||
|
assert "download" not in script.lower()
|
||||||
|
assert "fixture_mode" not in script
|
||||||
@@ -229,6 +229,42 @@ The multi-sample summary exposes `best_overall_by_score`,
|
|||||||
model-quality decisions are based on repeated persisted QA/QC evidence rather
|
model-quality decisions are based on repeated persisted QA/QC evidence rather
|
||||||
than one AOI.
|
than one AOI.
|
||||||
|
|
||||||
|
When repeated public model benchmarks remain too weak, the operator can convert
|
||||||
|
the prepared real-data samples into a local YOLO training dataset:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it geointel python /app/scripts/export_operator_yolo_dataset.py \
|
||||||
|
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
|
||||||
|
--output-dir /app/storage/operator-data/yolo-building-dataset \
|
||||||
|
--val-samples turnhout \
|
||||||
|
--force
|
||||||
|
```
|
||||||
|
|
||||||
|
The exporter creates a standard YOLO detection layout with `dataset.yaml`,
|
||||||
|
`images/train`, `labels/train`, `images/val` and `labels/val`. It converts GRB
|
||||||
|
building reference geometries to pixel-space bounding boxes for the matching
|
||||||
|
orthophoto sample and records `yolo_dataset_summary.json`.
|
||||||
|
|
||||||
|
A minimal local training smoke can then be run explicitly in an AI-enabled
|
||||||
|
runtime:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec \
|
||||||
|
-e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-dataset \
|
||||||
|
-e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \
|
||||||
|
-e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-detector.pt \
|
||||||
|
-e TRAIN_EPOCHS=8 \
|
||||||
|
-e TRAIN_IMGSZ=512 \
|
||||||
|
-e TRAIN_BATCH=2 \
|
||||||
|
-e TRAIN_WORKERS=0 \
|
||||||
|
-e TRAIN_DEVICE=cpu \
|
||||||
|
geointel bash /app/scripts/train_operator_yolo_detector.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
This remains operator tooling only. GeoIntel does not expose Training Studio in
|
||||||
|
V1, does not generate labels from predictions and does not treat the trained
|
||||||
|
artifact as useful until it passes the same real-data Detection + QA matrix.
|
||||||
|
|
||||||
For visual error inspection, export the persisted QA evidence from a calibration
|
For visual error inspection, export the persisted QA evidence from a calibration
|
||||||
summary:
|
summary:
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,35 @@
|
|||||||
|
## Sprint 129 Operator YOLO training dataset tooling (2026-07-07)
|
||||||
|
|
||||||
|
Changed:
|
||||||
|
- Added `scripts/export_operator_yolo_dataset.py` to export prepared operator samples into a local YOLO detection dataset:
|
||||||
|
- input manifest: `operator_samples_manifest.json`
|
||||||
|
- output: `dataset.yaml`, `images/train`, `labels/train`, `images/val`, `labels/val`, `yolo_dataset_summary.json`
|
||||||
|
- labels are derived from GRB building references with `source_name=grb` and `reference_layer_name=buildings`.
|
||||||
|
- Added `scripts/train_operator_yolo_detector.sh` as an explicit operator/runtime wrapper around a local Ultralytics training smoke:
|
||||||
|
- requires `OPERATOR_YOLO_DATASET_DIR`
|
||||||
|
- requires an existing `YOLO_BASE_MODEL_PATH`
|
||||||
|
- writes a local `TRAIN_MODEL_OUTPUT_PATH`
|
||||||
|
- writes `training_summary.json`.
|
||||||
|
- Added readiness coverage for exporter compile and train-wrapper shell syntax.
|
||||||
|
- Added regression coverage in `backend/tests/test_sprint129_operator_yolo_training_dataset.py`.
|
||||||
|
- Updated `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`.
|
||||||
|
|
||||||
|
Tested:
|
||||||
|
- RED: `python -m pytest backend\tests\test_sprint129_operator_yolo_training_dataset.py -q` failed while the exporter and training wrapper contracts were incomplete.
|
||||||
|
- `python -m pytest backend\tests\test_sprint129_operator_yolo_training_dataset.py -q` passed.
|
||||||
|
- `python scripts\export_operator_yolo_dataset.py --help` passed without requiring local GIS dependencies.
|
||||||
|
- `python -m py_compile scripts\export_operator_yolo_dataset.py` passed.
|
||||||
|
- `bash -n scripts/train_operator_yolo_detector.sh` passed.
|
||||||
|
|
||||||
|
Open:
|
||||||
|
- Run the exporter and training smoke inside the AI-enabled Tower runtime, then benchmark the trained artifact through the existing multi-sample Detection + QA matrix.
|
||||||
|
|
||||||
|
Limitations:
|
||||||
|
- This is operator tooling only. It does not add Training Studio, browser training controls, provider fetching, fake detections, model auto-provisioning or API contract changes.
|
||||||
|
|
||||||
|
Next recommended pass:
|
||||||
|
- Generate the local YOLO dataset from the current Geel/Mol/Turnhout samples, train a small local model smoke from `yolov8n.pt`, and compare it against the current `yolov8s-building-segmentation-pt` benchmark.
|
||||||
|
|
||||||
## Sprint 128 Stronger building model runtime benchmark (2026-07-07)
|
## Sprint 128 Stronger building model runtime benchmark (2026-07-07)
|
||||||
|
|
||||||
Changed:
|
Changed:
|
||||||
|
|||||||
@@ -404,3 +404,5 @@ This file now starts with the current implementation status. Older preparation/b
|
|||||||
- [x] Render persisted QA/QC feature-level evidence as Map workspace overlays.
|
- [x] Render persisted QA/QC feature-level evidence as Map workspace overlays.
|
||||||
- [x] Add safe local YOLO model env configuration helper for Unraid/Tower runtime activation.
|
- [x] Add safe local YOLO model env configuration helper for Unraid/Tower runtime activation.
|
||||||
- [x] Harden configured YOLO inference for single-band raster tiles and wrapped runtime errors.
|
- [x] Harden configured YOLO inference for single-band raster tiles and wrapped runtime errors.
|
||||||
|
- [x] Add operator-only YOLO dataset export and local training-smoke wrapper for real sample calibration.
|
||||||
|
- [ ] Train/evaluate a local GeoIntel building detector from the operator samples and only activate it after QA/QC matrix improvement.
|
||||||
|
|||||||
@@ -261,6 +261,46 @@ plus a combined `multi_sample_quality_summary.json` with
|
|||||||
container-style `/app/storage/...` manifest paths to repo-relative
|
container-style `/app/storage/...` manifest paths to repo-relative
|
||||||
`storage/...` paths when run from the Tower host checkout.
|
`storage/...` paths when run from the Tower host checkout.
|
||||||
|
|
||||||
|
Export the same operator samples to a local YOLO detection dataset when the
|
||||||
|
public model candidates are not strong enough for the target imagery:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -it geointel python /app/scripts/export_operator_yolo_dataset.py \
|
||||||
|
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
|
||||||
|
--output-dir /app/storage/operator-data/yolo-building-dataset \
|
||||||
|
--val-samples turnhout \
|
||||||
|
--force
|
||||||
|
```
|
||||||
|
|
||||||
|
The exporter writes `dataset.yaml`, `images/train`, `labels/train`,
|
||||||
|
`images/val`, `labels/val` and `yolo_dataset_summary.json`. It uses only the
|
||||||
|
explicit operator sample manifest and GRB building references where
|
||||||
|
`source_name=grb` and `reference_layer_name=buildings`. It does not call
|
||||||
|
GeoIntel APIs, create provider data, run inference or train a model.
|
||||||
|
|
||||||
|
Run a small local training smoke only in an AI-enabled runtime with an existing
|
||||||
|
local base model file:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec \
|
||||||
|
-e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-dataset \
|
||||||
|
-e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \
|
||||||
|
-e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-detector.pt \
|
||||||
|
-e TRAIN_EPOCHS=8 \
|
||||||
|
-e TRAIN_IMGSZ=512 \
|
||||||
|
-e TRAIN_BATCH=2 \
|
||||||
|
-e TRAIN_WORKERS=0 \
|
||||||
|
-e TRAIN_DEVICE=cpu \
|
||||||
|
geointel bash /app/scripts/train_operator_yolo_detector.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The training wrapper is intentionally outside the product UI. It runs
|
||||||
|
Ultralytics from the existing runtime, copies the best trained artifact to
|
||||||
|
`TRAIN_MODEL_OUTPUT_PATH` and writes `training_summary.json`. Afterward, treat
|
||||||
|
the resulting `.pt` file like any other local model asset: verify preflight,
|
||||||
|
run the real-data matrix and compare persisted QA/QC metrics before activating
|
||||||
|
it as a useful default.
|
||||||
|
|
||||||
Export calibration QA evidence for visual review:
|
Export calibration QA evidence for visual review:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -0,0 +1,255 @@
|
|||||||
|
"""Export operator real-data samples to a YOLO detection dataset.
|
||||||
|
|
||||||
|
This is an operator/runtime helper. It converts the explicit orthophoto + GRB
|
||||||
|
reference sample manifest into local YOLO images/labels for model experiments.
|
||||||
|
It does not call GeoIntel APIs, does not train automatically and does not fetch
|
||||||
|
new provider data.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
|
||||||
|
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-dataset")
|
||||||
|
rasterio: Any = None
|
||||||
|
Transformer: Any = None
|
||||||
|
Image: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Export operator real-data samples to a YOLO detection dataset.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--manifest-path",
|
||||||
|
type=Path,
|
||||||
|
default=Path(os.environ.get("OPERATOR_SAMPLE_MANIFEST_PATH", DEFAULT_MANIFEST_PATH)),
|
||||||
|
help="operator_samples_manifest.json created by prepare_operator_real_data_samples.py.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output-dir",
|
||||||
|
type=Path,
|
||||||
|
default=Path(os.environ.get("OPERATOR_YOLO_DATASET_DIR", DEFAULT_OUTPUT_DIR)),
|
||||||
|
help="Output directory for images, labels, dataset.yaml and summary JSON.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--val-samples",
|
||||||
|
default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"),
|
||||||
|
help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--force",
|
||||||
|
action="store_true",
|
||||||
|
help="Remove and recreate output-dir before exporting.",
|
||||||
|
)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_dependencies() -> None:
|
||||||
|
global Image, Transformer, rasterio
|
||||||
|
try:
|
||||||
|
import rasterio as rasterio_module
|
||||||
|
from PIL import Image as image_module
|
||||||
|
from pyproj import Transformer as transformer_class
|
||||||
|
except Exception as exc: # pragma: no cover - runtime environment only.
|
||||||
|
raise SystemExit(
|
||||||
|
"export_operator_yolo_dataset.py requires rasterio, pyproj and Pillow. "
|
||||||
|
"Run it inside the GeoIntel all-in-one container or an equivalent GIS Python environment."
|
||||||
|
) from exc
|
||||||
|
rasterio = rasterio_module
|
||||||
|
Transformer = transformer_class
|
||||||
|
Image = image_module
|
||||||
|
|
||||||
|
|
||||||
|
def split_slugs(raw: str) -> set[str]:
|
||||||
|
return {value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()}
|
||||||
|
|
||||||
|
|
||||||
|
def iter_geometry_coords(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
|
||||||
|
geometry_type = geometry.get("type")
|
||||||
|
coordinates = geometry.get("coordinates")
|
||||||
|
if geometry_type == "Polygon":
|
||||||
|
for ring in coordinates or []:
|
||||||
|
for point in ring:
|
||||||
|
if len(point) >= 2:
|
||||||
|
yield float(point[0]), float(point[1])
|
||||||
|
elif geometry_type == "MultiPolygon":
|
||||||
|
for polygon in coordinates or []:
|
||||||
|
for ring in polygon:
|
||||||
|
for point in ring:
|
||||||
|
if len(point) >= 2:
|
||||||
|
yield float(point[0]), float(point[1])
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_manifest_path(raw: str, manifest_path: Path) -> Path:
|
||||||
|
path = Path(raw)
|
||||||
|
if path.exists():
|
||||||
|
return path
|
||||||
|
if raw.startswith("/app/"):
|
||||||
|
relative = Path(raw.removeprefix("/app/"))
|
||||||
|
candidates = [
|
||||||
|
Path.cwd() / relative,
|
||||||
|
manifest_path.resolve().parent.parent.parent / relative,
|
||||||
|
]
|
||||||
|
for candidate in candidates:
|
||||||
|
if candidate.exists():
|
||||||
|
return candidate
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def image_array_from_raster(dataset: Any) -> Any:
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
data = dataset.read()
|
||||||
|
if data.shape[0] == 1:
|
||||||
|
rgb = np.repeat(data[:1], 3, axis=0)
|
||||||
|
else:
|
||||||
|
rgb = data[:3]
|
||||||
|
rgb = np.moveaxis(rgb, 0, -1)
|
||||||
|
if rgb.dtype != np.uint8:
|
||||||
|
rgb_min = float(np.nanmin(rgb))
|
||||||
|
rgb_max = float(np.nanmax(rgb))
|
||||||
|
if rgb_max > rgb_min:
|
||||||
|
rgb = ((rgb - rgb_min) / (rgb_max - rgb_min) * 255.0).clip(0, 255).astype("uint8")
|
||||||
|
else:
|
||||||
|
rgb = np.zeros(rgb.shape, dtype="uint8")
|
||||||
|
return rgb
|
||||||
|
|
||||||
|
|
||||||
|
def yolo_boxes_for_reference(reference_path: Path, dataset: Any) -> list[str]:
|
||||||
|
reference = json.loads(reference_path.read_text(encoding="utf-8-sig"))
|
||||||
|
features = reference.get("features") or []
|
||||||
|
transformer = Transformer.from_crs("EPSG:4326", dataset.crs, always_xy=True)
|
||||||
|
width = dataset.width
|
||||||
|
height = dataset.height
|
||||||
|
labels: list[str] = []
|
||||||
|
|
||||||
|
for feature in features:
|
||||||
|
properties = feature.get("properties") or {}
|
||||||
|
if properties.get("source_name") != "grb":
|
||||||
|
continue
|
||||||
|
if properties.get("reference_layer_name") != "buildings":
|
||||||
|
continue
|
||||||
|
coords = list(iter_geometry_coords(feature.get("geometry") or {}))
|
||||||
|
if not coords:
|
||||||
|
continue
|
||||||
|
xs, ys = zip(*(transformer.transform(lon, lat) for lon, lat in coords), strict=False)
|
||||||
|
rows_cols = [dataset.index(x, y) for x, y in zip(xs, ys, strict=False)]
|
||||||
|
rows = [row for row, _ in rows_cols]
|
||||||
|
cols = [col for _, col in rows_cols]
|
||||||
|
min_col = max(0, min(cols))
|
||||||
|
max_col = min(width - 1, max(cols))
|
||||||
|
min_row = max(0, min(rows))
|
||||||
|
max_row = min(height - 1, max(rows))
|
||||||
|
box_width = max_col - min_col
|
||||||
|
box_height = max_row - min_row
|
||||||
|
if box_width < 2 or box_height < 2:
|
||||||
|
continue
|
||||||
|
x_center = (min_col + max_col) / 2.0 / width
|
||||||
|
y_center = (min_row + max_row) / 2.0 / height
|
||||||
|
norm_width = box_width / width
|
||||||
|
norm_height = box_height / height
|
||||||
|
labels.append(f"0 {x_center:.8f} {y_center:.8f} {norm_width:.8f} {norm_height:.8f}")
|
||||||
|
return labels
|
||||||
|
|
||||||
|
|
||||||
|
def export_sample(sample: dict[str, Any], manifest_path: Path, output_dir: Path, val_slugs: set[str]) -> dict[str, Any]:
|
||||||
|
sample_slug = str(sample["sample_slug"])
|
||||||
|
split = "val" if sample_slug.lower() in val_slugs else "train"
|
||||||
|
raster_path = resolve_manifest_path(str(sample["raster_path"]), manifest_path)
|
||||||
|
reference_path = resolve_manifest_path(str(sample["reference_path"]), manifest_path)
|
||||||
|
if not raster_path.exists():
|
||||||
|
raise SystemExit(f"Raster path is not readable for sample {sample_slug}: {raster_path}")
|
||||||
|
if not reference_path.exists():
|
||||||
|
raise SystemExit(f"Reference path is not readable for sample {sample_slug}: {reference_path}")
|
||||||
|
|
||||||
|
image_path = output_dir / "images" / split / f"{sample_slug}.png"
|
||||||
|
label_path = output_dir / "labels" / split / f"{sample_slug}.txt"
|
||||||
|
image_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
label_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
with rasterio.open(raster_path) as dataset:
|
||||||
|
image_array = image_array_from_raster(dataset)
|
||||||
|
Image.fromarray(image_array).save(image_path)
|
||||||
|
labels = yolo_boxes_for_reference(reference_path, dataset)
|
||||||
|
label_path.write_text("\n".join(labels) + ("\n" if labels else ""), encoding="utf-8")
|
||||||
|
return {
|
||||||
|
"sample_slug": sample_slug,
|
||||||
|
"split": split,
|
||||||
|
"image_path": str(image_path),
|
||||||
|
"label_path": str(label_path),
|
||||||
|
"label_count": len(labels),
|
||||||
|
"reference_feature_count": sample.get("reference_feature_count"),
|
||||||
|
"raster_path": str(raster_path),
|
||||||
|
"reference_path": str(reference_path),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def write_dataset_yaml(output_dir: Path) -> Path:
|
||||||
|
yaml_path = output_dir / "dataset.yaml"
|
||||||
|
yaml_path.write_text(
|
||||||
|
"\n".join(
|
||||||
|
[
|
||||||
|
f"path: {output_dir}",
|
||||||
|
"train: images/train",
|
||||||
|
"val: images/val",
|
||||||
|
"names:",
|
||||||
|
" 0: building",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return yaml_path
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_yolo_directories(output_dir: Path) -> None:
|
||||||
|
for relative_path in ("images/train", "labels/train", "images/val", "labels/val"):
|
||||||
|
(output_dir / relative_path).mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = parse_args()
|
||||||
|
ensure_dependencies()
|
||||||
|
if args.force and args.output_dir.exists():
|
||||||
|
shutil.rmtree(args.output_dir)
|
||||||
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
ensure_yolo_directories(args.output_dir)
|
||||||
|
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
|
||||||
|
samples = manifest.get("samples") or []
|
||||||
|
if not samples:
|
||||||
|
raise SystemExit("Operator sample manifest contains no samples")
|
||||||
|
val_slugs = split_slugs(args.val_samples)
|
||||||
|
exported = [export_sample(sample, args.manifest_path, args.output_dir, val_slugs) for sample in samples]
|
||||||
|
if not any(item["split"] == "train" for item in exported):
|
||||||
|
raise SystemExit("YOLO dataset export produced no training samples")
|
||||||
|
if not any(item["split"] == "val" for item in exported):
|
||||||
|
raise SystemExit("YOLO dataset export produced no validation samples")
|
||||||
|
dataset_yaml = write_dataset_yaml(args.output_dir)
|
||||||
|
summary = {
|
||||||
|
"status": "ok",
|
||||||
|
"dataset_yaml": str(dataset_yaml),
|
||||||
|
"output_dir": str(args.output_dir),
|
||||||
|
"class_names": ["building"],
|
||||||
|
"sample_count": len(exported),
|
||||||
|
"train_sample_count": sum(1 for item in exported if item["split"] == "train"),
|
||||||
|
"val_sample_count": sum(1 for item in exported if item["split"] == "val"),
|
||||||
|
"label_count": sum(item["label_count"] for item in exported),
|
||||||
|
"samples": exported,
|
||||||
|
}
|
||||||
|
summary_path = args.output_dir / "yolo_dataset_summary.json"
|
||||||
|
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
|
||||||
|
print(json.dumps(summary, indent=2, sort_keys=True))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -42,6 +42,7 @@ ${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py
|
|||||||
${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py
|
${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py
|
||||||
${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py
|
${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
|
${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
||||||
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
|
||||||
${PYTHON_BIN} -m compileall backend/app
|
${PYTHON_BIN} -m compileall backend/app
|
||||||
@@ -60,6 +61,7 @@ bash -n scripts/run_detection_calibration_sweep.sh
|
|||||||
bash -n scripts/export_detection_calibration_evidence.sh
|
bash -n scripts/export_detection_calibration_evidence.sh
|
||||||
bash -n scripts/run_detection_quality_matrix.sh
|
bash -n scripts/run_detection_quality_matrix.sh
|
||||||
bash -n scripts/run_multi_sample_detection_quality_matrix.sh
|
bash -n scripts/run_multi_sample_detection_quality_matrix.sh
|
||||||
|
bash -n scripts/train_operator_yolo_detector.sh
|
||||||
bash -n scripts/verify_workbench_default_state.sh
|
bash -n scripts/verify_workbench_default_state.sh
|
||||||
bash -n scripts/verify_workbench_interactions.sh
|
bash -n scripts/verify_workbench_interactions.sh
|
||||||
bash -n scripts/verify_gis_runtime.sh
|
bash -n scripts/verify_gis_runtime.sh
|
||||||
|
|||||||
@@ -0,0 +1,131 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'EOF'
|
||||||
|
Train a local operator YOLO building detector from an exported GeoIntel dataset.
|
||||||
|
|
||||||
|
Environment variables:
|
||||||
|
OPERATOR_YOLO_DATASET_DIR Directory containing dataset.yaml.
|
||||||
|
Default: /app/storage/operator-data/yolo-building-dataset
|
||||||
|
YOLO_BASE_MODEL_PATH Existing local base .pt model path.
|
||||||
|
Default: /app/models/yolov8n.pt
|
||||||
|
TRAIN_OUTPUT_DIR Ultralytics project output directory.
|
||||||
|
Default: /app/storage/training/operator-yolo
|
||||||
|
TRAIN_RUN_NAME Ultralytics run name.
|
||||||
|
Default: geointel-building-detector
|
||||||
|
TRAIN_MODEL_OUTPUT_PATH Destination for the best trained .pt file.
|
||||||
|
Default: /app/models/geointel-building-detector.pt
|
||||||
|
TRAIN_EPOCHS Training epochs. Default: 8
|
||||||
|
TRAIN_IMGSZ Image size. Default: 512
|
||||||
|
TRAIN_BATCH Batch size. Default: 2
|
||||||
|
TRAIN_WORKERS Data-loader workers. Default: 0
|
||||||
|
TRAIN_DEVICE Device passed to Ultralytics. Default: cpu
|
||||||
|
|
||||||
|
This helper is an operator/runtime smoke wrapper. It requires an existing local
|
||||||
|
base model and an existing local dataset.yaml. It does not create app features.
|
||||||
|
EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
OPERATOR_YOLO_DATASET_DIR="${OPERATOR_YOLO_DATASET_DIR:-/app/storage/operator-data/yolo-building-dataset}"
|
||||||
|
YOLO_BASE_MODEL_PATH="${YOLO_BASE_MODEL_PATH:-/app/models/yolov8n.pt}"
|
||||||
|
TRAIN_OUTPUT_DIR="${TRAIN_OUTPUT_DIR:-/app/storage/training/operator-yolo}"
|
||||||
|
TRAIN_RUN_NAME="${TRAIN_RUN_NAME:-geointel-building-detector}"
|
||||||
|
TRAIN_MODEL_OUTPUT_PATH="${TRAIN_MODEL_OUTPUT_PATH:-/app/models/geointel-building-detector.pt}"
|
||||||
|
TRAIN_EPOCHS="${TRAIN_EPOCHS:-8}"
|
||||||
|
TRAIN_IMGSZ="${TRAIN_IMGSZ:-512}"
|
||||||
|
TRAIN_BATCH="${TRAIN_BATCH:-2}"
|
||||||
|
TRAIN_WORKERS="${TRAIN_WORKERS:-0}"
|
||||||
|
TRAIN_DEVICE="${TRAIN_DEVICE:-cpu}"
|
||||||
|
|
||||||
|
DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
|
||||||
|
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
|
||||||
|
export DATASET_YAML
|
||||||
|
export YOLO_BASE_MODEL_PATH
|
||||||
|
export TRAIN_OUTPUT_DIR
|
||||||
|
export TRAIN_RUN_NAME
|
||||||
|
export TRAIN_MODEL_OUTPUT_PATH
|
||||||
|
export TRAIN_EPOCHS
|
||||||
|
export TRAIN_IMGSZ
|
||||||
|
export TRAIN_BATCH
|
||||||
|
export TRAIN_WORKERS
|
||||||
|
export TRAIN_DEVICE
|
||||||
|
export SUMMARY_PATH
|
||||||
|
|
||||||
|
if [[ ! -f "${DATASET_YAML}" ]]; then
|
||||||
|
echo "Dataset YAML not found: ${DATASET_YAML}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
|
||||||
|
echo "Base model file not found: ${YOLO_BASE_MODEL_PATH}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
mkdir -p "${TRAIN_OUTPUT_DIR}" "$(dirname "${TRAIN_MODEL_OUTPUT_PATH}")"
|
||||||
|
|
||||||
|
python - <<'PY'
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ultralytics import YOLO
|
||||||
|
|
||||||
|
dataset_yaml = Path(os.environ["DATASET_YAML"])
|
||||||
|
base_model_path = Path(os.environ["YOLO_BASE_MODEL_PATH"])
|
||||||
|
train_output_dir = Path(os.environ["TRAIN_OUTPUT_DIR"])
|
||||||
|
run_name = os.environ["TRAIN_RUN_NAME"]
|
||||||
|
trained_model_output_path = Path(os.environ["TRAIN_MODEL_OUTPUT_PATH"])
|
||||||
|
summary_path = Path(os.environ["SUMMARY_PATH"])
|
||||||
|
epochs = int(os.environ["TRAIN_EPOCHS"])
|
||||||
|
image_size = int(os.environ["TRAIN_IMGSZ"])
|
||||||
|
batch_size = int(os.environ["TRAIN_BATCH"])
|
||||||
|
workers = int(os.environ["TRAIN_WORKERS"])
|
||||||
|
device = os.environ["TRAIN_DEVICE"]
|
||||||
|
|
||||||
|
model = YOLO(str(base_model_path))
|
||||||
|
model.train(
|
||||||
|
data=str(dataset_yaml),
|
||||||
|
epochs=epochs,
|
||||||
|
imgsz=image_size,
|
||||||
|
batch=batch_size,
|
||||||
|
workers=workers,
|
||||||
|
device=device,
|
||||||
|
project=str(train_output_dir),
|
||||||
|
name=run_name,
|
||||||
|
exist_ok=True,
|
||||||
|
pretrained=True,
|
||||||
|
verbose=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
best_path = train_output_dir / run_name / "weights" / "best.pt"
|
||||||
|
if not best_path.exists():
|
||||||
|
raise SystemExit(f"Expected trained model artifact was not created: {best_path}")
|
||||||
|
|
||||||
|
shutil.copy2(best_path, trained_model_output_path)
|
||||||
|
summary = {
|
||||||
|
"status": "ok",
|
||||||
|
"dataset_yaml": str(dataset_yaml),
|
||||||
|
"base_model_path": str(base_model_path),
|
||||||
|
"train_output_dir": str(train_output_dir),
|
||||||
|
"train_run_name": run_name,
|
||||||
|
"trained_model_path": str(trained_model_output_path),
|
||||||
|
"best_artifact_path": str(best_path),
|
||||||
|
"epochs": epochs,
|
||||||
|
"image_size": image_size,
|
||||||
|
"batch_size": batch_size,
|
||||||
|
"workers": workers,
|
||||||
|
"device": device,
|
||||||
|
}
|
||||||
|
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
|
||||||
|
print(json.dumps(summary, indent=2, sort_keys=True))
|
||||||
|
PY
|
||||||
Reference in New Issue
Block a user