Add operator YOLO training dataset tooling
GeoIntel CI / docs-smoke (push) Has been cancelled
GeoIntel CI / contract-smoke (push) Has been cancelled

This commit is contained in:
Codex
2026-07-07 05:19:16 +02:00
parent 306fcd1b24
commit 31a2aa6138
10 changed files with 607 additions and 0 deletions
+9
View File
@@ -7,6 +7,15 @@
# Changelog # Changelog
## Sprint 129 Operator YOLO training dataset tooling (2026-07-07)
- Added `scripts/export_operator_yolo_dataset.py` to convert prepared operator orthophoto/GRB sample pairs into a standard local YOLO detection dataset with `dataset.yaml`, train/validation image folders, label folders and `yolo_dataset_summary.json`.
- Added `scripts/train_operator_yolo_detector.sh` as an operator-only training smoke wrapper that uses an existing local base `.pt` model and writes a trained local `.pt` artifact plus `training_summary.json`.
- Added readiness coverage for the exporter Python compile check and training wrapper shell syntax.
- Added regression coverage in `backend/tests/test_sprint129_operator_yolo_training_dataset.py`.
- Updated operator documentation for dataset export, training smoke usage and the requirement to benchmark any trained model through the existing real-data Detection + QA matrix before treating it as useful.
- No Training Studio UI, API contract change, provider fetching, model auto-provisioning or app-side model training behavior was introduced.
## Sprint 128 Stronger building model runtime benchmark (2026-07-07) ## Sprint 128 Stronger building model runtime benchmark (2026-07-07)
- Added `keremberke/yolov8s-building-segmentation` as an explicit Tower runtime model asset at `/mnt/user/appdata/geointel/models/yolov8s-building-segmentation.pt`; the file is not committed to Git. - Added `keremberke/yolov8s-building-segmentation` as an explicit Tower runtime model asset at `/mnt/user/appdata/geointel/models/yolov8s-building-segmentation.pt`; the file is not committed to Git.
+32
View File
@@ -278,6 +278,38 @@ python scripts/configure_yolo_model.py \
The smoke loads only the supplied local model file, does not run inference and The smoke loads only the supplied local model file, does not run inference and
does not download weights. does not download weights.
Operator-only local training preparation is available when real public model
candidates are too weak for the target imagery. It is not a browser feature and
does not change API contracts:
```bash
docker exec -it geointel python /app/scripts/export_operator_yolo_dataset.py \
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
--output-dir /app/storage/operator-data/yolo-building-dataset \
--val-samples turnhout \
--force
```
In an AI-enabled runtime with an existing local base model:
```bash
docker exec \
-e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-dataset \
-e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \
-e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-detector.pt \
-e TRAIN_EPOCHS=8 \
-e TRAIN_IMGSZ=512 \
-e TRAIN_BATCH=2 \
-e TRAIN_WORKERS=0 \
-e TRAIN_DEVICE=cpu \
geointel bash /app/scripts/train_operator_yolo_detector.sh
```
The exporter creates a YOLO `dataset.yaml` plus image/label folders from the
explicit operator sample manifest. The training wrapper writes
`training_summary.json` and a local `.pt` artifact, which still must be
validated through model preflight and the real-data QA matrix before use.
The backend also exposes a read-only model asset catalog for the mounted model The backend also exposes a read-only model asset catalog for the mounted model
directory: directory:
@@ -0,0 +1,68 @@
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[2]
def test_operator_yolo_dataset_export_script_contract() -> None:
script_path = ROOT / "scripts" / "export_operator_yolo_dataset.py"
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
assert script_path.exists()
script = script_path.read_text(encoding="utf-8")
assert "py_compile scripts/export_operator_yolo_dataset.py" in readiness
assert "operator_samples_manifest.json" in script
assert "dataset.yaml" in script
assert "images/train" in script
assert "labels/train" in script
assert "images/val" in script
assert "labels/val" in script
assert "building" in script
assert "reference_feature_count" in script
assert "source_name" in script
assert "reference_layer_name" in script
assert "rasterio" in script
assert "Transformer" in script
assert "demo/workflow" not in script
assert "fixture_mode" not in script
assert "will_download_models" not in script
def test_operator_yolo_dataset_export_help_does_not_require_gis_dependencies() -> None:
script_path = ROOT / "scripts" / "export_operator_yolo_dataset.py"
result = subprocess.run(
[sys.executable, str(script_path), "--help"],
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0
assert "Export operator real-data samples to a YOLO detection dataset" in result.stdout
assert "--manifest-path" in result.stdout
assert "--val-samples" in result.stdout
def test_operator_yolo_train_smoke_script_contract() -> None:
script_path = ROOT / "scripts" / "train_operator_yolo_detector.sh"
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
assert script_path.exists()
script = script_path.read_text(encoding="utf-8")
assert "bash -n scripts/train_operator_yolo_detector.sh" in readiness
assert "OPERATOR_YOLO_DATASET_DIR" in script
assert "YOLO_BASE_MODEL_PATH" in script
assert "TRAIN_MODEL_OUTPUT_PATH" in script
assert "TRAIN_EPOCHS" in script
assert "TRAIN_IMGSZ" in script
assert "dataset.yaml" in script
assert "from ultralytics import YOLO" in script
assert "model.train" in script
assert "training_summary.json" in script
assert "download" not in script.lower()
assert "fixture_mode" not in script
+36
View File
@@ -229,6 +229,42 @@ The multi-sample summary exposes `best_overall_by_score`,
model-quality decisions are based on repeated persisted QA/QC evidence rather model-quality decisions are based on repeated persisted QA/QC evidence rather
than one AOI. than one AOI.
When repeated public model benchmarks remain too weak, the operator can convert
the prepared real-data samples into a local YOLO training dataset:
```bash
docker exec -it geointel python /app/scripts/export_operator_yolo_dataset.py \
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
--output-dir /app/storage/operator-data/yolo-building-dataset \
--val-samples turnhout \
--force
```
The exporter creates a standard YOLO detection layout with `dataset.yaml`,
`images/train`, `labels/train`, `images/val` and `labels/val`. It converts GRB
building reference geometries to pixel-space bounding boxes for the matching
orthophoto sample and records `yolo_dataset_summary.json`.
A minimal local training smoke can then be run explicitly in an AI-enabled
runtime:
```bash
docker exec \
-e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-dataset \
-e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \
-e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-detector.pt \
-e TRAIN_EPOCHS=8 \
-e TRAIN_IMGSZ=512 \
-e TRAIN_BATCH=2 \
-e TRAIN_WORKERS=0 \
-e TRAIN_DEVICE=cpu \
geointel bash /app/scripts/train_operator_yolo_detector.sh
```
This remains operator tooling only. GeoIntel does not expose Training Studio in
V1, does not generate labels from predictions and does not treat the trained
artifact as useful until it passes the same real-data Detection + QA matrix.
For visual error inspection, export the persisted QA evidence from a calibration For visual error inspection, export the persisted QA evidence from a calibration
summary: summary:
+32
View File
@@ -1,3 +1,35 @@
## Sprint 129 Operator YOLO training dataset tooling (2026-07-07)
Changed:
- Added `scripts/export_operator_yolo_dataset.py` to export prepared operator samples into a local YOLO detection dataset:
- input manifest: `operator_samples_manifest.json`
- output: `dataset.yaml`, `images/train`, `labels/train`, `images/val`, `labels/val`, `yolo_dataset_summary.json`
- labels are derived from GRB building references with `source_name=grb` and `reference_layer_name=buildings`.
- Added `scripts/train_operator_yolo_detector.sh` as an explicit operator/runtime wrapper around a local Ultralytics training smoke:
- requires `OPERATOR_YOLO_DATASET_DIR`
- requires an existing `YOLO_BASE_MODEL_PATH`
- writes a local `TRAIN_MODEL_OUTPUT_PATH`
- writes `training_summary.json`.
- Added readiness coverage for exporter compile and train-wrapper shell syntax.
- Added regression coverage in `backend/tests/test_sprint129_operator_yolo_training_dataset.py`.
- Updated `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`.
Tested:
- RED: `python -m pytest backend\tests\test_sprint129_operator_yolo_training_dataset.py -q` failed while the exporter and training wrapper contracts were incomplete.
- `python -m pytest backend\tests\test_sprint129_operator_yolo_training_dataset.py -q` passed.
- `python scripts\export_operator_yolo_dataset.py --help` passed without requiring local GIS dependencies.
- `python -m py_compile scripts\export_operator_yolo_dataset.py` passed.
- `bash -n scripts/train_operator_yolo_detector.sh` passed.
Open:
- Run the exporter and training smoke inside the AI-enabled Tower runtime, then benchmark the trained artifact through the existing multi-sample Detection + QA matrix.
Limitations:
- This is operator tooling only. It does not add Training Studio, browser training controls, provider fetching, fake detections, model auto-provisioning or API contract changes.
Next recommended pass:
- Generate the local YOLO dataset from the current Geel/Mol/Turnhout samples, train a small local model smoke from `yolov8n.pt`, and compare it against the current `yolov8s-building-segmentation-pt` benchmark.
## Sprint 128 Stronger building model runtime benchmark (2026-07-07) ## Sprint 128 Stronger building model runtime benchmark (2026-07-07)
Changed: Changed:
+2
View File
@@ -404,3 +404,5 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Render persisted QA/QC feature-level evidence as Map workspace overlays. - [x] Render persisted QA/QC feature-level evidence as Map workspace overlays.
- [x] Add safe local YOLO model env configuration helper for Unraid/Tower runtime activation. - [x] Add safe local YOLO model env configuration helper for Unraid/Tower runtime activation.
- [x] Harden configured YOLO inference for single-band raster tiles and wrapped runtime errors. - [x] Harden configured YOLO inference for single-band raster tiles and wrapped runtime errors.
- [x] Add operator-only YOLO dataset export and local training-smoke wrapper for real sample calibration.
- [ ] Train/evaluate a local GeoIntel building detector from the operator samples and only activate it after QA/QC matrix improvement.
+40
View File
@@ -261,6 +261,46 @@ plus a combined `multi_sample_quality_summary.json` with
container-style `/app/storage/...` manifest paths to repo-relative container-style `/app/storage/...` manifest paths to repo-relative
`storage/...` paths when run from the Tower host checkout. `storage/...` paths when run from the Tower host checkout.
Export the same operator samples to a local YOLO detection dataset when the
public model candidates are not strong enough for the target imagery:
```bash
docker exec -it geointel python /app/scripts/export_operator_yolo_dataset.py \
--manifest-path /app/storage/operator-data/operator_samples_manifest.json \
--output-dir /app/storage/operator-data/yolo-building-dataset \
--val-samples turnhout \
--force
```
The exporter writes `dataset.yaml`, `images/train`, `labels/train`,
`images/val`, `labels/val` and `yolo_dataset_summary.json`. It uses only the
explicit operator sample manifest and GRB building references where
`source_name=grb` and `reference_layer_name=buildings`. It does not call
GeoIntel APIs, create provider data, run inference or train a model.
Run a small local training smoke only in an AI-enabled runtime with an existing
local base model file:
```bash
docker exec \
-e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-dataset \
-e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \
-e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-detector.pt \
-e TRAIN_EPOCHS=8 \
-e TRAIN_IMGSZ=512 \
-e TRAIN_BATCH=2 \
-e TRAIN_WORKERS=0 \
-e TRAIN_DEVICE=cpu \
geointel bash /app/scripts/train_operator_yolo_detector.sh
```
The training wrapper is intentionally outside the product UI. It runs
Ultralytics from the existing runtime, copies the best trained artifact to
`TRAIN_MODEL_OUTPUT_PATH` and writes `training_summary.json`. Afterward, treat
the resulting `.pt` file like any other local model asset: verify preflight,
run the real-data matrix and compare persisted QA/QC metrics before activating
it as a useful default.
Export calibration QA evidence for visual review: Export calibration QA evidence for visual review:
```bash ```bash
+255
View File
@@ -0,0 +1,255 @@
"""Export operator real-data samples to a YOLO detection dataset.
This is an operator/runtime helper. It converts the explicit orthophoto + GRB
reference sample manifest into local YOLO images/labels for model experiments.
It does not call GeoIntel APIs, does not train automatically and does not fetch
new provider data.
"""
from __future__ import annotations
import argparse
import json
import os
import shutil
import sys
from pathlib import Path
from typing import Any, Iterable
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-dataset")
rasterio: Any = None
Transformer: Any = None
Image: Any = None
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Export operator real-data samples to a YOLO detection dataset.",
)
parser.add_argument(
"--manifest-path",
type=Path,
default=Path(os.environ.get("OPERATOR_SAMPLE_MANIFEST_PATH", DEFAULT_MANIFEST_PATH)),
help="operator_samples_manifest.json created by prepare_operator_real_data_samples.py.",
)
parser.add_argument(
"--output-dir",
type=Path,
default=Path(os.environ.get("OPERATOR_YOLO_DATASET_DIR", DEFAULT_OUTPUT_DIR)),
help="Output directory for images, labels, dataset.yaml and summary JSON.",
)
parser.add_argument(
"--val-samples",
default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"),
help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.",
)
parser.add_argument(
"--force",
action="store_true",
help="Remove and recreate output-dir before exporting.",
)
return parser.parse_args()
def ensure_dependencies() -> None:
global Image, Transformer, rasterio
try:
import rasterio as rasterio_module
from PIL import Image as image_module
from pyproj import Transformer as transformer_class
except Exception as exc: # pragma: no cover - runtime environment only.
raise SystemExit(
"export_operator_yolo_dataset.py requires rasterio, pyproj and Pillow. "
"Run it inside the GeoIntel all-in-one container or an equivalent GIS Python environment."
) from exc
rasterio = rasterio_module
Transformer = transformer_class
Image = image_module
def split_slugs(raw: str) -> set[str]:
return {value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()}
def iter_geometry_coords(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
geometry_type = geometry.get("type")
coordinates = geometry.get("coordinates")
if geometry_type == "Polygon":
for ring in coordinates or []:
for point in ring:
if len(point) >= 2:
yield float(point[0]), float(point[1])
elif geometry_type == "MultiPolygon":
for polygon in coordinates or []:
for ring in polygon:
for point in ring:
if len(point) >= 2:
yield float(point[0]), float(point[1])
def resolve_manifest_path(raw: str, manifest_path: Path) -> Path:
path = Path(raw)
if path.exists():
return path
if raw.startswith("/app/"):
relative = Path(raw.removeprefix("/app/"))
candidates = [
Path.cwd() / relative,
manifest_path.resolve().parent.parent.parent / relative,
]
for candidate in candidates:
if candidate.exists():
return candidate
return path
def image_array_from_raster(dataset: Any) -> Any:
import numpy as np
data = dataset.read()
if data.shape[0] == 1:
rgb = np.repeat(data[:1], 3, axis=0)
else:
rgb = data[:3]
rgb = np.moveaxis(rgb, 0, -1)
if rgb.dtype != np.uint8:
rgb_min = float(np.nanmin(rgb))
rgb_max = float(np.nanmax(rgb))
if rgb_max > rgb_min:
rgb = ((rgb - rgb_min) / (rgb_max - rgb_min) * 255.0).clip(0, 255).astype("uint8")
else:
rgb = np.zeros(rgb.shape, dtype="uint8")
return rgb
def yolo_boxes_for_reference(reference_path: Path, dataset: Any) -> list[str]:
reference = json.loads(reference_path.read_text(encoding="utf-8-sig"))
features = reference.get("features") or []
transformer = Transformer.from_crs("EPSG:4326", dataset.crs, always_xy=True)
width = dataset.width
height = dataset.height
labels: list[str] = []
for feature in features:
properties = feature.get("properties") or {}
if properties.get("source_name") != "grb":
continue
if properties.get("reference_layer_name") != "buildings":
continue
coords = list(iter_geometry_coords(feature.get("geometry") or {}))
if not coords:
continue
xs, ys = zip(*(transformer.transform(lon, lat) for lon, lat in coords), strict=False)
rows_cols = [dataset.index(x, y) for x, y in zip(xs, ys, strict=False)]
rows = [row for row, _ in rows_cols]
cols = [col for _, col in rows_cols]
min_col = max(0, min(cols))
max_col = min(width - 1, max(cols))
min_row = max(0, min(rows))
max_row = min(height - 1, max(rows))
box_width = max_col - min_col
box_height = max_row - min_row
if box_width < 2 or box_height < 2:
continue
x_center = (min_col + max_col) / 2.0 / width
y_center = (min_row + max_row) / 2.0 / height
norm_width = box_width / width
norm_height = box_height / height
labels.append(f"0 {x_center:.8f} {y_center:.8f} {norm_width:.8f} {norm_height:.8f}")
return labels
def export_sample(sample: dict[str, Any], manifest_path: Path, output_dir: Path, val_slugs: set[str]) -> dict[str, Any]:
sample_slug = str(sample["sample_slug"])
split = "val" if sample_slug.lower() in val_slugs else "train"
raster_path = resolve_manifest_path(str(sample["raster_path"]), manifest_path)
reference_path = resolve_manifest_path(str(sample["reference_path"]), manifest_path)
if not raster_path.exists():
raise SystemExit(f"Raster path is not readable for sample {sample_slug}: {raster_path}")
if not reference_path.exists():
raise SystemExit(f"Reference path is not readable for sample {sample_slug}: {reference_path}")
image_path = output_dir / "images" / split / f"{sample_slug}.png"
label_path = output_dir / "labels" / split / f"{sample_slug}.txt"
image_path.parent.mkdir(parents=True, exist_ok=True)
label_path.parent.mkdir(parents=True, exist_ok=True)
with rasterio.open(raster_path) as dataset:
image_array = image_array_from_raster(dataset)
Image.fromarray(image_array).save(image_path)
labels = yolo_boxes_for_reference(reference_path, dataset)
label_path.write_text("\n".join(labels) + ("\n" if labels else ""), encoding="utf-8")
return {
"sample_slug": sample_slug,
"split": split,
"image_path": str(image_path),
"label_path": str(label_path),
"label_count": len(labels),
"reference_feature_count": sample.get("reference_feature_count"),
"raster_path": str(raster_path),
"reference_path": str(reference_path),
}
def write_dataset_yaml(output_dir: Path) -> Path:
yaml_path = output_dir / "dataset.yaml"
yaml_path.write_text(
"\n".join(
[
f"path: {output_dir}",
"train: images/train",
"val: images/val",
"names:",
" 0: building",
"",
]
),
encoding="utf-8",
)
return yaml_path
def ensure_yolo_directories(output_dir: Path) -> None:
for relative_path in ("images/train", "labels/train", "images/val", "labels/val"):
(output_dir / relative_path).mkdir(parents=True, exist_ok=True)
def main() -> int:
args = parse_args()
ensure_dependencies()
if args.force and args.output_dir.exists():
shutil.rmtree(args.output_dir)
args.output_dir.mkdir(parents=True, exist_ok=True)
ensure_yolo_directories(args.output_dir)
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
samples = manifest.get("samples") or []
if not samples:
raise SystemExit("Operator sample manifest contains no samples")
val_slugs = split_slugs(args.val_samples)
exported = [export_sample(sample, args.manifest_path, args.output_dir, val_slugs) for sample in samples]
if not any(item["split"] == "train" for item in exported):
raise SystemExit("YOLO dataset export produced no training samples")
if not any(item["split"] == "val" for item in exported):
raise SystemExit("YOLO dataset export produced no validation samples")
dataset_yaml = write_dataset_yaml(args.output_dir)
summary = {
"status": "ok",
"dataset_yaml": str(dataset_yaml),
"output_dir": str(args.output_dir),
"class_names": ["building"],
"sample_count": len(exported),
"train_sample_count": sum(1 for item in exported if item["split"] == "train"),
"val_sample_count": sum(1 for item in exported if item["split"] == "val"),
"label_count": sum(item["label_count"] for item in exported),
"samples": exported,
}
summary_path = args.output_dir / "yolo_dataset_summary.json"
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
print(json.dumps(summary, indent=2, sort_keys=True))
return 0
if __name__ == "__main__":
sys.exit(main())
+2
View File
@@ -42,6 +42,7 @@ ${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py
${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py ${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py
${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py ${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py
${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py ${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py ${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
${PYTHON_BIN} -m compileall backend/app ${PYTHON_BIN} -m compileall backend/app
@@ -60,6 +61,7 @@ bash -n scripts/run_detection_calibration_sweep.sh
bash -n scripts/export_detection_calibration_evidence.sh bash -n scripts/export_detection_calibration_evidence.sh
bash -n scripts/run_detection_quality_matrix.sh bash -n scripts/run_detection_quality_matrix.sh
bash -n scripts/run_multi_sample_detection_quality_matrix.sh bash -n scripts/run_multi_sample_detection_quality_matrix.sh
bash -n scripts/train_operator_yolo_detector.sh
bash -n scripts/verify_workbench_default_state.sh bash -n scripts/verify_workbench_default_state.sh
bash -n scripts/verify_workbench_interactions.sh bash -n scripts/verify_workbench_interactions.sh
bash -n scripts/verify_gis_runtime.sh bash -n scripts/verify_gis_runtime.sh
+131
View File
@@ -0,0 +1,131 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Train a local operator YOLO building detector from an exported GeoIntel dataset.
Environment variables:
OPERATOR_YOLO_DATASET_DIR Directory containing dataset.yaml.
Default: /app/storage/operator-data/yolo-building-dataset
YOLO_BASE_MODEL_PATH Existing local base .pt model path.
Default: /app/models/yolov8n.pt
TRAIN_OUTPUT_DIR Ultralytics project output directory.
Default: /app/storage/training/operator-yolo
TRAIN_RUN_NAME Ultralytics run name.
Default: geointel-building-detector
TRAIN_MODEL_OUTPUT_PATH Destination for the best trained .pt file.
Default: /app/models/geointel-building-detector.pt
TRAIN_EPOCHS Training epochs. Default: 8
TRAIN_IMGSZ Image size. Default: 512
TRAIN_BATCH Batch size. Default: 2
TRAIN_WORKERS Data-loader workers. Default: 0
TRAIN_DEVICE Device passed to Ultralytics. Default: cpu
This helper is an operator/runtime smoke wrapper. It requires an existing local
base model and an existing local dataset.yaml. It does not create app features.
EOF
}
if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
usage
exit 0
fi
OPERATOR_YOLO_DATASET_DIR="${OPERATOR_YOLO_DATASET_DIR:-/app/storage/operator-data/yolo-building-dataset}"
YOLO_BASE_MODEL_PATH="${YOLO_BASE_MODEL_PATH:-/app/models/yolov8n.pt}"
TRAIN_OUTPUT_DIR="${TRAIN_OUTPUT_DIR:-/app/storage/training/operator-yolo}"
TRAIN_RUN_NAME="${TRAIN_RUN_NAME:-geointel-building-detector}"
TRAIN_MODEL_OUTPUT_PATH="${TRAIN_MODEL_OUTPUT_PATH:-/app/models/geointel-building-detector.pt}"
TRAIN_EPOCHS="${TRAIN_EPOCHS:-8}"
TRAIN_IMGSZ="${TRAIN_IMGSZ:-512}"
TRAIN_BATCH="${TRAIN_BATCH:-2}"
TRAIN_WORKERS="${TRAIN_WORKERS:-0}"
TRAIN_DEVICE="${TRAIN_DEVICE:-cpu}"
DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
export DATASET_YAML
export YOLO_BASE_MODEL_PATH
export TRAIN_OUTPUT_DIR
export TRAIN_RUN_NAME
export TRAIN_MODEL_OUTPUT_PATH
export TRAIN_EPOCHS
export TRAIN_IMGSZ
export TRAIN_BATCH
export TRAIN_WORKERS
export TRAIN_DEVICE
export SUMMARY_PATH
if [[ ! -f "${DATASET_YAML}" ]]; then
echo "Dataset YAML not found: ${DATASET_YAML}" >&2
exit 1
fi
if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
echo "Base model file not found: ${YOLO_BASE_MODEL_PATH}" >&2
exit 1
fi
mkdir -p "${TRAIN_OUTPUT_DIR}" "$(dirname "${TRAIN_MODEL_OUTPUT_PATH}")"
python - <<'PY'
from __future__ import annotations
import json
import os
import shutil
from pathlib import Path
from ultralytics import YOLO
dataset_yaml = Path(os.environ["DATASET_YAML"])
base_model_path = Path(os.environ["YOLO_BASE_MODEL_PATH"])
train_output_dir = Path(os.environ["TRAIN_OUTPUT_DIR"])
run_name = os.environ["TRAIN_RUN_NAME"]
trained_model_output_path = Path(os.environ["TRAIN_MODEL_OUTPUT_PATH"])
summary_path = Path(os.environ["SUMMARY_PATH"])
epochs = int(os.environ["TRAIN_EPOCHS"])
image_size = int(os.environ["TRAIN_IMGSZ"])
batch_size = int(os.environ["TRAIN_BATCH"])
workers = int(os.environ["TRAIN_WORKERS"])
device = os.environ["TRAIN_DEVICE"]
model = YOLO(str(base_model_path))
model.train(
data=str(dataset_yaml),
epochs=epochs,
imgsz=image_size,
batch=batch_size,
workers=workers,
device=device,
project=str(train_output_dir),
name=run_name,
exist_ok=True,
pretrained=True,
verbose=True,
)
best_path = train_output_dir / run_name / "weights" / "best.pt"
if not best_path.exists():
raise SystemExit(f"Expected trained model artifact was not created: {best_path}")
shutil.copy2(best_path, trained_model_output_path)
summary = {
"status": "ok",
"dataset_yaml": str(dataset_yaml),
"base_model_path": str(base_model_path),
"train_output_dir": str(train_output_dir),
"train_run_name": run_name,
"trained_model_path": str(trained_model_output_path),
"best_artifact_path": str(best_path),
"epochs": epochs,
"image_size": image_size,
"batch_size": batch_size,
"workers": workers,
"device": device,
}
summary_path.parent.mkdir(parents=True, exist_ok=True)
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
print(json.dumps(summary, indent=2, sort_keys=True))
PY