Add operator YOLO training dataset tooling

scripts/README.md

@@ -261,6 +261,46 @@ plus a combined `multi_sample_quality_summary.json` with
 container-style `/app/storage/...` manifest paths to repo-relative
 `storage/...` paths when run from the Tower host checkout.
 
+Export the same operator samples to a local YOLO detection dataset when the
+public model candidates are not strong enough for the target imagery:
+
+```bash
+docker exec -it geointel python /app/scripts/export_operator_yolo_dataset.py \
+  --manifest-path /app/storage/operator-data/operator_samples_manifest.json \
+  --output-dir /app/storage/operator-data/yolo-building-dataset \
+  --val-samples turnhout \
+  --force
+```
+
+The exporter writes `dataset.yaml`, `images/train`, `labels/train`,
+`images/val`, `labels/val` and `yolo_dataset_summary.json`. It uses only the
+explicit operator sample manifest and GRB building references where
+`source_name=grb` and `reference_layer_name=buildings`. It does not call
+GeoIntel APIs, create provider data, run inference or train a model.
+
+Run a small local training smoke only in an AI-enabled runtime with an existing
+local base model file:
+
+```bash
+docker exec \
+  -e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-dataset \
+  -e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \
+  -e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-detector.pt \
+  -e TRAIN_EPOCHS=8 \
+  -e TRAIN_IMGSZ=512 \
+  -e TRAIN_BATCH=2 \
+  -e TRAIN_WORKERS=0 \
+  -e TRAIN_DEVICE=cpu \
+  geointel bash /app/scripts/train_operator_yolo_detector.sh
+```
+
+The training wrapper is intentionally outside the product UI. It runs
+Ultralytics from the existing runtime, copies the best trained artifact to
+`TRAIN_MODEL_OUTPUT_PATH` and writes `training_summary.json`. Afterward, treat
+the resulting `.pt` file like any other local model asset: verify preflight,
+run the real-data matrix and compare persisted QA/QC metrics before activating
+it as a useful default.
+
 Export calibration QA evidence for visual review:
 
 ```bash

scripts/export_operator_yolo_dataset.py

@@ -0,0 +1,255 @@
+"""Export operator real-data samples to a YOLO detection dataset.
+
+This is an operator/runtime helper. It converts the explicit orthophoto + GRB
+reference sample manifest into local YOLO images/labels for model experiments.
+It does not call GeoIntel APIs, does not train automatically and does not fetch
+new provider data.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import sys
+from pathlib import Path
+from typing import Any, Iterable
+
+
+DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
+DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-dataset")
+rasterio: Any = None
+Transformer: Any = None
+Image: Any = None
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Export operator real-data samples to a YOLO detection dataset.",
+    )
+    parser.add_argument(
+        "--manifest-path",
+        type=Path,
+        default=Path(os.environ.get("OPERATOR_SAMPLE_MANIFEST_PATH", DEFAULT_MANIFEST_PATH)),
+        help="operator_samples_manifest.json created by prepare_operator_real_data_samples.py.",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=Path(os.environ.get("OPERATOR_YOLO_DATASET_DIR", DEFAULT_OUTPUT_DIR)),
+        help="Output directory for images, labels, dataset.yaml and summary JSON.",
+    )
+    parser.add_argument(
+        "--val-samples",
+        default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"),
+        help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Remove and recreate output-dir before exporting.",
+    )
+    return parser.parse_args()
+
+
+def ensure_dependencies() -> None:
+    global Image, Transformer, rasterio
+    try:
+        import rasterio as rasterio_module
+        from PIL import Image as image_module
+        from pyproj import Transformer as transformer_class
+    except Exception as exc:  # pragma: no cover - runtime environment only.
+        raise SystemExit(
+            "export_operator_yolo_dataset.py requires rasterio, pyproj and Pillow. "
+            "Run it inside the GeoIntel all-in-one container or an equivalent GIS Python environment."
+        ) from exc
+    rasterio = rasterio_module
+    Transformer = transformer_class
+    Image = image_module
+
+
+def split_slugs(raw: str) -> set[str]:
+    return {value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()}
+
+
+def iter_geometry_coords(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
+    geometry_type = geometry.get("type")
+    coordinates = geometry.get("coordinates")
+    if geometry_type == "Polygon":
+        for ring in coordinates or []:
+            for point in ring:
+                if len(point) >= 2:
+                    yield float(point[0]), float(point[1])
+    elif geometry_type == "MultiPolygon":
+        for polygon in coordinates or []:
+            for ring in polygon:
+                for point in ring:
+                    if len(point) >= 2:
+                        yield float(point[0]), float(point[1])
+
+
+def resolve_manifest_path(raw: str, manifest_path: Path) -> Path:
+    path = Path(raw)
+    if path.exists():
+        return path
+    if raw.startswith("/app/"):
+        relative = Path(raw.removeprefix("/app/"))
+        candidates = [
+            Path.cwd() / relative,
+            manifest_path.resolve().parent.parent.parent / relative,
+        ]
+        for candidate in candidates:
+            if candidate.exists():
+                return candidate
+    return path
+
+
+def image_array_from_raster(dataset: Any) -> Any:
+    import numpy as np
+
+    data = dataset.read()
+    if data.shape[0] == 1:
+        rgb = np.repeat(data[:1], 3, axis=0)
+    else:
+        rgb = data[:3]
+    rgb = np.moveaxis(rgb, 0, -1)
+    if rgb.dtype != np.uint8:
+        rgb_min = float(np.nanmin(rgb))
+        rgb_max = float(np.nanmax(rgb))
+        if rgb_max > rgb_min:
+            rgb = ((rgb - rgb_min) / (rgb_max - rgb_min) * 255.0).clip(0, 255).astype("uint8")
+        else:
+            rgb = np.zeros(rgb.shape, dtype="uint8")
+    return rgb
+
+
+def yolo_boxes_for_reference(reference_path: Path, dataset: Any) -> list[str]:
+    reference = json.loads(reference_path.read_text(encoding="utf-8-sig"))
+    features = reference.get("features") or []
+    transformer = Transformer.from_crs("EPSG:4326", dataset.crs, always_xy=True)
+    width = dataset.width
+    height = dataset.height
+    labels: list[str] = []
+
+    for feature in features:
+        properties = feature.get("properties") or {}
+        if properties.get("source_name") != "grb":
+            continue
+        if properties.get("reference_layer_name") != "buildings":
+            continue
+        coords = list(iter_geometry_coords(feature.get("geometry") or {}))
+        if not coords:
+            continue
+        xs, ys = zip(*(transformer.transform(lon, lat) for lon, lat in coords), strict=False)
+        rows_cols = [dataset.index(x, y) for x, y in zip(xs, ys, strict=False)]
+        rows = [row for row, _ in rows_cols]
+        cols = [col for _, col in rows_cols]
+        min_col = max(0, min(cols))
+        max_col = min(width - 1, max(cols))
+        min_row = max(0, min(rows))
+        max_row = min(height - 1, max(rows))
+        box_width = max_col - min_col
+        box_height = max_row - min_row
+        if box_width < 2 or box_height < 2:
+            continue
+        x_center = (min_col + max_col) / 2.0 / width
+        y_center = (min_row + max_row) / 2.0 / height
+        norm_width = box_width / width
+        norm_height = box_height / height
+        labels.append(f"0 {x_center:.8f} {y_center:.8f} {norm_width:.8f} {norm_height:.8f}")
+    return labels
+
+
+def export_sample(sample: dict[str, Any], manifest_path: Path, output_dir: Path, val_slugs: set[str]) -> dict[str, Any]:
+    sample_slug = str(sample["sample_slug"])
+    split = "val" if sample_slug.lower() in val_slugs else "train"
+    raster_path = resolve_manifest_path(str(sample["raster_path"]), manifest_path)
+    reference_path = resolve_manifest_path(str(sample["reference_path"]), manifest_path)
+    if not raster_path.exists():
+        raise SystemExit(f"Raster path is not readable for sample {sample_slug}: {raster_path}")
+    if not reference_path.exists():
+        raise SystemExit(f"Reference path is not readable for sample {sample_slug}: {reference_path}")
+
+    image_path = output_dir / "images" / split / f"{sample_slug}.png"
+    label_path = output_dir / "labels" / split / f"{sample_slug}.txt"
+    image_path.parent.mkdir(parents=True, exist_ok=True)
+    label_path.parent.mkdir(parents=True, exist_ok=True)
+
+    with rasterio.open(raster_path) as dataset:
+        image_array = image_array_from_raster(dataset)
+        Image.fromarray(image_array).save(image_path)
+        labels = yolo_boxes_for_reference(reference_path, dataset)
+    label_path.write_text("\n".join(labels) + ("\n" if labels else ""), encoding="utf-8")
+    return {
+        "sample_slug": sample_slug,
+        "split": split,
+        "image_path": str(image_path),
+        "label_path": str(label_path),
+        "label_count": len(labels),
+        "reference_feature_count": sample.get("reference_feature_count"),
+        "raster_path": str(raster_path),
+        "reference_path": str(reference_path),
+    }
+
+
+def write_dataset_yaml(output_dir: Path) -> Path:
+    yaml_path = output_dir / "dataset.yaml"
+    yaml_path.write_text(
+        "\n".join(
+            [
+                f"path: {output_dir}",
+                "train: images/train",
+                "val: images/val",
+                "names:",
+                "  0: building",
+                "",
+            ]
+        ),
+        encoding="utf-8",
+    )
+    return yaml_path
+
+
+def ensure_yolo_directories(output_dir: Path) -> None:
+    for relative_path in ("images/train", "labels/train", "images/val", "labels/val"):
+        (output_dir / relative_path).mkdir(parents=True, exist_ok=True)
+
+
+def main() -> int:
+    args = parse_args()
+    ensure_dependencies()
+    if args.force and args.output_dir.exists():
+        shutil.rmtree(args.output_dir)
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    ensure_yolo_directories(args.output_dir)
+    manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
+    samples = manifest.get("samples") or []
+    if not samples:
+        raise SystemExit("Operator sample manifest contains no samples")
+    val_slugs = split_slugs(args.val_samples)
+    exported = [export_sample(sample, args.manifest_path, args.output_dir, val_slugs) for sample in samples]
+    if not any(item["split"] == "train" for item in exported):
+        raise SystemExit("YOLO dataset export produced no training samples")
+    if not any(item["split"] == "val" for item in exported):
+        raise SystemExit("YOLO dataset export produced no validation samples")
+    dataset_yaml = write_dataset_yaml(args.output_dir)
+    summary = {
+        "status": "ok",
+        "dataset_yaml": str(dataset_yaml),
+        "output_dir": str(args.output_dir),
+        "class_names": ["building"],
+        "sample_count": len(exported),
+        "train_sample_count": sum(1 for item in exported if item["split"] == "train"),
+        "val_sample_count": sum(1 for item in exported if item["split"] == "val"),
+        "label_count": sum(item["label_count"] for item in exported),
+        "samples": exported,
+    }
+    summary_path = args.output_dir / "yolo_dataset_summary.json"
+    summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
+    print(json.dumps(summary, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/run_readiness_check.sh

@@ -42,6 +42,7 @@ ${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py
 ${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py
 ${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py
 ${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
+${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
 ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
 ${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
 ${PYTHON_BIN} -m compileall backend/app
@@ -60,6 +61,7 @@ bash -n scripts/run_detection_calibration_sweep.sh
 bash -n scripts/export_detection_calibration_evidence.sh
 bash -n scripts/run_detection_quality_matrix.sh
 bash -n scripts/run_multi_sample_detection_quality_matrix.sh
+bash -n scripts/train_operator_yolo_detector.sh
 bash -n scripts/verify_workbench_default_state.sh
 bash -n scripts/verify_workbench_interactions.sh
 bash -n scripts/verify_gis_runtime.sh

scripts/train_operator_yolo_detector.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Train a local operator YOLO building detector from an exported GeoIntel dataset.
+
+Environment variables:
+  OPERATOR_YOLO_DATASET_DIR   Directory containing dataset.yaml.
+                               Default: /app/storage/operator-data/yolo-building-dataset
+  YOLO_BASE_MODEL_PATH         Existing local base .pt model path.
+                               Default: /app/models/yolov8n.pt
+  TRAIN_OUTPUT_DIR             Ultralytics project output directory.
+                               Default: /app/storage/training/operator-yolo
+  TRAIN_RUN_NAME               Ultralytics run name.
+                               Default: geointel-building-detector
+  TRAIN_MODEL_OUTPUT_PATH      Destination for the best trained .pt file.
+                               Default: /app/models/geointel-building-detector.pt
+  TRAIN_EPOCHS                 Training epochs. Default: 8
+  TRAIN_IMGSZ                  Image size. Default: 512
+  TRAIN_BATCH                  Batch size. Default: 2
+  TRAIN_WORKERS                Data-loader workers. Default: 0
+  TRAIN_DEVICE                 Device passed to Ultralytics. Default: cpu
+
+This helper is an operator/runtime smoke wrapper. It requires an existing local
+base model and an existing local dataset.yaml. It does not create app features.
+EOF
+}
+
+if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
+  usage
+  exit 0
+fi
+
+OPERATOR_YOLO_DATASET_DIR="${OPERATOR_YOLO_DATASET_DIR:-/app/storage/operator-data/yolo-building-dataset}"
+YOLO_BASE_MODEL_PATH="${YOLO_BASE_MODEL_PATH:-/app/models/yolov8n.pt}"
+TRAIN_OUTPUT_DIR="${TRAIN_OUTPUT_DIR:-/app/storage/training/operator-yolo}"
+TRAIN_RUN_NAME="${TRAIN_RUN_NAME:-geointel-building-detector}"
+TRAIN_MODEL_OUTPUT_PATH="${TRAIN_MODEL_OUTPUT_PATH:-/app/models/geointel-building-detector.pt}"
+TRAIN_EPOCHS="${TRAIN_EPOCHS:-8}"
+TRAIN_IMGSZ="${TRAIN_IMGSZ:-512}"
+TRAIN_BATCH="${TRAIN_BATCH:-2}"
+TRAIN_WORKERS="${TRAIN_WORKERS:-0}"
+TRAIN_DEVICE="${TRAIN_DEVICE:-cpu}"
+
+DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
+SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
+export DATASET_YAML
+export YOLO_BASE_MODEL_PATH
+export TRAIN_OUTPUT_DIR
+export TRAIN_RUN_NAME
+export TRAIN_MODEL_OUTPUT_PATH
+export TRAIN_EPOCHS
+export TRAIN_IMGSZ
+export TRAIN_BATCH
+export TRAIN_WORKERS
+export TRAIN_DEVICE
+export SUMMARY_PATH
+
+if [[ ! -f "${DATASET_YAML}" ]]; then
+  echo "Dataset YAML not found: ${DATASET_YAML}" >&2
+  exit 1
+fi
+
+if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
+  echo "Base model file not found: ${YOLO_BASE_MODEL_PATH}" >&2
+  exit 1
+fi
+
+mkdir -p "${TRAIN_OUTPUT_DIR}" "$(dirname "${TRAIN_MODEL_OUTPUT_PATH}")"
+
+python - <<'PY'
+from __future__ import annotations
+
+import json
+import os
+import shutil
+from pathlib import Path
+
+from ultralytics import YOLO
+
+dataset_yaml = Path(os.environ["DATASET_YAML"])
+base_model_path = Path(os.environ["YOLO_BASE_MODEL_PATH"])
+train_output_dir = Path(os.environ["TRAIN_OUTPUT_DIR"])
+run_name = os.environ["TRAIN_RUN_NAME"]
+trained_model_output_path = Path(os.environ["TRAIN_MODEL_OUTPUT_PATH"])
+summary_path = Path(os.environ["SUMMARY_PATH"])
+epochs = int(os.environ["TRAIN_EPOCHS"])
+image_size = int(os.environ["TRAIN_IMGSZ"])
+batch_size = int(os.environ["TRAIN_BATCH"])
+workers = int(os.environ["TRAIN_WORKERS"])
+device = os.environ["TRAIN_DEVICE"]
+
+model = YOLO(str(base_model_path))
+model.train(
+    data=str(dataset_yaml),
+    epochs=epochs,
+    imgsz=image_size,
+    batch=batch_size,
+    workers=workers,
+    device=device,
+    project=str(train_output_dir),
+    name=run_name,
+    exist_ok=True,
+    pretrained=True,
+    verbose=True,
+)
+
+best_path = train_output_dir / run_name / "weights" / "best.pt"
+if not best_path.exists():
+    raise SystemExit(f"Expected trained model artifact was not created: {best_path}")
+
+shutil.copy2(best_path, trained_model_output_path)
+summary = {
+    "status": "ok",
+    "dataset_yaml": str(dataset_yaml),
+    "base_model_path": str(base_model_path),
+    "train_output_dir": str(train_output_dir),
+    "train_run_name": run_name,
+    "trained_model_path": str(trained_model_output_path),
+    "best_artifact_path": str(best_path),
+    "epochs": epochs,
+    "image_size": image_size,
+    "batch_size": batch_size,
+    "workers": workers,
+    "device": device,
+}
+summary_path.parent.mkdir(parents=True, exist_ok=True)
+summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
+print(json.dumps(summary, indent=2, sort_keys=True))
+PY