geointel/scripts/train_operator_yolo_detector.sh

#!/usr/bin/env bash
set -euo pipefail

usage() {
  cat <<'EOF'
Train a local operator YOLO detector from an exported, governed GeoIntel dataset.

Environment variables:
  OPERATOR_YOLO_DATASET_DIR   Directory containing dataset.yaml.
                               Default: /app/storage/operator-data/yolo-building-dataset
  YOLO_BASE_MODEL_PATH         Existing local base .pt model path.
                               Default: /app/models/yolov8n.pt
  TRAIN_OUTPUT_DIR             Ultralytics project output directory.
                               Default: /app/storage/training/operator-yolo
  TRAIN_RUN_NAME               Ultralytics run name.
                               Default: geointel-building-detector
  TRAIN_MODEL_OUTPUT_PATH      Destination for the best trained .pt file.
                               Default: /app/models/geointel-building-detector.pt
  TRAIN_EPOCHS                 Training epochs. Default: 8
  TRAIN_IMGSZ                  Image size. Default: 512
  TRAIN_BATCH                  Batch size. Default: 2
  TRAIN_WORKERS                Data-loader workers. Default: 0
  TRAIN_DEVICE                 Device passed to Ultralytics. Default: cpu
  PYTHON_BIN                   Python executable. Default: /opt/geointel/venv/bin/python
                               when present, otherwise python3.

This helper is an operator/runtime smoke wrapper. It requires an existing local
base model and an existing local dataset.yaml. It does not create app features.
EOF
}

if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
  usage
  exit 0
fi

OPERATOR_YOLO_DATASET_DIR="${OPERATOR_YOLO_DATASET_DIR:-/app/storage/operator-data/yolo-building-dataset}"
YOLO_BASE_MODEL_PATH="${YOLO_BASE_MODEL_PATH:-/app/models/yolov8n.pt}"
TRAIN_OUTPUT_DIR="${TRAIN_OUTPUT_DIR:-/app/storage/training/operator-yolo}"
TRAIN_RUN_NAME="${TRAIN_RUN_NAME:-geointel-building-detector}"
TRAIN_MODEL_OUTPUT_PATH="${TRAIN_MODEL_OUTPUT_PATH:-/app/models/geointel-building-detector.pt}"
TRAIN_EPOCHS="${TRAIN_EPOCHS:-8}"
TRAIN_IMGSZ="${TRAIN_IMGSZ:-512}"
TRAIN_BATCH="${TRAIN_BATCH:-2}"
TRAIN_WORKERS="${TRAIN_WORKERS:-0}"
TRAIN_DEVICE="${TRAIN_DEVICE:-cpu}"
TRAIN_REQUIRE_CUDA="${TRAIN_REQUIRE_CUDA:-false}"
if [[ -z "${PYTHON_BIN:-}" ]]; then
  if [[ -x "/opt/geointel/venv/bin/python" ]]; then
    PYTHON_BIN="/opt/geointel/venv/bin/python"
  else
    PYTHON_BIN="python3"
  fi
fi

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
NO_TRAINING_MARKER="${OPERATOR_YOLO_DATASET_DIR%/}/NO_TRAINING.json"
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
export DATASET_YAML
export YOLO_BASE_MODEL_PATH
export TRAIN_OUTPUT_DIR
export TRAIN_RUN_NAME
export TRAIN_MODEL_OUTPUT_PATH
export TRAIN_EPOCHS
export TRAIN_IMGSZ
export TRAIN_BATCH
export TRAIN_WORKERS
export TRAIN_DEVICE
export TRAIN_REQUIRE_CUDA
export PYTHON_BIN
export SUMMARY_PATH

if [[ ! -f "${DATASET_YAML}" ]]; then
  echo "Dataset YAML not found: ${DATASET_YAML}" >&2
  exit 1
fi

if [[ -f "${NO_TRAINING_MARKER}" ]]; then
  echo "Training is prohibited for this evaluation-only dataset: ${NO_TRAINING_MARKER}" >&2
  exit 1
fi

if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
  echo "Base model file not found: ${YOLO_BASE_MODEL_PATH}" >&2
  exit 1
fi

# A filename is not a training identity.  Verify the immutable YAML/corpus/
# asset/review release before importing Ultralytics or allocating CUDA work.
"${PYTHON_BIN}" "${SCRIPT_DIR}/training_release_manifest.py" verify \
  --train-yaml "${DATASET_YAML}"

mkdir -p "${TRAIN_OUTPUT_DIR}" "$(dirname "${TRAIN_MODEL_OUTPUT_PATH}")"

"${PYTHON_BIN}" - <<'PY'
from __future__ import annotations

import hashlib
import json
import os
import shutil
from pathlib import Path


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def seed_ultralytics_font() -> None:
    font_candidates = [
        Path("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"),
        Path("/usr/share/fonts/truetype/liberation2/LiberationSans-Regular.ttf"),
        Path("/usr/share/fonts/truetype/freefont/FreeSans.ttf"),
    ]
    source_font = next((font for font in font_candidates if font.exists()), None)
    if source_font is None:
        return
    config_root = Path(os.environ.get("YOLO_CONFIG_DIR", str(Path.home() / ".config")))
    target_font = config_root / "Ultralytics" / "Arial.ttf"
    target_font.parent.mkdir(parents=True, exist_ok=True)
    if not target_font.exists():
        shutil.copy2(source_font, target_font)


seed_ultralytics_font()

from ultralytics import YOLO
import torch

dataset_yaml = Path(os.environ["DATASET_YAML"])
base_model_path = Path(os.environ["YOLO_BASE_MODEL_PATH"])
train_output_dir = Path(os.environ["TRAIN_OUTPUT_DIR"])
run_name = os.environ["TRAIN_RUN_NAME"]
trained_model_output_path = Path(os.environ["TRAIN_MODEL_OUTPUT_PATH"])
summary_path = Path(os.environ["SUMMARY_PATH"])
epochs = int(os.environ["TRAIN_EPOCHS"])
image_size = int(os.environ["TRAIN_IMGSZ"])
batch_size = int(os.environ["TRAIN_BATCH"])
workers = int(os.environ["TRAIN_WORKERS"])
device = os.environ["TRAIN_DEVICE"]
require_cuda = os.environ["TRAIN_REQUIRE_CUDA"].strip().lower() in {"1", "true", "yes", "on"}
if require_cuda and (not device.lower().startswith("cuda") or not torch.cuda.is_available()):
    raise SystemExit("TRAIN_REQUIRE_CUDA is enabled but the requested CUDA device is unavailable")

model = YOLO(str(base_model_path))
model.train(
    data=str(dataset_yaml),
    epochs=epochs,
    imgsz=image_size,
    batch=batch_size,
    workers=workers,
    device=device,
    project=str(train_output_dir),
    name=run_name,
    exist_ok=True,
    pretrained=True,
    plots=False,
    verbose=True,
)

best_path = train_output_dir / run_name / "weights" / "best.pt"
if not best_path.exists():
    raise SystemExit(f"Expected trained model artifact was not created: {best_path}")

shutil.copy2(best_path, trained_model_output_path)
dataset_summary_path = next(
    (
        candidate
        for candidate in (
            dataset_yaml.parent / "yolo_tile_dataset_summary.json",
            dataset_yaml.parent / "yolo_dataset_summary.json",
        )
        if candidate.is_file()
    ),
    None,
)
summary = {
    "status": "ok",
    "dataset_yaml": str(dataset_yaml),
    "base_model_path": str(base_model_path),
    "train_output_dir": str(train_output_dir),
    "train_run_name": run_name,
    "trained_model_path": str(trained_model_output_path),
    "best_artifact_path": str(best_path),
    "dataset_yaml_sha256": sha256_file(dataset_yaml),
    "dataset_summary_path": str(dataset_summary_path) if dataset_summary_path else None,
    "dataset_summary_sha256": sha256_file(dataset_summary_path) if dataset_summary_path else None,
    "base_model_sha256": sha256_file(base_model_path),
    "trained_model_sha256": sha256_file(trained_model_output_path),
    "epochs": epochs,
    "image_size": image_size,
    "batch_size": batch_size,
    "workers": workers,
    "device": device,
    "cuda_required": require_cuda,
    "cuda_available": torch.cuda.is_available(),
    "torch_version": torch.__version__,
}
summary_path.parent.mkdir(parents=True, exist_ok=True)
summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
print(json.dumps(summary, indent=2, sort_keys=True))
PY