Expand operator samples for YOLO hard negatives

2026-07-07 21:42:23 +02:00
parent e8d79fccbb
commit 89c5729d33
8 changed files with 288 additions and 23 deletions
@@ -175,13 +175,18 @@ To prepare the documented operator samples reproducibly inside the all-in-one
 runtime container, run:

 ```bash
-docker exec -it geointel python /app/scripts/prepare_operator_real_data_samples.py --samples geel,mol,turnhout
+docker exec -it geointel python3 /app/scripts/prepare_operator_real_data_samples.py
 ```

 This writes GeoTIFF/GeoJSON pairs and `operator_samples_manifest.json` under
 `/app/storage/operator-data` inside the container, which maps to
-`storage/operator-data` in the Tower appdata checkout. The helper fetches only
-the explicit documented AOIs, records Digitaal Vlaanderen attribution and
+`storage/operator-data` in the Tower appdata checkout. The default corpus
+contains reference AOIs for Geel, Mol, Turnhout, Herentals, Balen, Retie and
+Westerlo plus background candidates for Postel-bos, Lommel-heide and
+Kasterlee-bos. Normal reference AOIs still fail when GRB returns no buildings;
+background candidates are explicitly marked with `sample_role` and may write an
+empty reference FeatureCollection for negative-tile training. The helper fetches
+only the explicit documented AOIs, records Digitaal Vlaanderen attribution and
 reuses existing files by default. Use `--force` only when the local runtime
 artifacts should be regenerated.

@@ -308,11 +313,11 @@ with overlapping raster windows:
 ```bash
 docker exec -it geointel python3 /app/scripts/export_operator_yolo_tile_dataset.py \
  --manifest-path /app/storage/operator-data/operator_samples_manifest.json \
-  --output-dir /app/storage/operator-data/yolo-building-tile-dataset \
-  --tile-size 192 \
-  --stride 96 \
-  --negative-keep-ratio 0.5 \
-  --val-samples turnhout \
+  --output-dir /app/storage/operator-data/yolo-building-tile-expanded160 \
+  --tile-size 160 \
+  --stride 80 \
+  --negative-keep-ratio 1.0 \
+  --val-samples turnhout,retie,kasterlee_bos \
  --force
 ```

@@ -328,18 +333,38 @@ output directory:

 ```bash
 docker exec \
-  -e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-tile-dataset \
+  -e OPERATOR_YOLO_DATASET_DIR=/app/storage/operator-data/yolo-building-tile-expanded160 \
  -e YOLO_BASE_MODEL_PATH=/app/models/yolov8n.pt \
-  -e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-tile-detector.pt \
-  -e TRAIN_EPOCHS=30 \
+  -e TRAIN_OUTPUT_DIR=/app/storage/training/operator-yolo \
+  -e TRAIN_RUN_NAME=geointel-building-yolov8n-expanded160e50 \
+  -e TRAIN_MODEL_OUTPUT_PATH=/app/models/geointel-building-yolov8n-expanded160e50.pt \
+  -e TRAIN_EPOCHS=50 \
  -e TRAIN_IMGSZ=256 \
-  -e TRAIN_BATCH=4 \
+  -e TRAIN_BATCH=8 \
  -e TRAIN_WORKERS=0 \
  -e TRAIN_DEVICE=cpu \
  -e PYTHON_BIN=python3 \
  geointel bash /app/scripts/train_operator_yolo_detector.sh
 ```

+Benchmark any trained candidate through the same persisted QA/QC matrix before
+using it operationally:
+
+```bash
+OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
+OPERATOR_SAMPLE_SLUGS="geel mol turnhout retie kasterlee_bos" \
+QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-building-yolov8n-tile30-pt yolov8s-building-segmentation-pt" \
+QUALITY_TILE_SIZES="640" \
+QUALITY_TILE_OVERLAPS="64" \
+QUALITY_THRESHOLDS="0.25 0.15 0.05" \
+MULTI_SAMPLE_OUTPUT_DIR=artifacts/detection-quality-matrix/multi-sample/expanded160e50-live \
+bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.168.10.150:1202
+```
+
+The expanded 50-epoch candidate improved dense Geel/Mol/Turnhout/Retie scores,
+but the sparse Kasterlee-bos run still showed too many false positives. Treat it
+as the best current experimental dense-AOI candidate, not as a V1 default.
+
 Export calibration QA evidence for visual review:

 ```bash
@@ -36,6 +36,8 @@ class OperatorSample:
    half_size_m: float = 250.0
    width: int = 512
    height: int = 512
+    sample_role: str = "reference"
+    allow_empty_reference: bool = False


 SAMPLES: dict[str, OperatorSample] = {
@@ -57,6 +59,61 @@ SAMPLES: dict[str, OperatorSample] = {
        center_lon=4.9488,
        center_lat=51.3225,
    ),
+    "herentals": OperatorSample(
+        slug="herentals",
+        display_name="Herentals center",
+        center_lon=4.8339,
+        center_lat=51.1766,
+        half_size_m=220.0,
+    ),
+    "balen": OperatorSample(
+        slug="balen",
+        display_name="Balen center",
+        center_lon=5.1703,
+        center_lat=51.1688,
+        half_size_m=220.0,
+    ),
+    "retie": OperatorSample(
+        slug="retie",
+        display_name="Retie center",
+        center_lon=5.0827,
+        center_lat=51.2665,
+        half_size_m=220.0,
+    ),
+    "westerlo": OperatorSample(
+        slug="westerlo",
+        display_name="Westerlo center",
+        center_lon=4.9158,
+        center_lat=51.0909,
+        half_size_m=220.0,
+    ),
+    "postel_bos": OperatorSample(
+        slug="postel_bos",
+        display_name="Postel forest background candidate",
+        center_lon=5.16,
+        center_lat=51.305,
+        half_size_m=260.0,
+        sample_role="background_candidate",
+        allow_empty_reference=True,
+    ),
+    "lommel_heide": OperatorSample(
+        slug="lommel_heide",
+        display_name="Lommel forest background candidate",
+        center_lon=5.287,
+        center_lat=51.249,
+        half_size_m=260.0,
+        sample_role="background_candidate",
+        allow_empty_reference=True,
+    ),
+    "kasterlee_bos": OperatorSample(
+        slug="kasterlee_bos",
+        display_name="Kasterlee forest background candidate",
+        center_lon=4.965,
+        center_lat=51.273,
+        half_size_m=260.0,
+        sample_role="background_candidate",
+        allow_empty_reference=True,
+    ),
 }


@@ -218,7 +275,7 @@ def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list
    response.raise_for_status()
    reference = response.json()
    features = reference.get("features") or []
-    if not features:
+    if not features and not sample.allow_empty_reference:
        raise SystemExit(f"GRB GBG returned no building features for {sample.slug} bbox {geo_bbox}")

    reference["name"] = f"GRB GBG buildings - {sample.display_name} sample AOI"
@@ -227,11 +284,14 @@ def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list
    reference["attribution"] = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen"
    reference["bbox"] = geo_bbox
    reference["sample_slug"] = sample.slug
+    reference["sample_role"] = sample.sample_role
+    reference["allow_empty_reference"] = sample.allow_empty_reference
    for feature in features:
        props = feature.setdefault("properties", {})
        props.setdefault("source_name", "grb")
        props.setdefault("reference_layer_name", "buildings")
        props.setdefault("sample_slug", sample.slug)
+        props.setdefault("sample_role", sample.sample_role)

    reference_path.write_text(json.dumps(reference, ensure_ascii=False), encoding="utf-8")
    return prepared_url(GRB_GBG_URL, ogc_params), len(features)
@@ -256,6 +316,8 @@ def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dic
        "center_lon": sample.center_lon,
        "center_lat": sample.center_lat,
        "half_size_m": sample.half_size_m,
+        "sample_role": sample.sample_role,
+        "allow_empty_reference": sample.allow_empty_reference,
        "raster_path": str(ortho_path),
        "reference_path": str(reference_path),
        "reference_feature_count": reference_feature_count,
@@ -288,7 +350,7 @@ def write_readme(output_dir: Path, samples: list[dict[str, Any]]) -> None:
        lines.append(
            f"- `{sample['sample_slug']}`: `{Path(sample['raster_path']).name}` and "
            f"`{Path(sample['reference_path']).name}`, "
-            f"{sample['reference_feature_count']} reference features."
+            f"{sample['reference_feature_count']} reference features, role `{sample['sample_role']}`."
        )
    lines.append("")
    lines.append("Purpose: configured-YOLO detection + persisted QA/QC validation with operator-provided files.")