Classify operator background corpus

scripts/README.md

@@ -187,8 +187,11 @@ Kasterlee-bos, Dessel-heide, Ravels-bos, Meerhout-bos, Geel-Bel,
 Arendonk-heide and Herenthout-bos. Normal reference AOIs still fail when GRB
 returns no buildings; background candidates are explicitly marked with
 `sample_role` and may write an empty reference FeatureCollection for
-negative-tile training. The helper fetches only the explicit documented AOIs,
-records Digitaal Vlaanderen attribution and reuses existing files by default.
+negative-tile training. Generated manifests also classify background samples as
+`pure_empty_negative` when GRB returns zero reference buildings or
+`sparse_building_context` when GRB returns one or more contextual buildings.
+The helper fetches only the explicit documented AOIs, records Digitaal
+Vlaanderen attribution and reuses existing files by default.
 Use `--force` only when the local runtime artifacts should be regenerated.
 GRB building references are fetched through the provider's OGC API
 `rel=next` pagination links, so dense AOIs are not silently limited to the
@@ -472,6 +475,7 @@ before changing model defaults:
 
 ```bash
 OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
+OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative" \
 OPERATOR_BACKGROUND_SAMPLE_SLUGS="postel_bos lommel_heide kasterlee_bos dessel_heide ravels_bos meerhout_bos geel_bel arendonk_heide herenthout_bos" \
 QUALITY_MODEL_ASSET_IDS="geointel-building-yolov8n-expanded160e50-pt geointel-building-yolov8n-tile30-pt yolov8s-building-segmentation-pt" \
 QUALITY_TILE_SIZES="640" \
@@ -485,7 +489,12 @@ The hard-negative matrix uploads only the background raster, generates tiles,
 runs configured-YOLO detection and counts persisted detections as
 `false_positive_pressure`. It does not upload a reference vector and does not
 run QA/QC, because empty or sparse background AOIs do not have a meaningful
-precision/recall target. In the first live run, `geointel-building-yolov8n-expanded160e50-pt`
+precision/recall target. Use `OPERATOR_BACKGROUND_CATEGORIES="pure_empty_negative"`
+for the strict default-promotion false-positive gate. Run
+`OPERATOR_BACKGROUND_CATEGORIES="sparse_building_context"` separately for
+contextual review; sparse-context detections should be inspected, not counted
+as fake precision/recall metrics. In the first live run,
+`geointel-building-yolov8n-expanded160e50-pt`
 was clean on Postel-bos and Lommel-heide at thresholds `0.25` and `0.15`, but
 produced 38 detections on Kasterlee-bos even at `0.25`. That blocks it from
 becoming a V1 default until a hard-negative-balanced candidate improves.

scripts/export_operator_yolo_tile_dataset.py

@@ -315,6 +315,7 @@ def export_sample_tiles(
 ) -> list[dict[str, Any]]:
     sample_slug = str(sample["sample_slug"])
     sample_role = str(sample.get("sample_role") or "reference")
+    background_category = str(sample.get("background_category") or "reference_aoi")
     split = "val" if sample_slug.lower() in val_slugs else "train"
     raster_path = resolve_manifest_path(str(sample["raster_path"]), manifest_path)
     reference_path = resolve_manifest_path(str(sample["reference_path"]), manifest_path)
@@ -366,6 +367,7 @@ def export_sample_tiles(
                     {
                         "sample_slug": sample_slug,
                         "sample_role": sample_role,
+                        "background_category": background_category,
                         "split": split,
                         "tile_index": tile_index,
                         "repeat_index": repeat_index,

scripts/prepare_operator_real_data_samples.py

@@ -22,6 +22,9 @@ GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG
 DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data")
 DEFAULT_GRB_PAGE_LIMIT = 1000
 DEFAULT_GRB_MAX_FEATURES = 100000
+REFERENCE_AOI_CATEGORY = "reference_aoi"
+PURE_EMPTY_BACKGROUND_CATEGORY = "pure_empty_negative"
+SPARSE_BACKGROUND_CATEGORY = "sparse_building_context"
 requests: Any = None
 rasterio: Any = None
 Transformer: Any = None
@@ -261,6 +264,12 @@ def selected_samples(raw: str) -> list[OperatorSample]:
     return [SAMPLES[slug] for slug in slugs]
 
 
+def background_category_for_sample(sample: OperatorSample, reference_feature_count: int) -> str:
+    if sample.sample_role != "background_candidate":
+        return REFERENCE_AOI_CATEGORY
+    return PURE_EMPTY_BACKGROUND_CATEGORY if reference_feature_count <= 0 else SPARSE_BACKGROUND_CATEGORY
+
+
 def apply_sample_overrides(
     sample: OperatorSample,
     *,
@@ -471,6 +480,7 @@ def fetch_reference(
     reference["sample_slug"] = sample.slug
     reference["sample_role"] = sample.sample_role
     reference["allow_empty_reference"] = sample.allow_empty_reference
+    reference["background_category"] = background_category_for_sample(sample, len(features))
     reference["reference_page_limit"] = page_limit
     reference["reference_max_features"] = max_features
     reference["reference_pages_fetched"] = len(pages)
@@ -484,6 +494,7 @@ def fetch_reference(
         props.setdefault("reference_layer_name", "buildings")
         props.setdefault("sample_slug", sample.slug)
         props.setdefault("sample_role", sample.sample_role)
+        props.setdefault("background_category", background_category_for_sample(sample, len(features)))
 
     reference_path.write_text(json.dumps(reference, ensure_ascii=False), encoding="utf-8")
     return prepared_url(GRB_GBG_URL, ogc_params), len(features)
@@ -513,6 +524,7 @@ def prepare_sample(
         )
     else:
         reference_feature_count = geojson_feature_count(reference_path)
+    background_category = background_category_for_sample(sample, reference_feature_count)
 
     return {
         "sample_slug": sample.slug,
@@ -524,6 +536,7 @@ def prepare_sample(
         "height": sample.height,
         "sample_role": sample.sample_role,
         "allow_empty_reference": sample.allow_empty_reference,
+        "background_category": background_category,
         "raster_path": str(ortho_path),
         "reference_path": str(reference_path),
         "reference_feature_count": reference_feature_count,
@@ -558,7 +571,8 @@ def write_readme(output_dir: Path, samples: list[dict[str, Any]]) -> None:
         lines.append(
             f"- `{sample['sample_slug']}`: `{Path(sample['raster_path']).name}` and "
             f"`{Path(sample['reference_path']).name}`, "
-            f"{sample['reference_feature_count']} reference features, role `{sample['sample_role']}`."
+            f"{sample['reference_feature_count']} reference features, role `{sample['sample_role']}`, "
+            f"background category `{sample['background_category']}`."
         )
     lines.append("")
     lines.append("Purpose: configured-YOLO detection + persisted QA/QC validation with operator-provided files.")

scripts/run_operator_hard_negative_detection_matrix.sh

@@ -13,6 +13,7 @@ Usage:
 Optional environment:
   OPERATOR_SAMPLE_MANIFEST_PATH      Manifest from prepare_operator_real_data_samples.py.
   OPERATOR_BACKGROUND_SAMPLE_SLUGS   Optional comma/space separated filter. Defaults to samples marked background_candidate or allow_empty_reference.
+  OPERATOR_BACKGROUND_CATEGORIES     Optional comma/space separated filter, e.g. pure_empty_negative or sparse_building_context.
   HARD_NEGATIVE_OUTPUT_DIR           Output directory, default: artifacts/detection-hard-negatives/<timestamp>.
   QUALITY_MODEL_ASSET_IDS            Space/comma separated local model asset IDs. Default: active configured model asset.
   QUALITY_TILE_SIZES                 Space/comma separated raster tile sizes, default: 640.
@@ -33,6 +34,7 @@ cd "$ROOT"
 BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
 OPERATOR_SAMPLE_MANIFEST_PATH="${OPERATOR_SAMPLE_MANIFEST_PATH:-storage/operator-data/operator_samples_manifest.json}"
 OPERATOR_BACKGROUND_SAMPLE_SLUGS="${OPERATOR_BACKGROUND_SAMPLE_SLUGS:-}"
+OPERATOR_BACKGROUND_CATEGORIES="${OPERATOR_BACKGROUND_CATEGORIES:-}"
 HARD_NEGATIVE_OUTPUT_DIR="${HARD_NEGATIVE_OUTPUT_DIR:-artifacts/detection-hard-negatives/$(date -u +%Y%m%dT%H%M%SZ)}"
 QUALITY_MODEL_ASSET_IDS="${QUALITY_MODEL_ASSET_IDS:-${REAL_MODEL_ASSET_ID:-__active__}}"
 QUALITY_TILE_SIZES="${QUALITY_TILE_SIZES:-640}"
@@ -79,6 +81,7 @@ matrix_manifest="${HARD_NEGATIVE_OUTPUT_DIR}/hard_negative_requests.tsv"
   "${ROOT}" \
   "${OPERATOR_SAMPLE_MANIFEST_PATH}" \
   "${OPERATOR_BACKGROUND_SAMPLE_SLUGS}" \
+  "${OPERATOR_BACKGROUND_CATEGORIES}" \
   "${sample_manifest_tsv}" <<'PY'
 import json
 import sys
@@ -87,7 +90,8 @@ from pathlib import Path
 root = Path(sys.argv[1]).resolve()
 manifest_path = Path(sys.argv[2])
 slug_filter_raw = sys.argv[3]
-output_path = Path(sys.argv[4])
+category_filter_raw = sys.argv[4]
+output_path = Path(sys.argv[5])
 
 payload = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
 samples = payload.get("samples") or []
@@ -99,6 +103,11 @@ requested_slugs = {
     for value in slug_filter_raw.replace(",", " ").split()
     if value.strip()
 }
+requested_categories = {
+    value.strip().lower()
+    for value in category_filter_raw.replace(",", " ").split()
+    if value.strip()
+}
 
 
 def resolve_path(raw: str) -> str:
@@ -129,11 +138,22 @@ with output_path.open("w", encoding="utf-8") as handle:
             continue
         raster_path = resolve_path(str(sample.get("raster_path") or ""))
         reference_count = int(sample.get("reference_feature_count") or 0)
-        handle.write(f"{sample_slug}\t{raster_path}\t{sample_role}\t{allow_empty_reference}\t{reference_count}\n")
+        background_category = str(sample.get("background_category") or "").lower()
+        if not background_category:
+            if sample_role == "background_candidate" or allow_empty_reference:
+                background_category = "pure_empty_negative" if reference_count == 0 else "sparse_building_context"
+            else:
+                background_category = "reference_aoi"
+        if requested_categories and background_category not in requested_categories:
+            continue
+        handle.write(
+            f"{sample_slug}\t{raster_path}\t{sample_role}\t{allow_empty_reference}\t"
+            f"{reference_count}\t{background_category}\n"
+        )
         selected += 1
 
 if selected == 0:
-    raise SystemExit("No background_candidate samples matched OPERATOR_BACKGROUND_SAMPLE_SLUGS")
+    raise SystemExit("No background_candidate samples matched OPERATOR_BACKGROUND_SAMPLE_SLUGS/OPERATOR_BACKGROUND_CATEGORIES")
 PY
 
 model_requests_normalized="$(printf '%s' "${QUALITY_MODEL_ASSET_IDS}" | tr ',' ' ')"
@@ -236,14 +256,15 @@ echo "== GeoIntel operator hard-negative detection matrix =="
 echo "Base URL: ${BASE_URL}"
 echo "Manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}"
 echo "Background filter: ${OPERATOR_BACKGROUND_SAMPLE_SLUGS:-background_candidate samples}"
+echo "Background categories: ${OPERATOR_BACKGROUND_CATEGORIES:-all}"
 echo "Models: ${model_requests_normalized}"
 echo "Tile sizes: ${tile_sizes_normalized}"
 echo "Tile overlaps: ${tile_overlaps_normalized}"
 echo "Thresholds: ${thresholds_normalized}"
 echo "Output: ${HARD_NEGATIVE_OUTPUT_DIR}"
 
-while IFS=$'\t' read -r sample_slug raster_path sample_role allow_empty_reference reference_feature_count; do
-  echo "-- Background sample ${sample_slug}: role=${sample_role} allow_empty_reference=${allow_empty_reference} reference_features=${reference_feature_count} --"
+while IFS=$'\t' read -r sample_slug raster_path sample_role allow_empty_reference reference_feature_count background_category; do
+  echo "-- Background sample ${sample_slug}: role=${sample_role} category=${background_category} allow_empty_reference=${allow_empty_reference} reference_features=${reference_feature_count} --"
   while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label; do
     sample_output_dir="${HARD_NEGATIVE_OUTPUT_DIR}/${sample_slug}"
     mkdir -p "${sample_output_dir}"
@@ -379,6 +400,7 @@ PY
         "${sample_role}" \
         "${allow_empty_reference}" \
         "${reference_feature_count}" \
+        "${background_category}" \
         "${model_request}" \
         "${model_asset_id}" \
         "${tile_size}" \
@@ -401,6 +423,7 @@ import sys
     sample_role,
     allow_empty_reference,
     reference_feature_count,
+    background_category,
     model_request,
     model_asset_id,
     tile_size,
@@ -414,7 +437,7 @@ import sys
     detections_list_count,
     tile_count,
     run_log,
-) = sys.argv[1:19]
+) = sys.argv[1:20]
 
 detections = int(detection_count)
 listed = int(detections_list_count)
@@ -426,6 +449,7 @@ summary = {
     "sample_role": sample_role,
     "allow_empty_reference": allow_empty_reference == "True",
     "reference_feature_count": int(reference_feature_count),
+    "background_category": background_category,
     "model_request": model_request,
     "model_asset_id": model_asset_id,
     "tile_size": int(tile_size),
@@ -443,7 +467,7 @@ summary = {
 with open(output_path, "w", encoding="utf-8") as handle:
     json.dump(summary, handle, indent=2, sort_keys=True)
 print(
-    "sample={sample_slug} model={model_asset_id} tile={tile_size} overlap={tile_overlap} "
+    "sample={sample_slug} category={background_category} model={model_asset_id} tile={tile_size} overlap={tile_overlap} "
     "threshold={threshold} detections={detection_count} false_positive_pressure={false_positive_pressure}".format(
         **summary
     )
@@ -460,6 +484,7 @@ done < "${sample_manifest_tsv}"
 import glob
 import json
 import sys
+from collections import Counter
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -486,6 +511,7 @@ summary = {
     "base_url": base_url,
     "operator_sample_manifest_path": manifest_path,
     "sample_count": len({item["sample_slug"] for item in items}),
+    "background_category_counts": dict(Counter(str(item.get("background_category") or "unknown") for item in items)),
     "run_count": len(items),
     "best_by_lowest_pressure": best_by_lowest_pressure,
     "items": items,
@@ -495,10 +521,10 @@ summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding=
 
 print("")
 print("Operator hard-negative detection summary")
-print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\tfalse_positive_pressure")
+print("sample\tcategory\tmodel\ttile\toverlap\tthreshold\tdetections\tfalse_positive_pressure")
 for item in items:
     print(
-        "{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{false_positive_pressure}".format(
+        "{sample_slug}\t{background_category}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{false_positive_pressure}".format(
             **item
         )
     )