Classify operator background corpus

scripts/run_operator_hard_negative_detection_matrix.sh

@@ -13,6 +13,7 @@ Usage:
 Optional environment:
   OPERATOR_SAMPLE_MANIFEST_PATH      Manifest from prepare_operator_real_data_samples.py.
   OPERATOR_BACKGROUND_SAMPLE_SLUGS   Optional comma/space separated filter. Defaults to samples marked background_candidate or allow_empty_reference.
+  OPERATOR_BACKGROUND_CATEGORIES     Optional comma/space separated filter, e.g. pure_empty_negative or sparse_building_context.
   HARD_NEGATIVE_OUTPUT_DIR           Output directory, default: artifacts/detection-hard-negatives/<timestamp>.
   QUALITY_MODEL_ASSET_IDS            Space/comma separated local model asset IDs. Default: active configured model asset.
   QUALITY_TILE_SIZES                 Space/comma separated raster tile sizes, default: 640.
@@ -33,6 +34,7 @@ cd "$ROOT"
 BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
 OPERATOR_SAMPLE_MANIFEST_PATH="${OPERATOR_SAMPLE_MANIFEST_PATH:-storage/operator-data/operator_samples_manifest.json}"
 OPERATOR_BACKGROUND_SAMPLE_SLUGS="${OPERATOR_BACKGROUND_SAMPLE_SLUGS:-}"
+OPERATOR_BACKGROUND_CATEGORIES="${OPERATOR_BACKGROUND_CATEGORIES:-}"
 HARD_NEGATIVE_OUTPUT_DIR="${HARD_NEGATIVE_OUTPUT_DIR:-artifacts/detection-hard-negatives/$(date -u +%Y%m%dT%H%M%SZ)}"
 QUALITY_MODEL_ASSET_IDS="${QUALITY_MODEL_ASSET_IDS:-${REAL_MODEL_ASSET_ID:-__active__}}"
 QUALITY_TILE_SIZES="${QUALITY_TILE_SIZES:-640}"
@@ -79,6 +81,7 @@ matrix_manifest="${HARD_NEGATIVE_OUTPUT_DIR}/hard_negative_requests.tsv"
   "${ROOT}" \
   "${OPERATOR_SAMPLE_MANIFEST_PATH}" \
   "${OPERATOR_BACKGROUND_SAMPLE_SLUGS}" \
+  "${OPERATOR_BACKGROUND_CATEGORIES}" \
   "${sample_manifest_tsv}" <<'PY'
 import json
 import sys
@@ -87,7 +90,8 @@ from pathlib import Path
 root = Path(sys.argv[1]).resolve()
 manifest_path = Path(sys.argv[2])
 slug_filter_raw = sys.argv[3]
-output_path = Path(sys.argv[4])
+category_filter_raw = sys.argv[4]
+output_path = Path(sys.argv[5])
 
 payload = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
 samples = payload.get("samples") or []
@@ -99,6 +103,11 @@ requested_slugs = {
     for value in slug_filter_raw.replace(",", " ").split()
     if value.strip()
 }
+requested_categories = {
+    value.strip().lower()
+    for value in category_filter_raw.replace(",", " ").split()
+    if value.strip()
+}
 
 
 def resolve_path(raw: str) -> str:
@@ -129,11 +138,22 @@ with output_path.open("w", encoding="utf-8") as handle:
             continue
         raster_path = resolve_path(str(sample.get("raster_path") or ""))
         reference_count = int(sample.get("reference_feature_count") or 0)
-        handle.write(f"{sample_slug}\t{raster_path}\t{sample_role}\t{allow_empty_reference}\t{reference_count}\n")
+        background_category = str(sample.get("background_category") or "").lower()
+        if not background_category:
+            if sample_role == "background_candidate" or allow_empty_reference:
+                background_category = "pure_empty_negative" if reference_count == 0 else "sparse_building_context"
+            else:
+                background_category = "reference_aoi"
+        if requested_categories and background_category not in requested_categories:
+            continue
+        handle.write(
+            f"{sample_slug}\t{raster_path}\t{sample_role}\t{allow_empty_reference}\t"
+            f"{reference_count}\t{background_category}\n"
+        )
         selected += 1
 
 if selected == 0:
-    raise SystemExit("No background_candidate samples matched OPERATOR_BACKGROUND_SAMPLE_SLUGS")
+    raise SystemExit("No background_candidate samples matched OPERATOR_BACKGROUND_SAMPLE_SLUGS/OPERATOR_BACKGROUND_CATEGORIES")
 PY
 
 model_requests_normalized="$(printf '%s' "${QUALITY_MODEL_ASSET_IDS}" | tr ',' ' ')"
@@ -236,14 +256,15 @@ echo "== GeoIntel operator hard-negative detection matrix =="
 echo "Base URL: ${BASE_URL}"
 echo "Manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}"
 echo "Background filter: ${OPERATOR_BACKGROUND_SAMPLE_SLUGS:-background_candidate samples}"
+echo "Background categories: ${OPERATOR_BACKGROUND_CATEGORIES:-all}"
 echo "Models: ${model_requests_normalized}"
 echo "Tile sizes: ${tile_sizes_normalized}"
 echo "Tile overlaps: ${tile_overlaps_normalized}"
 echo "Thresholds: ${thresholds_normalized}"
 echo "Output: ${HARD_NEGATIVE_OUTPUT_DIR}"
 
-while IFS=$'\t' read -r sample_slug raster_path sample_role allow_empty_reference reference_feature_count; do
-  echo "-- Background sample ${sample_slug}: role=${sample_role} allow_empty_reference=${allow_empty_reference} reference_features=${reference_feature_count} --"
+while IFS=$'\t' read -r sample_slug raster_path sample_role allow_empty_reference reference_feature_count background_category; do
+  echo "-- Background sample ${sample_slug}: role=${sample_role} category=${background_category} allow_empty_reference=${allow_empty_reference} reference_features=${reference_feature_count} --"
   while IFS=$'\t' read -r model_request tile_size tile_overlap threshold run_label; do
     sample_output_dir="${HARD_NEGATIVE_OUTPUT_DIR}/${sample_slug}"
     mkdir -p "${sample_output_dir}"
@@ -379,6 +400,7 @@ PY
         "${sample_role}" \
         "${allow_empty_reference}" \
         "${reference_feature_count}" \
+        "${background_category}" \
         "${model_request}" \
         "${model_asset_id}" \
         "${tile_size}" \
@@ -401,6 +423,7 @@ import sys
     sample_role,
     allow_empty_reference,
     reference_feature_count,
+    background_category,
     model_request,
     model_asset_id,
     tile_size,
@@ -414,7 +437,7 @@ import sys
     detections_list_count,
     tile_count,
     run_log,
-) = sys.argv[1:19]
+) = sys.argv[1:20]
 
 detections = int(detection_count)
 listed = int(detections_list_count)
@@ -426,6 +449,7 @@ summary = {
     "sample_role": sample_role,
     "allow_empty_reference": allow_empty_reference == "True",
     "reference_feature_count": int(reference_feature_count),
+    "background_category": background_category,
     "model_request": model_request,
     "model_asset_id": model_asset_id,
     "tile_size": int(tile_size),
@@ -443,7 +467,7 @@ summary = {
 with open(output_path, "w", encoding="utf-8") as handle:
     json.dump(summary, handle, indent=2, sort_keys=True)
 print(
-    "sample={sample_slug} model={model_asset_id} tile={tile_size} overlap={tile_overlap} "
+    "sample={sample_slug} category={background_category} model={model_asset_id} tile={tile_size} overlap={tile_overlap} "
     "threshold={threshold} detections={detection_count} false_positive_pressure={false_positive_pressure}".format(
         **summary
     )
@@ -460,6 +484,7 @@ done < "${sample_manifest_tsv}"
 import glob
 import json
 import sys
+from collections import Counter
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -486,6 +511,7 @@ summary = {
     "base_url": base_url,
     "operator_sample_manifest_path": manifest_path,
     "sample_count": len({item["sample_slug"] for item in items}),
+    "background_category_counts": dict(Counter(str(item.get("background_category") or "unknown") for item in items)),
     "run_count": len(items),
     "best_by_lowest_pressure": best_by_lowest_pressure,
     "items": items,
@@ -495,10 +521,10 @@ summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding=
 
 print("")
 print("Operator hard-negative detection summary")
-print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\tfalse_positive_pressure")
+print("sample\tcategory\tmodel\ttile\toverlap\tthreshold\tdetections\tfalse_positive_pressure")
 for item in items:
     print(
-        "{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{false_positive_pressure}".format(
+        "{sample_slug}\t{background_category}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{false_positive_pressure}".format(
             **item
         )
     )