Promote focused small-building detector

scripts/README.md

@@ -531,8 +531,27 @@ Current Tower audit status:
   `0.000694274766`, small-box share `0.3832694151486098`, no invalid labels and
   no missing label files. The balanced visual pass rendered 40 tiles across all
   19 source samples that retained at least one tile, with no invalid labels,
-  missing images or low-variance selections. A new candidate may be trained,
-  but remains inactive until positive and split-background promotion gates pass.
+  missing images or low-variance selections. Its promoted model remains the
+  higher-precision legacy `0.15` operator profile.
+- `yolo-building-aoi1024-smallbld-minpx3vis035`: focused small-building corpus
+  exported from an explicit 23-sample subset. Beerse, Rijkevorsel, Hoogstraten
+  and Vorselaar extend training; Vosselaar and Grobbendonk are validation-only;
+  Turnhout, Retie and Westerlo remain external operation-level holdouts. The
+  Tower export retained 198 tiles and 58,820 labels. Its small-object-aware
+  audit passed with no invalid/missing labels, and the 48-tile balanced visual
+  review contained no missing, invalid or low-variance selections. The trained
+  `geointel-building-yolov8s-smallbld-minpx3-img640-ft30-pt` candidate passed
+  seven positive-AOI and three pure-empty background gates at tile `512`,
+  overlap `64`, threshold `0.15` and QA match IoU `0.25`. Mean F1 is `0.5825`
+  and all pure-empty samples remain at zero detections. Persisted comparison
+  found 1,571 fewer false negatives than the previous balanced model, with a
+  lower mean precision and therefore a higher operator review load.
+
+Use `--samples` or `OPERATOR_YOLO_SAMPLES` to make an experimental corpus
+membership explicit. Dataset summaries preserve the complete manifest count,
+selected sample slugs and excluded sample slugs. Split validation still applies
+after filtering, so a manifest-backed holdout cannot be selected as training by
+omitting it from `--val-samples`.
 
 After rebuilding the all-in-one image, the operator scripts are available inside
 the container at `/app/scripts/...`. Before rebuilding, use the host checkout or
@@ -765,7 +784,10 @@ python scripts/audit_detection_false_negative_evidence.py \
 ```
 
 The audit reports false-negative rates and area buckets per AOI/model, plus
-reference buildings missed by every compared portfolio. Stable
+reference buildings missed by every compared portfolio. It writes the combined
+`persistent_false_negatives.geojson`, records geodetic persistent-miss area and
+adds persistent area buckets so operators can inspect the shared misses on a
+map instead of relying only on counts. Stable
 `source_feature_id` values are preferred; a normalized geometry fingerprint is
 used only when source IDs are absent. Invalid or missing geometry fails the
 audit instead of being silently skipped. The tools do not run inference,

scripts/audit_detection_false_negative_evidence.py

@@ -119,6 +119,7 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) ->
     reference_ids: set[str] = set()
     false_negative_areas: list[float] = []
     matched_reference_areas: list[float] = []
+    reference_records: dict[str, dict[str, Any]] = {}
     bucket_counts = {
         label: {"false_negative": 0, "matched_reference": 0, "total_reference": 0, "false_negative_rate": None}
         for label, _, _ in AREA_BUCKETS
@@ -144,6 +145,11 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) ->
         bucket = area_bucket(area_m2)
         reference_id = stable_reference_id(feature, geometry)
         reference_ids.add(reference_id)
+        reference_records[reference_id] = {
+            "area_m2": area_m2,
+            "area_bucket": bucket,
+            "feature": feature,
+        }
         bucket_role = "false_negative" if role == "false_negative" else "matched_reference"
         bucket_counts[bucket][bucket_role] += 1
         bucket_counts[bucket]["total_reference"] += 1
@@ -161,6 +167,7 @@ def audit_feature_collection(payload: dict[str, Any], geod: Any, shape: Any) ->
     return {
         "false_negative_ids": false_negative_ids,
         "reference_ids": reference_ids,
+        "reference_records": reference_records,
         "false_negative_count": len(false_negative_areas),
         "matched_reference_count": len(matched_reference_areas),
         "total_reference_count": total_reference,
@@ -285,6 +292,7 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]:
             )
 
     sample_reports: list[dict[str, Any]] = []
+    persistent_evidence_features: list[dict[str, Any]] = []
     for sample_slug in sorted(expected_slugs):
         portfolio_rows = []
         false_negative_sets = []
@@ -297,7 +305,7 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]:
                 {
                     key: value
                     for key, value in raw.items()
-                    if key not in {"false_negative_ids", "reference_ids"}
+                    if key not in {"false_negative_ids", "reference_ids", "reference_records"}
                 }
             )
         if any(reference_ids != reference_sets[0] for reference_ids in reference_sets[1:]):
@@ -309,16 +317,61 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]:
                 f"Sample {sample_slug} has different reference populations across portfolios ({counts})"
             )
         persistent_ids = sorted(set.intersection(*false_negative_sets))
+        reference_records = portfolio_samples[parsed_portfolios[0][0]][sample_slug]["reference_records"]
+        persistent_areas = [float(reference_records[reference_id]["area_m2"]) for reference_id in persistent_ids]
+        persistent_bucket_counts = {
+            label: {"count": 0, "share": 0.0}
+            for label, _, _ in AREA_BUCKETS
+        }
+        for reference_id in persistent_ids:
+            record = reference_records[reference_id]
+            persistent_bucket_counts[record["area_bucket"]]["count"] += 1
+            source_feature = record["feature"]
+            properties = dict(source_feature.get("properties") or {})
+            properties.update(
+                {
+                    "qa_evidence_role": "persistent_false_negative",
+                    "sample_slug": sample_slug,
+                    "persistent_reference_id": reference_id,
+                    "area_m2": record["area_m2"],
+                    "area_bucket": record["area_bucket"],
+                    "compared_portfolios": labels,
+                }
+            )
+            persistent_evidence_features.append(
+                {
+                    "type": "Feature",
+                    "id": f"{sample_slug}:{reference_id}",
+                    "properties": properties,
+                    "geometry": source_feature["geometry"],
+                }
+            )
+        if persistent_ids:
+            for values in persistent_bucket_counts.values():
+                values["share"] = values["count"] / len(persistent_ids)
         sample_reports.append(
             {
                 "sample_slug": sample_slug,
                 "reference_population_count": len(reference_sets[0]),
                 "persistent_false_negative_count": len(persistent_ids),
                 "persistent_reference_ids": persistent_ids,
+                "persistent_false_negative_area_m2": area_stats(persistent_areas),
+                "persistent_area_buckets": persistent_bucket_counts,
                 "portfolios": portfolio_rows,
             }
         )
 
+    output_dir = output_dir.expanduser().resolve()
+    output_dir.mkdir(parents=True, exist_ok=True)
+    persistent_geojson_path = output_dir / "persistent_false_negatives.geojson"
+    persistent_geojson_path.write_text(
+        json.dumps(
+            {"type": "FeatureCollection", "features": persistent_evidence_features},
+            indent=2,
+            sort_keys=True,
+        ),
+        encoding="utf-8",
+    )
     report = {
         "generated_at": datetime.now(timezone.utc).isoformat(),
         "schema_version": 1,
@@ -328,10 +381,9 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]:
         "portfolios": portfolio_meta,
         "sample_count": len(sample_reports),
         "samples": sample_reports,
+        "persistent_evidence_geojson_path": str(persistent_geojson_path),
         "recommendations": build_recommendations(sample_reports),
     }
-    output_dir = output_dir.expanduser().resolve()
-    output_dir.mkdir(parents=True, exist_ok=True)
     json_path = output_dir / "detection_false_negative_audit.json"
     json_path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8")
 
@@ -345,18 +397,26 @@ def run_audit(portfolio_args: list[str], output_dir: Path) -> tuple[Path, Path]:
         "",
         "## AOI comparison",
         "",
-        "| AOI | Persistent misses | "
+        "| AOI | Persistent misses | Persistent tiny/small | Persistent median m2 | "
         + " | ".join(f"{label} FN rate" for label, _ in parsed_portfolios)
         + " |",
-        "|---|---:|" + "---:|" * len(parsed_portfolios),
+        "|---|---:|---:|---:|" + "---:|" * len(parsed_portfolios),
     ]
     for sample in sample_reports:
         rates = [
             f"{(row['false_negative_rate'] or 0.0):.3f}"
             for row in sample["portfolios"]
         ]
+        persistent_buckets = sample["persistent_area_buckets"]
+        persistent_small_count = sum(
+            persistent_buckets[key]["count"]
+            for key in ("tiny_lt_25_m2", "small_25_100_m2")
+        )
+        persistent_median = sample["persistent_false_negative_area_m2"]["median"]
+        persistent_median_text = f"{persistent_median:.1f}" if persistent_median is not None else "n/a"
         lines.append(
             f"| {sample['sample_slug']} | {sample['persistent_false_negative_count']} | "
+            f"{persistent_small_count} | {persistent_median_text} | "
             + " | ".join(rates)
             + " |"
         )

scripts/export_operator_yolo_tile_dataset.py

@@ -26,7 +26,14 @@ PURE_EMPTY_BACKGROUND_CATEGORY = "pure_empty_negative"
 SPARSE_BACKGROUND_CATEGORY = "sparse_building_context"
 LOW_VARIANCE_NEGATIVE_SKIP_REASON = "low_visual_variance_negative"
 DEFAULT_VALIDATION_SAMPLE_SLUGS = frozenset(
-    {"turnhout", "retie", "westerlo", "arendonk_heide"}
+    {
+        "turnhout",
+        "retie",
+        "westerlo",
+        "arendonk_heide",
+        "vosselaar_center",
+        "grobbendonk_center",
+    }
 )
 DEFAULT_VALIDATION_SAMPLES = ",".join(sorted(DEFAULT_VALIDATION_SAMPLE_SLUGS))
 rasterio: Any = None
@@ -69,12 +76,20 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument("--tile-size", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_SIZE", "256")))
     parser.add_argument("--stride", type=int, default=int(os.environ.get("OPERATOR_YOLO_TILE_STRIDE", "128")))
+    parser.add_argument(
+        "--samples",
+        default=os.environ.get("OPERATOR_YOLO_SAMPLES", ""),
+        help=(
+            "Optional comma/space separated manifest sample slugs to export. "
+            "An empty value keeps every manifest sample."
+        ),
+    )
     parser.add_argument(
         "--val-samples",
         default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", DEFAULT_VALIDATION_SAMPLES),
         help=(
             "Comma/space separated sample slugs assigned to validation. "
-            "Defaults to the documented Turnhout, Retie, Westerlo and Arendonk-heide holdouts."
+            "Defaults to all documented validation samples."
         ),
     )
     parser.add_argument(
@@ -152,6 +167,31 @@ def split_slugs(raw: str) -> set[str]:
     return {value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()}
 
 
+def select_manifest_samples(
+    samples: list[dict[str, Any]],
+    requested_slugs: set[str],
+) -> tuple[list[dict[str, Any]], list[str]]:
+    manifest_slugs = {
+        str(sample.get("sample_slug") or "").strip().lower()
+        for sample in samples
+        if str(sample.get("sample_slug") or "").strip()
+    }
+    if not requested_slugs:
+        return samples, []
+    unknown = requested_slugs - manifest_slugs
+    if unknown:
+        raise SystemExit(
+            "YOLO sample selection references unknown samples: " + ", ".join(sorted(unknown))
+        )
+    selected = [
+        sample
+        for sample in samples
+        if str(sample.get("sample_slug") or "").strip().lower() in requested_slugs
+    ]
+    excluded = sorted(manifest_slugs - requested_slugs)
+    return selected, excluded
+
+
 def validate_validation_split(samples: list[dict[str, Any]], val_slugs: set[str]) -> set[str]:
     sample_slugs = {
         str(sample.get("sample_slug") or "").strip().lower()
@@ -533,9 +573,13 @@ def main() -> int:
     ensure_yolo_directories(args.output_dir)
 
     manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
-    samples = manifest.get("samples") or []
-    if not samples:
+    manifest_samples = manifest.get("samples") or []
+    if not manifest_samples:
         raise SystemExit("Operator sample manifest contains no samples")
+    samples, excluded_sample_slugs = select_manifest_samples(
+        manifest_samples,
+        split_slugs(args.samples),
+    )
     val_slugs = validate_validation_split(samples, split_slugs(args.val_samples))
     exported_tiles: list[dict[str, Any]] = []
     for sample in samples:
@@ -584,7 +628,12 @@ def main() -> int:
         "min_label_visible_ratio": args.min_label_visible_ratio,
         "drop_low_variance_negatives": args.drop_low_variance_negatives,
         "blank_range_threshold": args.blank_range_threshold,
+        "source_manifest_sample_count": len(manifest_samples),
         "source_sample_count": len(samples),
+        "selected_sample_slugs": sorted(
+            str(sample.get("sample_slug") or "").strip().lower() for sample in samples
+        ),
+        "excluded_sample_slugs": excluded_sample_slugs,
         "validation_sample_slugs": sorted(val_slugs),
         **validation_coverage,
         "tile_count": len(kept_tiles),

scripts/prepare_operator_real_data_samples.py

@@ -28,8 +28,20 @@ SPARSE_BACKGROUND_CATEGORY = "sparse_building_context"
 TRAINING_EXPANSION_SAMPLE_SLUGS = frozenset(
     {"olen_center", "lille_center", "oud_turnhout_center", "kasterlee_center"}
 )
+SMALL_BUILDING_TRAINING_SAMPLE_SLUGS = frozenset(
+    {"beerse_center", "rijkevorsel_center", "hoogstraten_center", "vorselaar_center"}
+)
+SMALL_BUILDING_VALIDATION_SAMPLE_SLUGS = frozenset(
+    {"vosselaar_center", "grobbendonk_center"}
+)
 DEFAULT_VALIDATION_SAMPLE_SLUGS = frozenset(
-    {"turnhout", "retie", "westerlo", "arendonk_heide"}
+    {
+        "turnhout",
+        "retie",
+        "westerlo",
+        "arendonk_heide",
+        *SMALL_BUILDING_VALIDATION_SAMPLE_SLUGS,
+    }
 )
 requests: Any = None
 rasterio: Any = None
@@ -122,6 +134,42 @@ SAMPLES: dict[str, OperatorSample] = {
         center_lon=4.9678120,
         center_lat=51.2407915,
     ),
+    "beerse_center": OperatorSample(
+        slug="beerse_center",
+        display_name="Beerse center small-building training expansion",
+        center_lon=4.8534,
+        center_lat=51.3192,
+    ),
+    "rijkevorsel_center": OperatorSample(
+        slug="rijkevorsel_center",
+        display_name="Rijkevorsel center small-building training expansion",
+        center_lon=4.7604,
+        center_lat=51.3487,
+    ),
+    "hoogstraten_center": OperatorSample(
+        slug="hoogstraten_center",
+        display_name="Hoogstraten center small-building training expansion",
+        center_lon=4.7609,
+        center_lat=51.4002,
+    ),
+    "vorselaar_center": OperatorSample(
+        slug="vorselaar_center",
+        display_name="Vorselaar center small-building training expansion",
+        center_lon=4.7731,
+        center_lat=51.2020,
+    ),
+    "vosselaar_center": OperatorSample(
+        slug="vosselaar_center",
+        display_name="Vosselaar center small-building validation",
+        center_lon=4.8899,
+        center_lat=51.3095,
+    ),
+    "grobbendonk_center": OperatorSample(
+        slug="grobbendonk_center",
+        display_name="Grobbendonk center small-building validation",
+        center_lon=4.7358,
+        center_lat=51.1907,
+    ),
     "postel_bos": OperatorSample(
         slug="postel_bos",
         display_name="Postel forest background candidate",