Add multi-sample detection quality calibration

CHANGELOG.md

@@ -7,6 +7,13 @@
 
 # Changelog
 
+## Sprint 127 Multi-sample detection quality calibration tooling (2026-07-07)
+
+- Added `scripts/prepare_operator_real_data_samples.py` to prepare documented Geel, Mol and Turnhout orthophoto/GRB GBG building sample pairs as explicit runtime artifacts.
+- Added `scripts/run_multi_sample_detection_quality_matrix.sh` to run the existing real-data quality matrix for every prepared sample and combine the results.
+- The combined summary writes `multi_sample_quality_summary.json` with overall score/recall/precision rankings and per-sample best configurations.
+- Added readiness coverage and regression tests for the sample-preparation and multi-sample matrix contracts.
+
 ## Sprint 126 Detection quality matrix tooling (2026-07-07)
 
 - Added `scripts/run_detection_quality_matrix.sh` to compare local model assets, raster tile sizes, tile overlaps and confidence thresholds through the existing real-data detection + QA workflow.

backend/README.md

@@ -374,6 +374,17 @@ manifests generated for AI handoff include source CRS metadata so pixel-space
 model outputs can be transformed to WGS84 GeoJSON coordinates. Current V1 upload
 support is limited to GeoTIFF-style rasters and GeoJSON/JSON reference vectors.
 
+To prepare the documented Geel/Mol/Turnhout operator sample pairs inside the
+all-in-one runtime container, run:
+
+```bash
+docker exec -it geointel python /app/scripts/prepare_operator_real_data_samples.py --samples geel,mol,turnhout
+```
+
+The helper writes GeoTIFF orthophotos, GRB GBG building GeoJSON files and
+`operator_samples_manifest.json` under `/app/storage/operator-data`. These are
+runtime artifacts only and are not committed to Git.
+
 For model-quality calibration, run the confidence sweep wrapper:
 
 ```bash
@@ -409,6 +420,21 @@ and false-positive/false-negative counts. It ranks `best_by_score`,
 `best_by_recall` and `best_by_precision`. It does not download weights, create
 fake detections, fetch live providers or change backend API behavior.
 
+To aggregate the same matrix over every prepared operator sample, run:
+
+```bash
+OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
+QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \
+QUALITY_TILE_SIZES="512 640" \
+QUALITY_TILE_OVERLAPS="64" \
+QUALITY_THRESHOLDS="0.50 0.15" \
+bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.168.10.150:1202
+```
+
+The combined `multi_sample_quality_summary.json` reports per-sample and overall
+best configurations. It is an operator benchmarking command, not a backend API
+or provider import path.
+
 To inspect the evidence behind a calibration run, export the persisted QA
 evidence bundle:
 

backend/tests/test_sprint127_operator_sample_quality_matrix.py

@@ -0,0 +1,71 @@
+from pathlib import Path
+import subprocess
+import sys
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def test_prepare_operator_real_data_samples_fetches_documented_ortho_and_grb_pairs() -> None:
+    script_path = ROOT / "scripts" / "prepare_operator_real_data_samples.py"
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
+
+    assert script_path.exists()
+    script = script_path.read_text(encoding="utf-8")
+
+    assert "py_compile scripts/prepare_operator_real_data_samples.py" in readiness
+    assert "SAMPLES" in script
+    assert '"geel"' in script
+    assert '"mol"' in script
+    assert '"turnhout"' in script
+    assert "https://geo.api.vlaanderen.be/omwrgbmrvl/wms" in script
+    assert "LAYERS" in script
+    assert "Ortho" in script
+    assert "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items" in script
+    assert "source_name" in script
+    assert "reference_layer_name" in script
+    assert "operator_samples_manifest.json" in script
+    assert "skip_existing" in script
+    assert "demo/workflow" not in script
+    assert "fixture_mode" not in script
+
+
+def test_prepare_operator_real_data_samples_help_does_not_require_gis_dependencies() -> None:
+    script_path = ROOT / "scripts" / "prepare_operator_real_data_samples.py"
+
+    result = subprocess.run(
+        [sys.executable, str(script_path), "--help"],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 0
+    assert "Prepare real Digitaal Vlaanderen" in result.stdout
+    assert "--samples" in result.stdout
+
+
+def test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sample() -> None:
+    script_path = ROOT / "scripts" / "run_multi_sample_detection_quality_matrix.sh"
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
+
+    assert script_path.exists()
+    script = script_path.read_text(encoding="utf-8")
+
+    assert "bash -n scripts/run_multi_sample_detection_quality_matrix.sh" in readiness
+    assert "OPERATOR_SAMPLE_MANIFEST_PATH" in script
+    assert "operator_samples_manifest.json" in script
+    assert "run_detection_quality_matrix.sh" in script
+    assert "QUALITY_MODEL_ASSET_IDS" in script
+    assert "QUALITY_TILE_SIZES" in script
+    assert "QUALITY_TILE_OVERLAPS" in script
+    assert "QUALITY_THRESHOLDS" in script
+    assert "REAL_RASTER_PATH" in script
+    assert "REAL_REFERENCE_VECTOR_PATH" in script
+    assert "multi_sample_quality_summary.json" in script
+    assert "best_overall_by_score" in script
+    assert "best_by_sample" in script
+    assert "sample_slug" in script
+    assert "demo/workflow" not in script
+    assert "fixture_mode" not in script
+    assert "will_download_models" not in script

docs/AI_PIPELINES.md

@@ -167,6 +167,17 @@ download model weights. A zero detection count is valid as runtime evidence only
 when the selected model genuinely returns no usable detections after canonical
 class filtering; it does not prove the model is useful for the target imagery.
 
+Documented operator samples can be prepared inside the all-in-one runtime
+container:
+
+```bash
+docker exec -it geointel python /app/scripts/prepare_operator_real_data_samples.py --samples geel,mol,turnhout
+```
+
+The helper fetches explicit Digitaal Vlaanderen orthophoto/GRB GBG sample pairs
+for the documented AOIs only and writes `operator_samples_manifest.json`. The
+application itself still does not perform live provider fetching.
+
 For confidence-threshold calibration, use the sweep wrapper:
 
 ```bash
@@ -201,6 +212,23 @@ rankings `best_by_score`, `best_by_recall` and `best_by_precision` are operator
 decision aids only; GeoIntel still does not download models, seed fixture
 detections or treat AI detections as ground truth without QA/QC.
 
+To compare the same model/tile/threshold grid across all prepared operator
+samples, use:
+
+```bash
+OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
+QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \
+QUALITY_TILE_SIZES="512 640" \
+QUALITY_TILE_OVERLAPS="64" \
+QUALITY_THRESHOLDS="0.50 0.15" \
+bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.168.10.150:1202
+```
+
+The multi-sample summary exposes `best_overall_by_score`,
+`best_overall_by_recall`, `best_overall_by_precision` and `best_by_sample` so
+model-quality decisions are based on repeated persisted QA/QC evidence rather
+than one AOI.
+
 For visual error inspection, export the persisted QA evidence from a calibration
 summary:
 

docs/CODEX_EXECUTION_LOG.md

@@ -1,3 +1,37 @@
+## Sprint 127 Multi-sample detection quality calibration tooling (2026-07-07)
+
+Changed:
+- Added `scripts/prepare_operator_real_data_samples.py` as an explicit operator/runtime helper for documented Geel, Mol and Turnhout real-data samples.
+- The helper downloads small Digitaal Vlaanderen OMWRGBMRVL WMS `Ortho` GeoTIFFs and GRB OGC API Features `GBG` building GeoJSON references for the documented AOIs only.
+- The helper writes `operator_samples_manifest.json`, sample metadata, source URLs and attribution under the runtime operator-data directory and reuses existing files by default.
+- Added `scripts/run_multi_sample_detection_quality_matrix.sh` to run `scripts/run_detection_quality_matrix.sh` once per manifest sample.
+- The multi-sample wrapper combines per-sample `quality_matrix_summary.json` files into `multi_sample_quality_summary.json` with `best_overall_by_score`, `best_overall_by_recall`, `best_overall_by_precision` and `best_by_sample`.
+- Added readiness checks for Python compile and shell syntax.
+- Added regression coverage in `backend/tests/test_sprint127_operator_sample_quality_matrix.py`.
+- Updated `scripts/README.md`, `backend/README.md`, `docs/AI_PIPELINES.md`, `docs/TODO.md` and `CHANGELOG.md`.
+
+Tested:
+- RED: `python -m pytest backend\tests\test_sprint127_operator_sample_quality_matrix.py -q` failed because the sample-preparation and multi-sample scripts did not exist.
+- RED: `python -m pytest backend\tests\test_sprint127_operator_sample_quality_matrix.py::test_prepare_operator_real_data_samples_help_does_not_require_gis_dependencies -q` failed because `--help` required missing GIS dependencies.
+- `python -m pytest backend\tests\test_sprint127_operator_sample_quality_matrix.py -q` passed.
+- `python scripts\prepare_operator_real_data_samples.py --help` passed without requiring local GIS dependencies.
+- `python -m py_compile scripts\prepare_operator_real_data_samples.py` passed.
+- `bash -n scripts/run_multi_sample_detection_quality_matrix.sh` passed.
+- `python -m pytest backend\tests\test_sprint127_operator_sample_quality_matrix.py backend\tests\test_sprint126_detection_quality_matrix.py backend\tests\test_sprint125_detection_calibration_evidence_bundle.py backend\tests\test_sprint124_detection_calibration_sweep.py -q` passed.
+- `python scripts\smoke_docs.py` passed.
+- `git diff --check` passed.
+- `bash scripts/run_readiness_check.sh` passed: 393 backend tests, frontend typecheck/build, Alembic head `202606120900`, live smoke syntax checks and the new sample/multi-sample checks.
+
+Open:
+- Live Tower sample preparation and multi-sample matrix run still needed.
+
+Limitations:
+- This is operator tooling only. It does not add a live GRB provider, live orthophoto provider, application endpoint, migration, frontend feature, model download or fixture inference path.
+- The prepared sample files are runtime artifacts under appdata/storage and remain excluded from Git.
+
+Next recommended pass:
+- Run the sample-preparation helper in the Tower all-in-one container, then run the multi-sample matrix from the Tower checkout and document the combined quality baseline.
+
 ## Sprint 126 Detection quality matrix tooling (2026-07-07)
 
 Changed:

docs/TODO.md

@@ -98,7 +98,8 @@ This file now starts with the current implementation status. Older preparation/b
 - [x] Add real-data detection calibration sweep tooling for confidence-threshold and QA/QC metric comparison.
 - [x] Add calibration QA evidence export tooling for false-positive/false-negative inspection artifacts.
 - [x] Add real-data detection quality matrix tooling for model/tile/threshold comparison.
-- [ ] Calibrate confidence, IoU and model selection against persisted Geel detections and additional local orthophoto/reference samples.
+- [x] Add reproducible Geel/Mol/Turnhout operator sample preparation and multi-sample quality matrix tooling.
+- [ ] Calibrate confidence, IoU and model selection against persisted Geel/Mol/Turnhout detections and any additional local orthophoto/reference samples.
 
 ## Sprint 8 status
 

scripts/README.md

@@ -171,6 +171,20 @@ Those files are runtime artifacts generated from Digitaal Vlaanderen's
 OMWRGBMRVL WMS `Ortho` layer and GRB OGC API Features `GBG` building collection
 for a small Geel AOI. They are intentionally not repository fixtures.
 
+To prepare the documented operator samples reproducibly inside the all-in-one
+runtime container, run:
+
+```bash
+docker exec -it geointel python /app/scripts/prepare_operator_real_data_samples.py --samples geel,mol,turnhout
+```
+
+This writes GeoTIFF/GeoJSON pairs and `operator_samples_manifest.json` under
+`/app/storage/operator-data` inside the container, which maps to
+`storage/operator-data` in the Tower appdata checkout. The helper fetches only
+the explicit documented AOIs, records Digitaal Vlaanderen attribution and
+reuses existing files by default. Use `--force` only when the local runtime
+artifacts should be regenerated.
+
 The real-data smoke is intentionally mutating and refuses to run without
 operator-supplied files. Current V1 upload support expects a georeferenced
 `.tif`, `.tiff` or `.geotiff` raster and a `.geojson` or `.json` reference
@@ -229,6 +243,24 @@ set. The summary ranks `best_by_score`, `best_by_recall` and
 `QualityCheck`/`Metric` evidence rather than visual guesses. It does not create
 provider data, use fixtures or download model weights.
 
+Run the same matrix across every prepared operator sample:
+
+```bash
+OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
+QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \
+QUALITY_TILE_SIZES="512 640" \
+QUALITY_TILE_OVERLAPS="64" \
+QUALITY_THRESHOLDS="0.50 0.15" \
+bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.168.10.150:1202
+```
+
+The multi-sample wrapper writes one per-sample `quality_matrix_summary.json`
+plus a combined `multi_sample_quality_summary.json` with
+`best_overall_by_score`, `best_overall_by_recall`,
+`best_overall_by_precision` and `best_by_sample` rankings. It resolves
+container-style `/app/storage/...` manifest paths to repo-relative
+`storage/...` paths when run from the Tower host checkout.
+
 Export calibration QA evidence for visual review:
 
 ```bash

scripts/prepare_operator_real_data_samples.py

@@ -0,0 +1,319 @@
+"""Prepare explicit real operator samples for GeoIntel detection QA.
+
+This script downloads small orthophoto and GRB building reference pairs from
+Digitaal Vlaanderen for documented Kempen AOIs. It is an operator/runtime
+helper, not an application provider integration: no GeoIntel API route calls it
+and no production data is fetched silently by the app.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms"
+GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
+DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data")
+requests: Any = None
+rasterio: Any = None
+Transformer: Any = None
+MemoryFile: Any = None
+from_bounds: Any = None
+
+
+@dataclass(frozen=True)
+class OperatorSample:
+    slug: str
+    display_name: str
+    center_lon: float
+    center_lat: float
+    half_size_m: float = 250.0
+    width: int = 512
+    height: int = 512
+
+
+SAMPLES: dict[str, OperatorSample] = {
+    "geel": OperatorSample(
+        slug="geel",
+        display_name="Geel center",
+        center_lon=4.991,
+        center_lat=51.162,
+    ),
+    "mol": OperatorSample(
+        slug="mol",
+        display_name="Mol center",
+        center_lon=5.1167,
+        center_lat=51.1919,
+    ),
+    "turnhout": OperatorSample(
+        slug="turnhout",
+        display_name="Turnhout center",
+        center_lon=4.9488,
+        center_lat=51.3225,
+    ),
+}
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Prepare real Digitaal Vlaanderen orthophoto/GRB building samples for GeoIntel operator QA.",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=Path,
+        default=Path(os.environ.get("OPERATOR_DATA_DIR", DEFAULT_OUTPUT_DIR)),
+        help="Directory for generated GeoTIFF, GeoJSON and manifest files.",
+    )
+    parser.add_argument(
+        "--samples",
+        default=",".join(SAMPLES),
+        help="Comma/space separated sample slugs to prepare. Defaults to all documented samples.",
+    )
+    parser.add_argument(
+        "--force",
+        action="store_true",
+        help="Refetch and overwrite sample files. By default existing raster/reference pairs are reused.",
+    )
+    parser.add_argument(
+        "--manifest-name",
+        default="operator_samples_manifest.json",
+        help="Manifest filename written inside output-dir.",
+    )
+    return parser.parse_args()
+
+
+def ensure_gis_dependencies() -> None:
+    global MemoryFile, Transformer, from_bounds, rasterio, requests
+    try:
+        import requests as requests_module
+        import rasterio as rasterio_module
+        from pyproj import Transformer as transformer_class
+        from rasterio.io import MemoryFile as memory_file_class
+        from rasterio.transform import from_bounds as from_bounds_function
+    except Exception as exc:  # pragma: no cover - exercised only in runtime envs.
+        raise SystemExit(
+            "prepare_operator_real_data_samples.py requires requests, rasterio and pyproj. "
+            "Run it inside the GeoIntel all-in-one container or an equivalent GIS Python environment."
+        ) from exc
+    requests = requests_module
+    rasterio = rasterio_module
+    Transformer = transformer_class
+    MemoryFile = memory_file_class
+    from_bounds = from_bounds_function
+
+
+def selected_samples(raw: str) -> list[OperatorSample]:
+    slugs = [value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()]
+    if not slugs:
+        raise SystemExit("--samples must include at least one sample slug")
+    unknown = [slug for slug in slugs if slug not in SAMPLES]
+    if unknown:
+        raise SystemExit(f"Unknown sample slug(s): {', '.join(unknown)}. Known: {', '.join(SAMPLES)}")
+    return [SAMPLES[slug] for slug in slugs]
+
+
+def sample_bounds(sample: OperatorSample) -> tuple[tuple[float, float, float, float], list[float]]:
+    lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
+    wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
+    center_x, center_y = lambert.transform(sample.center_lon, sample.center_lat)
+    minx = center_x - sample.half_size_m
+    miny = center_y - sample.half_size_m
+    maxx = center_x + sample.half_size_m
+    maxy = center_y + sample.half_size_m
+    corners = [
+        wgs84.transform(x, y)
+        for x, y in ((minx, miny), (minx, maxy), (maxx, miny), (maxx, maxy))
+    ]
+    lon_values = [point[0] for point in corners]
+    lat_values = [point[1] for point in corners]
+    return (minx, miny, maxx, maxy), [
+        min(lon_values),
+        min(lat_values),
+        max(lon_values),
+        max(lat_values),
+    ]
+
+
+def prepared_url(url: str, params: dict[str, str]) -> str:
+    return requests.Request("GET", url, params=params).prepare().url
+
+
+def raster_summary(path: Path) -> dict[str, Any]:
+    with rasterio.open(path) as ds:
+        return {
+            "path": str(path),
+            "crs": str(ds.crs),
+            "bounds": list(ds.bounds),
+            "width": ds.width,
+            "height": ds.height,
+            "count": ds.count,
+            "dtypes": list(ds.dtypes),
+        }
+
+
+def geojson_feature_count(path: Path) -> int:
+    payload = json.loads(path.read_text(encoding="utf-8-sig"))
+    return len(payload.get("features") or [])
+
+
+def fetch_orthophoto(sample: OperatorSample, ortho_path: Path, lambert_bbox: tuple[float, float, float, float]) -> str:
+    minx, miny, maxx, maxy = lambert_bbox
+    wms_params = {
+        "SERVICE": "WMS",
+        "VERSION": "1.3.0",
+        "REQUEST": "GetMap",
+        "LAYERS": "Ortho",
+        "STYLES": "",
+        "FORMAT": "image/tiff",
+        "CRS": "EPSG:31370",
+        "BBOX": f"{minx},{miny},{maxx},{maxy}",
+        "WIDTH": str(sample.width),
+        "HEIGHT": str(sample.height),
+    }
+    response = requests.get(WMS_URL, params=wms_params, timeout=120)
+    response.raise_for_status()
+    content_type = response.headers.get("content-type", "")
+    if "image" not in content_type.lower() and "tiff" not in content_type.lower():
+        raise SystemExit(f"Orthophoto WMS did not return an image for {sample.slug}: {content_type}")
+
+    with MemoryFile(response.content) as memfile:
+        with memfile.open() as src:
+            image = src.read()
+            profile = src.profile.copy()
+            profile.update(
+                driver="GTiff",
+                width=src.width,
+                height=src.height,
+                count=src.count,
+                dtype=src.dtypes[0],
+                crs="EPSG:31370",
+                transform=from_bounds(minx, miny, maxx, maxy, src.width, src.height),
+                compress="deflate",
+                tiled=False,
+            )
+            with rasterio.open(ortho_path, "w", **profile) as dst:
+                dst.write(image)
+                dst.update_tags(
+                    source="Digitaal Vlaanderen OMWRGBMRVL WMS Ortho layer",
+                    source_url=prepared_url(WMS_URL, wms_params),
+                    attribution="Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen",
+                    aoi=f"{sample.display_name} sample AOI for GeoIntel operator validation",
+                )
+    return prepared_url(WMS_URL, wms_params)
+
+
+def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list[float]) -> tuple[str, int]:
+    ogc_params = {
+        "f": "application/geo+json",
+        "limit": "1000",
+        "bbox": ",".join(f"{value:.8f}" for value in geo_bbox),
+    }
+    response = requests.get(GRB_GBG_URL, params=ogc_params, timeout=120)
+    response.raise_for_status()
+    reference = response.json()
+    features = reference.get("features") or []
+    if not features:
+        raise SystemExit(f"GRB GBG returned no building features for {sample.slug} bbox {geo_bbox}")
+
+    reference["name"] = f"GRB GBG buildings - {sample.display_name} sample AOI"
+    reference["source"] = "Digitaal Vlaanderen GRB OGC API Features collection GBG"
+    reference["source_url"] = prepared_url(GRB_GBG_URL, ogc_params)
+    reference["attribution"] = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen"
+    reference["bbox"] = geo_bbox
+    reference["sample_slug"] = sample.slug
+    for feature in features:
+        props = feature.setdefault("properties", {})
+        props.setdefault("source_name", "grb")
+        props.setdefault("reference_layer_name", "buildings")
+        props.setdefault("sample_slug", sample.slug)
+
+    reference_path.write_text(json.dumps(reference, ensure_ascii=False), encoding="utf-8")
+    return prepared_url(GRB_GBG_URL, ogc_params), len(features)
+
+
+def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dict[str, Any]:
+    ortho_path = output_dir / f"{sample.slug}_orthophoto_wms_512.tif"
+    reference_path = output_dir / f"{sample.slug}_grb_gbg_buildings.geojson"
+    lambert_bbox, geo_bbox = sample_bounds(sample)
+    skip_existing = ortho_path.exists() and reference_path.exists() and not force
+
+    source_urls: dict[str, str | None] = {"orthophoto": None, "reference": None}
+    if not skip_existing:
+        source_urls["orthophoto"] = fetch_orthophoto(sample, ortho_path, lambert_bbox)
+        source_urls["reference"], reference_feature_count = fetch_reference(sample, reference_path, geo_bbox)
+    else:
+        reference_feature_count = geojson_feature_count(reference_path)
+
+    return {
+        "sample_slug": sample.slug,
+        "display_name": sample.display_name,
+        "center_lon": sample.center_lon,
+        "center_lat": sample.center_lat,
+        "half_size_m": sample.half_size_m,
+        "raster_path": str(ortho_path),
+        "reference_path": str(reference_path),
+        "reference_feature_count": reference_feature_count,
+        "raster": raster_summary(ortho_path),
+        "wgs84_bbox": geo_bbox,
+        "epsg31370_bbox": list(lambert_bbox),
+        "skip_existing": skip_existing,
+        "source_urls": source_urls,
+        "attribution": {
+            "orthophoto": "Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen",
+            "reference": "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen",
+        },
+    }
+
+
+def write_readme(output_dir: Path, samples: list[dict[str, Any]]) -> None:
+    lines = [
+        "# GeoIntel operator real-data samples",
+        "",
+        "Generated for runtime validation, not committed to the GeoIntel repository.",
+        "",
+        "Sources:",
+        "- Orthophoto rasters: Digitaal Vlaanderen OMWRGBMRVL WMS `Ortho` layer.",
+        "- Reference buildings: Digitaal Vlaanderen GRB OGC API Features `GBG` collection.",
+        "- Attribution: Bron: Orthofotomozaiek Vlaanderen / Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen.",
+        "",
+        "Samples:",
+    ]
+    for sample in samples:
+        lines.append(
+            f"- `{sample['sample_slug']}`: `{Path(sample['raster_path']).name}` and "
+            f"`{Path(sample['reference_path']).name}`, "
+            f"{sample['reference_feature_count']} reference features."
+        )
+    lines.append("")
+    lines.append("Purpose: configured-YOLO detection + persisted QA/QC validation with operator-provided files.")
+    (output_dir / "README.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def main() -> int:
+    args = parse_args()
+    ensure_gis_dependencies()
+    output_dir: Path = args.output_dir
+    output_dir.mkdir(parents=True, exist_ok=True)
+    samples = [prepare_sample(sample, output_dir, force=args.force) for sample in selected_samples(args.samples)]
+    write_readme(output_dir, samples)
+
+    manifest = {
+        "schema_version": 1,
+        "description": "GeoIntel operator real-data samples for configured-YOLO QA validation.",
+        "output_dir": str(output_dir),
+        "samples": samples,
+    }
+    manifest_path = output_dir / args.manifest_name
+    manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8")
+    print(json.dumps({"status": "ok", "manifest_path": str(manifest_path), "samples": samples}, indent=2, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/run_multi_sample_detection_quality_matrix.sh

@@ -0,0 +1,226 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat >&2 <<'EOF'
+Usage:
+  OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
+  QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \
+  QUALITY_TILE_SIZES="512 640" \
+  QUALITY_TILE_OVERLAPS="64" \
+  QUALITY_THRESHOLDS="0.50 0.15" \
+  bash scripts/run_multi_sample_detection_quality_matrix.sh [base_url]
+
+Optional environment:
+  OPERATOR_SAMPLE_MANIFEST_PATH  Manifest from prepare_operator_real_data_samples.py.
+  OPERATOR_SAMPLE_SLUGS          Optional comma/space separated sample slug filter.
+  MULTI_SAMPLE_OUTPUT_DIR        Output directory, default: artifacts/detection-quality-matrix/multi-sample/<timestamp>.
+  QUALITY_MODEL_ASSET_IDS        Forwarded to run_detection_quality_matrix.sh.
+  QUALITY_TILE_SIZES             Forwarded to run_detection_quality_matrix.sh.
+  QUALITY_TILE_OVERLAPS          Forwarded to run_detection_quality_matrix.sh.
+  QUALITY_THRESHOLDS             Forwarded to run_detection_quality_matrix.sh.
+  REAL_IOU_THRESHOLD             Forwarded to run_detection_quality_matrix.sh.
+
+This script does not run inference itself. It repeats the existing real-data
+quality matrix once per documented operator sample and combines the summaries.
+EOF
+}
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT"
+
+BASE_URL="${1:-${GE_INTEL_BASE_URL:-http://localhost:1202}}"
+OPERATOR_SAMPLE_MANIFEST_PATH="${OPERATOR_SAMPLE_MANIFEST_PATH:-storage/operator-data/operator_samples_manifest.json}"
+OPERATOR_SAMPLE_SLUGS="${OPERATOR_SAMPLE_SLUGS:-}"
+MULTI_SAMPLE_OUTPUT_DIR="${MULTI_SAMPLE_OUTPUT_DIR:-artifacts/detection-quality-matrix/multi-sample/$(date -u +%Y%m%dT%H%M%SZ)}"
+
+if [ "${BASE_URL}" = "-h" ] || [ "${BASE_URL}" = "--help" ]; then
+  usage
+  exit 0
+fi
+
+if [ ! -f "${OPERATOR_SAMPLE_MANIFEST_PATH}" ]; then
+  echo "OPERATOR_SAMPLE_MANIFEST_PATH does not point to a readable manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}" >&2
+  exit 2
+fi
+
+if [ -n "${PYTHON_BIN:-}" ]; then
+  PYTHON_BIN="${PYTHON_BIN}"
+else
+  PYTHON_BIN=""
+  for candidate in python3 python.exe python; do
+    if command -v "${candidate}" >/dev/null 2>&1 && "${candidate}" -c "import json, sys" >/dev/null 2>&1; then
+      PYTHON_BIN="${candidate}"
+      break
+    fi
+  done
+fi
+
+if [ -z "${PYTHON_BIN}" ]; then
+  echo "A Python interpreter is required for JSON parsing" >&2
+  exit 1
+fi
+
+mkdir -p "${MULTI_SAMPLE_OUTPUT_DIR}"
+sample_manifest_tsv="${MULTI_SAMPLE_OUTPUT_DIR}/multi_sample_requests.tsv"
+
+"${PYTHON_BIN}" - \
+  "${ROOT}" \
+  "${OPERATOR_SAMPLE_MANIFEST_PATH}" \
+  "${OPERATOR_SAMPLE_SLUGS}" \
+  "${sample_manifest_tsv}" <<'PY'
+import json
+import os
+import sys
+from pathlib import Path
+
+root = Path(sys.argv[1]).resolve()
+manifest_path = Path(sys.argv[2])
+slug_filter_raw = sys.argv[3]
+output_path = Path(sys.argv[4])
+
+payload = json.loads(manifest_path.read_text(encoding="utf-8-sig"))
+samples = payload.get("samples") or []
+if not samples:
+    raise SystemExit("Operator sample manifest contains no samples")
+
+requested_slugs = {
+    value.strip().lower()
+    for value in slug_filter_raw.replace(",", " ").split()
+    if value.strip()
+}
+
+
+def resolve_path(raw: str) -> str:
+    path = Path(raw)
+    if path.exists():
+        return str(path)
+    if raw.startswith("/app/"):
+        candidate = root / raw.removeprefix("/app/")
+        if candidate.exists():
+            return str(candidate)
+    candidate = root / raw
+    if candidate.exists():
+        return str(candidate)
+    raise SystemExit(f"Sample file is not readable from this host: {raw}")
+
+
+with output_path.open("w", encoding="utf-8") as handle:
+    selected = 0
+    for sample in samples:
+        sample_slug = str(sample.get("sample_slug") or "").lower()
+        if not sample_slug:
+            raise SystemExit("Operator sample is missing sample_slug")
+        if requested_slugs and sample_slug not in requested_slugs:
+            continue
+        raster_path = resolve_path(str(sample.get("raster_path") or ""))
+        reference_path = resolve_path(str(sample.get("reference_path") or ""))
+        reference_count = int(sample.get("reference_feature_count") or 0)
+        if reference_count < 1:
+            raise SystemExit(f"Operator sample has no reference features: {sample_slug}")
+        handle.write(f"{sample_slug}\t{raster_path}\t{reference_path}\t{reference_count}\n")
+        selected += 1
+
+if selected == 0:
+    raise SystemExit("No operator samples matched OPERATOR_SAMPLE_SLUGS")
+PY
+
+echo "== GeoIntel multi-sample detection quality matrix =="
+echo "Base URL: ${BASE_URL}"
+echo "Manifest: ${OPERATOR_SAMPLE_MANIFEST_PATH}"
+echo "Sample filter: ${OPERATOR_SAMPLE_SLUGS:-all}"
+echo "Output: ${MULTI_SAMPLE_OUTPUT_DIR}"
+
+while IFS=$'\t' read -r sample_slug raster_path reference_path reference_feature_count; do
+  sample_output_dir="${MULTI_SAMPLE_OUTPUT_DIR}/${sample_slug}"
+  mkdir -p "${sample_output_dir}"
+  echo "-- Sample ${sample_slug}: reference_features=${reference_feature_count} --"
+  REAL_RASTER_PATH="${raster_path}" \
+  REAL_REFERENCE_VECTOR_PATH="${reference_path}" \
+  QUALITY_OUTPUT_DIR="${sample_output_dir}" \
+  bash scripts/run_detection_quality_matrix.sh "${BASE_URL}"
+done < "${sample_manifest_tsv}"
+
+"${PYTHON_BIN}" - "${MULTI_SAMPLE_OUTPUT_DIR}" "${BASE_URL}" "${OPERATOR_SAMPLE_MANIFEST_PATH}" <<'PY'
+import glob
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+output_dir = Path(sys.argv[1])
+base_url = sys.argv[2]
+manifest_path = sys.argv[3]
+
+sample_summaries = []
+flat_items = []
+for summary_path in sorted(glob.glob(str(output_dir / "*" / "quality_matrix_summary.json"))):
+    sample_slug = Path(summary_path).parent.name
+    summary = json.loads(Path(summary_path).read_text(encoding="utf-8"))
+    items = summary.get("items") or []
+    for item in items:
+        enriched = dict(item)
+        enriched["sample_slug"] = sample_slug
+        flat_items.append(enriched)
+    sample_summaries.append(
+        {
+            "sample_slug": sample_slug,
+            "summary_path": summary_path,
+            "run_count": len(items),
+            "best_by_score": summary.get("best_by_score"),
+            "best_by_recall": summary.get("best_by_recall"),
+            "best_by_precision": summary.get("best_by_precision"),
+        }
+    )
+
+if not flat_items:
+    raise SystemExit("No sample quality matrix summaries were produced")
+
+
+def best(metric: str):
+    ranked = [item for item in flat_items if item.get(metric) is not None]
+    return max(ranked, key=lambda item: item[metric], default=None)
+
+
+best_by_sample = {
+    sample["sample_slug"]: sample.get("best_by_score")
+    for sample in sample_summaries
+}
+summary = {
+    "generated_at": datetime.now(timezone.utc).isoformat(),
+    "base_url": base_url,
+    "operator_sample_manifest_path": manifest_path,
+    "sample_count": len(sample_summaries),
+    "run_count": len(flat_items),
+    "best_overall_by_score": best("quality_score"),
+    "best_overall_by_recall": best("recall"),
+    "best_overall_by_precision": best("precision"),
+    "best_by_sample": best_by_sample,
+    "sample_summaries": sample_summaries,
+    "items": flat_items,
+}
+summary_path = output_dir / "multi_sample_quality_summary.json"
+summary_path.write_text(json.dumps(summary, indent=2, sort_keys=True), encoding="utf-8")
+
+print("")
+print("Multi-sample detection quality summary")
+print("sample\tmodel\ttile\toverlap\tthreshold\tdetections\tscore\tprecision\trecall\tf1\tmatches\tfp\tfn")
+for item in flat_items:
+    print(
+        "{sample_slug}\t{model_asset_id}\t{tile_size}\t{tile_overlap}\t{threshold:.2f}\t{detection_count}\t{quality_score}\t{precision}\t{recall}\t{f1_score}\t{matches}\t{false_positives}\t{false_negatives}".format(
+            **item
+        )
+    )
+print("")
+print(f"Summary: {summary_path}")
+for key in ("best_overall_by_score", "best_overall_by_recall", "best_overall_by_precision"):
+    item = summary.get(key)
+    if item:
+        print(
+            f"{key} sample={item['sample_slug']} model={item['model_asset_id']} "
+            f"tile={item['tile_size']} overlap={item['tile_overlap']} "
+            f"threshold={item['threshold']:.2f} score={item.get('quality_score')} "
+            f"precision={item.get('precision')} recall={item.get('recall')} f1={item.get('f1_score')}"
+        )
+PY

scripts/run_readiness_check.sh

@@ -41,6 +41,7 @@ ${PYTHON_BIN} -m py_compile scripts/gis_import_smoke.py
 ${PYTHON_BIN} -m py_compile scripts/seed_demo_workflow.py
 ${PYTHON_BIN} -m py_compile scripts/yolo_preflight.py
 ${PYTHON_BIN} -m py_compile backend/scripts/yolo_preflight.py
+${PYTHON_BIN} -m py_compile scripts/prepare_operator_real_data_samples.py
 ${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
 ${PYTHON_BIN} -m py_compile backend/scripts/cleanup_demo_artifacts.py
 ${PYTHON_BIN} -m compileall backend/app
@@ -58,6 +59,7 @@ bash -n scripts/verify_real_data_detection_qa_workflow.sh
 bash -n scripts/run_detection_calibration_sweep.sh
 bash -n scripts/export_detection_calibration_evidence.sh
 bash -n scripts/run_detection_quality_matrix.sh
+bash -n scripts/run_multi_sample_detection_quality_matrix.sh
 bash -n scripts/verify_workbench_default_state.sh
 bash -n scripts/verify_workbench_interactions.sh
 bash -n scripts/verify_gis_runtime.sh