Add multi-sample detection quality calibration

backend/README.md

@@ -374,6 +374,17 @@ manifests generated for AI handoff include source CRS metadata so pixel-space
 model outputs can be transformed to WGS84 GeoJSON coordinates. Current V1 upload
 support is limited to GeoTIFF-style rasters and GeoJSON/JSON reference vectors.
 
+To prepare the documented Geel/Mol/Turnhout operator sample pairs inside the
+all-in-one runtime container, run:
+
+```bash
+docker exec -it geointel python /app/scripts/prepare_operator_real_data_samples.py --samples geel,mol,turnhout
+```
+
+The helper writes GeoTIFF orthophotos, GRB GBG building GeoJSON files and
+`operator_samples_manifest.json` under `/app/storage/operator-data`. These are
+runtime artifacts only and are not committed to Git.
+
 For model-quality calibration, run the confidence sweep wrapper:
 
 ```bash
@@ -409,6 +420,21 @@ and false-positive/false-negative counts. It ranks `best_by_score`,
 `best_by_recall` and `best_by_precision`. It does not download weights, create
 fake detections, fetch live providers or change backend API behavior.
 
+To aggregate the same matrix over every prepared operator sample, run:
+
+```bash
+OPERATOR_SAMPLE_MANIFEST_PATH=storage/operator-data/operator_samples_manifest.json \
+QUALITY_MODEL_ASSET_IDS="yolov8n-building-segmentation-pt yolov8n-pt" \
+QUALITY_TILE_SIZES="512 640" \
+QUALITY_TILE_OVERLAPS="64" \
+QUALITY_THRESHOLDS="0.50 0.15" \
+bash scripts/run_multi_sample_detection_quality_matrix.sh http://192.168.10.150:1202
+```
+
+The combined `multi_sample_quality_summary.json` reports per-sample and overall
+best configurations. It is an operator benchmarking command, not a backend API
+or provider import path.
+
 To inspect the evidence behind a calibration run, export the persisted QA
 evidence bundle:
 

backend/tests/test_sprint127_operator_sample_quality_matrix.py

@@ -0,0 +1,71 @@
+from pathlib import Path
+import subprocess
+import sys
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def test_prepare_operator_real_data_samples_fetches_documented_ortho_and_grb_pairs() -> None:
+    script_path = ROOT / "scripts" / "prepare_operator_real_data_samples.py"
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
+
+    assert script_path.exists()
+    script = script_path.read_text(encoding="utf-8")
+
+    assert "py_compile scripts/prepare_operator_real_data_samples.py" in readiness
+    assert "SAMPLES" in script
+    assert '"geel"' in script
+    assert '"mol"' in script
+    assert '"turnhout"' in script
+    assert "https://geo.api.vlaanderen.be/omwrgbmrvl/wms" in script
+    assert "LAYERS" in script
+    assert "Ortho" in script
+    assert "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items" in script
+    assert "source_name" in script
+    assert "reference_layer_name" in script
+    assert "operator_samples_manifest.json" in script
+    assert "skip_existing" in script
+    assert "demo/workflow" not in script
+    assert "fixture_mode" not in script
+
+
+def test_prepare_operator_real_data_samples_help_does_not_require_gis_dependencies() -> None:
+    script_path = ROOT / "scripts" / "prepare_operator_real_data_samples.py"
+
+    result = subprocess.run(
+        [sys.executable, str(script_path), "--help"],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 0
+    assert "Prepare real Digitaal Vlaanderen" in result.stdout
+    assert "--samples" in result.stdout
+
+
+def test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sample() -> None:
+    script_path = ROOT / "scripts" / "run_multi_sample_detection_quality_matrix.sh"
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
+
+    assert script_path.exists()
+    script = script_path.read_text(encoding="utf-8")
+
+    assert "bash -n scripts/run_multi_sample_detection_quality_matrix.sh" in readiness
+    assert "OPERATOR_SAMPLE_MANIFEST_PATH" in script
+    assert "operator_samples_manifest.json" in script
+    assert "run_detection_quality_matrix.sh" in script
+    assert "QUALITY_MODEL_ASSET_IDS" in script
+    assert "QUALITY_TILE_SIZES" in script
+    assert "QUALITY_TILE_OVERLAPS" in script
+    assert "QUALITY_THRESHOLDS" in script
+    assert "REAL_RASTER_PATH" in script
+    assert "REAL_REFERENCE_VECTOR_PATH" in script
+    assert "multi_sample_quality_summary.json" in script
+    assert "best_overall_by_score" in script
+    assert "best_by_sample" in script
+    assert "sample_slug" in script
+    assert "demo/workflow" not in script
+    assert "fixture_mode" not in script
+    assert "will_download_models" not in script