Add multi-sample detection quality calibration

backend/tests/test_sprint127_operator_sample_quality_matrix.py

@@ -0,0 +1,71 @@
+from pathlib import Path
+import subprocess
+import sys
+
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def test_prepare_operator_real_data_samples_fetches_documented_ortho_and_grb_pairs() -> None:
+    script_path = ROOT / "scripts" / "prepare_operator_real_data_samples.py"
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
+
+    assert script_path.exists()
+    script = script_path.read_text(encoding="utf-8")
+
+    assert "py_compile scripts/prepare_operator_real_data_samples.py" in readiness
+    assert "SAMPLES" in script
+    assert '"geel"' in script
+    assert '"mol"' in script
+    assert '"turnhout"' in script
+    assert "https://geo.api.vlaanderen.be/omwrgbmrvl/wms" in script
+    assert "LAYERS" in script
+    assert "Ortho" in script
+    assert "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items" in script
+    assert "source_name" in script
+    assert "reference_layer_name" in script
+    assert "operator_samples_manifest.json" in script
+    assert "skip_existing" in script
+    assert "demo/workflow" not in script
+    assert "fixture_mode" not in script
+
+
+def test_prepare_operator_real_data_samples_help_does_not_require_gis_dependencies() -> None:
+    script_path = ROOT / "scripts" / "prepare_operator_real_data_samples.py"
+
+    result = subprocess.run(
+        [sys.executable, str(script_path), "--help"],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 0
+    assert "Prepare real Digitaal Vlaanderen" in result.stdout
+    assert "--samples" in result.stdout
+
+
+def test_multi_sample_detection_quality_matrix_runs_existing_matrix_for_each_sample() -> None:
+    script_path = ROOT / "scripts" / "run_multi_sample_detection_quality_matrix.sh"
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
+
+    assert script_path.exists()
+    script = script_path.read_text(encoding="utf-8")
+
+    assert "bash -n scripts/run_multi_sample_detection_quality_matrix.sh" in readiness
+    assert "OPERATOR_SAMPLE_MANIFEST_PATH" in script
+    assert "operator_samples_manifest.json" in script
+    assert "run_detection_quality_matrix.sh" in script
+    assert "QUALITY_MODEL_ASSET_IDS" in script
+    assert "QUALITY_TILE_SIZES" in script
+    assert "QUALITY_TILE_OVERLAPS" in script
+    assert "QUALITY_THRESHOLDS" in script
+    assert "REAL_RASTER_PATH" in script
+    assert "REAL_REFERENCE_VECTOR_PATH" in script
+    assert "multi_sample_quality_summary.json" in script
+    assert "best_overall_by_score" in script
+    assert "best_by_sample" in script
+    assert "sample_slug" in script
+    assert "demo/workflow" not in script
+    assert "fixture_mode" not in script
+    assert "will_download_models" not in script