diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7601aadb..a842e33b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,14 @@
# Changelog
+## Sprint 184 Detection QA coverage and matching diagnostics (2026-07-14)
+
+- Clipped configured-YOLO candidate and reference QA populations to the union of the exact persisted inference tile footprints before canonical IoU matching.
+- Added fail-closed validation for missing, invalid or cross-dataset tile-manifest provenance while retaining the existing unbounded behavior for explicit fixture and legacy runs without a manifest.
+- Kept precision, recall, F1 and mean IoU strictly based on candidate polygons versus persisted reference footprints; added a separately labelled reference-envelope comparison as diagnostic evidence only.
+- Persisted raw/evaluated/excluded/clipped population counts and diagnostic matching evidence in the existing `quality_checks.findings_json` structure without changing migrations or canonical metric rows.
+- Surfaced inference coverage and box-to-footprint diagnostics in Detection Lab and hardened the real-data workflow assertions, documentation and regression coverage.
+
## Sprint 183 Mol map source clarity and live AI validation (2026-07-14)
- Added an explicit Database/Analysis result map-content mode so an automatically loaded detection result can no longer mask a newly selected persisted municipality layer.
diff --git a/backend/README.md b/backend/README.md
index b3d8d3eb..c570bc3f 100644
--- a/backend/README.md
+++ b/backend/README.md
@@ -152,6 +152,12 @@ bash scripts/live_migration_smoke.sh
- compares persisted detection geometries against persisted `vector_features`
- persists `quality_checks` and `metrics`
- returns precision, recall, F1, mean IoU and false positive/negative counts
+ - configured-YOLO runs clip both QA populations to persisted tile-manifest
+ coverage before matching and fail closed on missing/mismatched coverage
+ provenance
+ - persists a diagnostic-only candidate-box versus reference-envelope pass so
+ box-to-footprint matching artifacts are visible without altering canonical
+ footprint-IoU metrics
- Segmentation, LiDAR, AI Copilot, Training Studio and Reports remain out of scope.
## Sprint 9 additions
diff --git a/backend/app/services/detection_qa_service.py b/backend/app/services/detection_qa_service.py
new file mode 100644
index 00000000..f4911741
--- /dev/null
+++ b/backend/app/services/detection_qa_service.py
@@ -0,0 +1,234 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from math import isfinite
+from typing import Any
+from uuid import UUID
+
+from pyproj import CRS, Transformer
+from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, box
+from shapely.geometry.base import BaseGeometry
+from shapely.ops import transform as shapely_transform
+from shapely.ops import unary_union
+from shapely.validation import make_valid
+
+from app.core.errors import AppError
+from app.services.qa_service import QaMatchEvidence
+
+
+@dataclass(frozen=True)
+class DetectionQaCoverage:
+ geometry: BaseGeometry
+ manifest_path: str
+ tile_count: int
+ source_crs_values: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class CoveragePopulation:
+ geometries: list[tuple[dict[str, Any], BaseGeometry]]
+ raw_count: int
+ evaluated_count: int
+ excluded_outside_count: int
+ clipped_boundary_count: int
+
+
+class DetectionQaService:
+ @staticmethod
+ def tile_manifest_path(parameters: Any) -> str | None:
+ if not isinstance(parameters, dict):
+ return None
+ value = parameters.get("tile_manifest_path")
+ if isinstance(value, str) and value.strip():
+ return value.strip()
+ nested = parameters.get("parameters_json")
+ if isinstance(nested, dict):
+ value = nested.get("tile_manifest_path")
+ if isinstance(value, str) and value.strip():
+ return value.strip()
+ return None
+
+ @staticmethod
+ def build_tile_coverage(
+ manifest: dict[str, Any],
+ *,
+ manifest_path: str,
+ expected_dataset_id: UUID | None,
+ ) -> DetectionQaCoverage:
+ manifest_dataset_id = manifest.get("source_dataset_id") or manifest.get("source_raster_id")
+ if expected_dataset_id is not None and manifest_dataset_id and str(manifest_dataset_id) != str(expected_dataset_id):
+ raise AppError(
+ code="DETECTION_QA_COVERAGE_MISMATCH",
+ message="Detection tile manifest belongs to a different raster dataset",
+ details={
+ "analysis_dataset_id": str(expected_dataset_id),
+ "manifest_dataset_id": str(manifest_dataset_id),
+ },
+ status_code=422,
+ )
+
+ tiles = manifest.get("tiles")
+ if not isinstance(tiles, list) or not tiles:
+ raise AppError(
+ code="DETECTION_QA_COVERAGE_INVALID",
+ message="Detection tile manifest has no usable tile coverage",
+ status_code=422,
+ )
+
+ default_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs")
+ coverage_parts: list[BaseGeometry] = []
+ source_crs_values: set[str] = set()
+ target_crs = CRS.from_epsg(4326)
+
+ for tile_index, tile in enumerate(tiles):
+ if not isinstance(tile, dict):
+ raise DetectionQaService._coverage_error("Tile manifest entries must be objects", tile_index)
+ raw_bounds = tile.get("bounds")
+ if not isinstance(raw_bounds, (list, tuple)) or len(raw_bounds) != 4:
+ raise DetectionQaService._coverage_error("Tile manifest entries require four bounds values", tile_index)
+ try:
+ left, bottom, right, top = (float(value) for value in raw_bounds)
+ except (TypeError, ValueError) as exc:
+ raise DetectionQaService._coverage_error("Tile bounds must be numeric", tile_index) from exc
+ if not all(isfinite(value) for value in (left, bottom, right, top)) or left >= right or bottom >= top:
+ raise DetectionQaService._coverage_error("Tile bounds must define a finite non-empty extent", tile_index)
+
+ raw_crs = tile.get("crs") or default_crs
+ if not isinstance(raw_crs, str) or not raw_crs.strip():
+ raise DetectionQaService._coverage_error("Tile coverage requires explicit CRS metadata", tile_index)
+ try:
+ source_crs = CRS.from_user_input(raw_crs)
+ except Exception as exc:
+ raise DetectionQaService._coverage_error("Tile coverage CRS is invalid", tile_index) from exc
+ source_crs_values.add(source_crs.to_string())
+
+ tile_geometry: BaseGeometry = box(left, bottom, right, top)
+ if source_crs != target_crs:
+ transformer = Transformer.from_crs(source_crs, target_crs, always_xy=True)
+ tile_geometry = shapely_transform(transformer.transform, tile_geometry)
+ tile_geometry = DetectionQaService._valid_geometry(tile_geometry, tile_index=tile_index)
+ coverage_parts.append(tile_geometry)
+
+ coverage_geometry = DetectionQaService._valid_geometry(unary_union(coverage_parts))
+ min_x, min_y, max_x, max_y = coverage_geometry.bounds
+ if min_x < -180 or max_x > 180 or min_y < -90 or max_y > 90:
+ raise AppError(
+ code="DETECTION_QA_COVERAGE_INVALID",
+ message="Transformed tile coverage falls outside EPSG:4326 bounds",
+ details={"bounds": [min_x, min_y, max_x, max_y]},
+ status_code=422,
+ )
+ return DetectionQaCoverage(
+ geometry=coverage_geometry,
+ manifest_path=manifest_path,
+ tile_count=len(tiles),
+ source_crs_values=tuple(sorted(source_crs_values)),
+ )
+
+ @staticmethod
+ def filter_population(
+ geometries: list[tuple[dict[str, Any], BaseGeometry]],
+ coverage: DetectionQaCoverage,
+ ) -> CoveragePopulation:
+ evaluated: list[tuple[dict[str, Any], BaseGeometry]] = []
+ excluded_outside_count = 0
+ clipped_boundary_count = 0
+
+ for feature, geometry in geometries:
+ if geometry.is_empty or not geometry.intersects(coverage.geometry):
+ excluded_outside_count += 1
+ continue
+ try:
+ clipped = geometry.intersection(coverage.geometry)
+ except Exception as exc:
+ raise AppError(
+ code="GEOMETRY_OPERATION_UNSUPPORTED",
+ message="Unable to clip QA geometry to persisted tile coverage",
+ details={"reason": str(exc)},
+ status_code=422,
+ ) from exc
+ if clipped.is_empty or (geometry.geom_type in {"Polygon", "MultiPolygon"} and clipped.area <= 0):
+ excluded_outside_count += 1
+ continue
+ clipped = DetectionQaService._valid_geometry(clipped)
+ if not coverage.geometry.covers(geometry):
+ clipped_boundary_count += 1
+ evaluated.append((feature, clipped))
+
+ return CoveragePopulation(
+ geometries=evaluated,
+ raw_count=len(geometries),
+ evaluated_count=len(evaluated),
+ excluded_outside_count=excluded_outside_count,
+ clipped_boundary_count=clipped_boundary_count,
+ )
+
+ @staticmethod
+ def box_to_footprint_diagnostics(
+ strict_evidence: QaMatchEvidence,
+ envelope_evidence: QaMatchEvidence,
+ *,
+ iou_threshold: float,
+ ) -> dict[str, Any]:
+ envelope_metrics = DetectionQaService._metrics(envelope_evidence)
+ return {
+ "diagnostic_only": True,
+ "canonical_method": "candidate_polygon_vs_reference_footprint_iou",
+ "diagnostic_method": "candidate_polygon_vs_reference_envelope_iou",
+ "iou_threshold": iou_threshold,
+ "strict_matches": strict_evidence.matches,
+ "envelope_matches": envelope_evidence.matches,
+ "possible_box_to_footprint_mismatch_count": max(0, envelope_evidence.matches - strict_evidence.matches),
+ **envelope_metrics,
+ }
+
+ @staticmethod
+ def _metrics(evidence: QaMatchEvidence) -> dict[str, Any]:
+ precision = (
+ evidence.matches / (evidence.matches + evidence.false_positives)
+ if evidence.matches + evidence.false_positives > 0
+ else None
+ )
+ recall = (
+ evidence.matches / (evidence.matches + evidence.false_negatives)
+ if evidence.matches + evidence.false_negatives > 0
+ else None
+ )
+ f1_score = None
+ if precision is not None and recall is not None:
+ f1_score = (2 * precision * recall) / (precision + recall) if precision + recall > 0 else 0.0
+ mean_iou = (
+ sum(evidence.match_iou_values) / len(evidence.match_iou_values)
+ if evidence.match_iou_values
+ else None
+ )
+ return {
+ "envelope_false_positives": evidence.false_positives,
+ "envelope_false_negatives": evidence.false_negatives,
+ "envelope_precision": precision,
+ "envelope_recall": recall,
+ "envelope_f1_score": f1_score,
+ "envelope_mean_iou": mean_iou,
+ }
+
+ @staticmethod
+ def _valid_geometry(geometry: BaseGeometry, *, tile_index: int | None = None) -> BaseGeometry:
+ if not geometry.is_valid:
+ geometry = make_valid(geometry)
+ if geometry.is_empty or not geometry.is_valid:
+ raise DetectionQaService._coverage_error("Tile coverage geometry is empty or invalid", tile_index)
+ if isinstance(geometry, GeometryCollection):
+ polygonal_parts = [part for part in geometry.geoms if isinstance(part, (Polygon, MultiPolygon)) and not part.is_empty]
+ if polygonal_parts:
+ geometry = unary_union(polygonal_parts)
+ return geometry
+
+ @staticmethod
+ def _coverage_error(message: str, tile_index: int | None = None) -> AppError:
+ details = {"tile_index": tile_index} if tile_index is not None else None
+ return AppError(
+ code="DETECTION_QA_COVERAGE_INVALID",
+ message=message,
+ details=details,
+ status_code=422,
+ )
diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py
index 5beb0554..3fd0012f 100644
--- a/backend/app/services/detection_service.py
+++ b/backend/app/services/detection_service.py
@@ -15,6 +15,7 @@ from app.core.errors import AppError
from app.models import AnalysisRun, Dataset, Detection, Job, Project, VectorFeature
from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
+from app.services.detection_qa_service import DetectionQaService
from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.model_registry_service import ModelRegistryService
from app.services.qa_service import QaService
@@ -295,13 +296,91 @@ class DetectionService:
status_code=422,
)
- candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
- reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references]
+ raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
+ raw_reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references]
+ candidate_geometries = raw_candidate_geometries
+ reference_geometries = raw_reference_geometries
+ run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
+ manifest_path = DetectionQaService.tile_manifest_path(run_parameters)
+ resolved_settings = get_settings()
+ is_configured_yolo = (
+ run_parameters.get("model_id") == resolved_settings.yolo_model_id
+ or run.model_name == resolved_settings.yolo_model_id
+ )
+ if is_configured_yolo and not manifest_path:
+ raise AppError(
+ code="DETECTION_QA_COVERAGE_UNAVAILABLE",
+ message="Configured YOLO QA requires persisted tile manifest provenance",
+ status_code=422,
+ )
+
+ coverage_summary: dict[str, Any] = {
+ "applied": False,
+ "mode": "unbounded_no_manifest",
+ "manifest_path": None,
+ "tile_count": 0,
+ "source_crs_values": [],
+ "candidate_raw_count": len(raw_candidate_geometries),
+ "candidate_evaluated_count": len(raw_candidate_geometries),
+ "candidate_excluded_outside_count": 0,
+ "candidate_clipped_boundary_count": 0,
+ "reference_raw_count": len(raw_reference_geometries),
+ "reference_evaluated_count": len(raw_reference_geometries),
+ "reference_excluded_outside_count": 0,
+ "reference_clipped_boundary_count": 0,
+ }
+ coverage_warnings: list[str] = []
+ if manifest_path:
+ manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles)
+ coverage = DetectionQaService.build_tile_coverage(
+ manifest,
+ manifest_path=manifest_path,
+ expected_dataset_id=run.dataset_id,
+ )
+ candidate_population = DetectionQaService.filter_population(raw_candidate_geometries, coverage)
+ reference_population = DetectionQaService.filter_population(raw_reference_geometries, coverage)
+ candidate_geometries = candidate_population.geometries
+ reference_geometries = reference_population.geometries
+ if not reference_geometries:
+ raise AppError(
+ code="REFERENCE_FEATURES_OUTSIDE_COVERAGE",
+ message="Reference dataset has no polygon features inside persisted inference tile coverage",
+ status_code=422,
+ )
+ coverage_summary = {
+ "applied": True,
+ "mode": "persisted_tile_manifest_union",
+ "manifest_path": coverage.manifest_path,
+ "tile_count": coverage.tile_count,
+ "source_crs_values": list(coverage.source_crs_values),
+ "candidate_raw_count": candidate_population.raw_count,
+ "candidate_evaluated_count": candidate_population.evaluated_count,
+ "candidate_excluded_outside_count": candidate_population.excluded_outside_count,
+ "candidate_clipped_boundary_count": candidate_population.clipped_boundary_count,
+ "reference_raw_count": reference_population.raw_count,
+ "reference_evaluated_count": reference_population.evaluated_count,
+ "reference_excluded_outside_count": reference_population.excluded_outside_count,
+ "reference_clipped_boundary_count": reference_population.clipped_boundary_count,
+ }
+ coverage_warnings.append(
+ "QA populations were clipped to the union of persisted inference tile footprints before matching."
+ )
evidence = QaService._match_io_u_evidence(
candidate_geometries,
reference_geometries,
iou_threshold,
)
+ reference_envelopes = [(feature, geometry.envelope) for feature, geometry in reference_geometries]
+ envelope_evidence = QaService._match_io_u_evidence(
+ candidate_geometries,
+ reference_envelopes,
+ iou_threshold,
+ )
+ box_to_footprint_diagnostics = DetectionQaService.box_to_footprint_diagnostics(
+ evidence,
+ envelope_evidence,
+ iou_threshold=iou_threshold,
+ )
mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None
recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None
@@ -324,13 +403,16 @@ class DetectionService:
"iou_threshold": iou_threshold,
"class_name": class_name,
"min_confidence": min_confidence,
+ "coverage_policy": coverage_summary["mode"],
},
findings={
"matches": evidence.matches,
"false_positives": evidence.false_positives,
"false_negatives": evidence.false_negatives,
- "warnings": evidence.warnings,
+ "warnings": coverage_warnings + evidence.warnings,
"unsupported_geometry": evidence.unsupported,
+ "coverage": coverage_summary,
+ "box_to_footprint_diagnostics": box_to_footprint_diagnostics,
"match_evidence": evidence.match_evidence,
"false_positive_evidence": evidence.false_positive_evidence,
"false_negative_evidence": evidence.false_negative_evidence,
@@ -351,6 +433,8 @@ class DetectionService:
"reference_dataset_id": str(reference_dataset_id),
"candidate_feature_count": len(candidate_geometries),
"reference_feature_count": len(reference_geometries),
+ "candidate_feature_count_raw": len(raw_candidate_geometries),
+ "reference_feature_count_raw": len(raw_reference_geometries),
"matches": evidence.matches,
"false_positives": evidence.false_positives,
"false_negatives": evidence.false_negatives,
@@ -359,7 +443,9 @@ class DetectionService:
"f1_score": f1_score,
"mean_iou": mean_iou,
"iou_threshold": iou_threshold,
- "warnings": evidence.warnings,
+ "warnings": coverage_warnings + evidence.warnings,
+ "coverage": coverage_summary,
+ "box_to_footprint_diagnostics": box_to_footprint_diagnostics,
"match_evidence": evidence.match_evidence,
"false_positive_evidence": evidence.false_positive_evidence,
"false_negative_evidence": evidence.false_negative_evidence,
diff --git a/backend/tests/test_sprint121_real_data_detection_qa_smoke.py b/backend/tests/test_sprint121_real_data_detection_qa_smoke.py
index 99860359..1d8c4e76 100644
--- a/backend/tests/test_sprint121_real_data_detection_qa_smoke.py
+++ b/backend/tests/test_sprint121_real_data_detection_qa_smoke.py
@@ -28,6 +28,9 @@ def test_real_data_detection_qa_smoke_requires_operator_inputs_and_checks_full_c
assert "/api/v1/detection/yolo/preflight" in script
assert "/api/v1/detection/run" in script
assert "/qa/reference" in script
+ assert "persisted_tile_manifest_union" in script
+ assert "box_to_footprint_diagnostics" in script
+ assert "candidate_polygon_vs_reference_footprint_iou" in script
assert "/api/v1/exports/geojson" in script
assert "Response is not a canonical GeoIntel data envelope" in script
assert "No local model assets are available" in script
diff --git a/backend/tests/test_sprint184_detection_qa_coverage.py b/backend/tests/test_sprint184_detection_qa_coverage.py
new file mode 100644
index 00000000..be6ec269
--- /dev/null
+++ b/backend/tests/test_sprint184_detection_qa_coverage.py
@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+from uuid import uuid4
+
+import pytest
+from pyproj import Transformer
+from shapely.geometry import box
+
+from app.core.errors import AppError
+from app.services.detection_qa_service import DetectionQaService
+
+
+def test_tile_coverage_transforms_projected_manifest_bounds_to_epsg4326() -> None:
+ dataset_id = uuid4()
+ to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
+ left, bottom = to_lambert.transform(5.11, 51.18)
+ right, top = to_lambert.transform(5.13, 51.20)
+ manifest = {
+ "source_dataset_id": str(dataset_id),
+ "crs": "EPSG:31370",
+ "tiles": [{"bounds": [left, bottom, right, top], "crs": "EPSG:31370"}],
+ }
+
+ coverage = DetectionQaService.build_tile_coverage(
+ manifest,
+ manifest_path="/app/storage/tiles/manifest.json",
+ expected_dataset_id=dataset_id,
+ )
+
+ min_x, min_y, max_x, max_y = coverage.geometry.bounds
+ assert min_x == pytest.approx(5.11, abs=0.001)
+ assert min_y == pytest.approx(51.18, abs=0.001)
+ assert max_x == pytest.approx(5.13, abs=0.001)
+ assert max_y == pytest.approx(51.20, abs=0.001)
+ assert coverage.tile_count == 1
+
+
+def test_tile_coverage_rejects_manifest_for_different_dataset() -> None:
+ manifest = {
+ "source_dataset_id": str(uuid4()),
+ "crs": "EPSG:4326",
+ "tiles": [{"bounds": [5.0, 51.0, 5.1, 51.1]}],
+ }
+
+ with pytest.raises(AppError) as exc_info:
+ DetectionQaService.build_tile_coverage(
+ manifest,
+ manifest_path="/app/storage/tiles/manifest.json",
+ expected_dataset_id=uuid4(),
+ )
+
+ assert exc_info.value.code == "DETECTION_QA_COVERAGE_MISMATCH"
+
+
+def test_coverage_filter_reports_outside_and_boundary_clipped_population() -> None:
+ dataset_id = uuid4()
+ coverage = DetectionQaService.build_tile_coverage(
+ {
+ "source_dataset_id": str(dataset_id),
+ "crs": "EPSG:4326",
+ "tiles": [{"bounds": [0.0, 0.0, 1.0, 1.0]}],
+ },
+ manifest_path="/app/storage/tiles/manifest.json",
+ expected_dataset_id=dataset_id,
+ )
+
+ population = DetectionQaService.filter_population(
+ [
+ ({"id": "inside"}, box(0.1, 0.1, 0.2, 0.2)),
+ ({"id": "crossing"}, box(0.8, 0.8, 1.2, 1.2)),
+ ({"id": "outside"}, box(2.0, 2.0, 3.0, 3.0)),
+ ],
+ coverage,
+ )
+
+ assert population.raw_count == 3
+ assert population.evaluated_count == 2
+ assert population.excluded_outside_count == 1
+ assert population.clipped_boundary_count == 1
+ assert population.geometries[1][1].bounds == pytest.approx((0.8, 0.8, 1.0, 1.0))
diff --git a/backend/tests/test_sprint8c_detection_visualization_qa.py b/backend/tests/test_sprint8c_detection_visualization_qa.py
index 94982b28..78ff5905 100644
--- a/backend/tests/test_sprint8c_detection_visualization_qa.py
+++ b/backend/tests/test_sprint8c_detection_visualization_qa.py
@@ -6,7 +6,7 @@ from uuid import uuid4
import pytest
from fastapi.testclient import TestClient
from geoalchemy2.shape import from_shape
-from shapely.geometry import box
+from shapely.geometry import Polygon, box
from app.main import app
from app.db.session import get_db
@@ -274,3 +274,150 @@ def test_detection_qa_no_match_case_persists_zero_scores() -> None:
assert result["precision"] == 0.0
assert result["recall"] == 0.0
assert result["f1_score"] == 0.0
+
+
+def _coverage_manifest(tmp_path, dataset_id, bounds=(-1.0, -1.0, 3.0, 3.0)):
+ manifest_path = tmp_path / "manifest.json"
+ manifest_path.write_text(
+ json.dumps(
+ {
+ "source_dataset_id": str(dataset_id),
+ "crs": "EPSG:4326",
+ "tiles": [
+ {
+ "index": 0,
+ "path": "tile_0000.tif",
+ "bounds": list(bounds),
+ "crs": "EPSG:4326",
+ }
+ ],
+ }
+ ),
+ encoding="utf-8",
+ )
+ return manifest_path
+
+
+def test_detection_qa_excludes_references_outside_persisted_tile_coverage(tmp_path) -> None:
+ project_id = uuid4()
+ dataset_id = uuid4()
+ reference_dataset_id = uuid4()
+ analysis_run_id = uuid4()
+ manifest_path = _coverage_manifest(tmp_path, dataset_id, bounds=(0.0, 0.0, 1.0, 1.0))
+ detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0.1, 0.1, 0.9, 0.9))
+ reference_dataset = Dataset(
+ id=reference_dataset_id,
+ project_id=project_id,
+ name="reference.geojson",
+ dataset_type="vector",
+ source="manual",
+ dataset_role="reference",
+ )
+ inside_reference = VectorFeature(
+ id=uuid4(),
+ dataset_id=reference_dataset_id,
+ feature_class="building",
+ geometry=from_shape(box(0.1, 0.1, 0.9, 0.9), srid=4326),
+ )
+ outside_reference = VectorFeature(
+ id=uuid4(),
+ dataset_id=reference_dataset_id,
+ feature_class="building",
+ geometry=from_shape(box(10.0, 10.0, 11.0, 11.0), srid=4326),
+ )
+ db = FakeSession(
+ objects={
+ (AnalysisRun, analysis_run_id): AnalysisRun(
+ id=analysis_run_id,
+ project_id=project_id,
+ dataset_id=dataset_id,
+ analysis_type="detection",
+ status="success",
+ model_name="yolo-configured",
+ parameters_json={
+ "model_id": "yolo-configured",
+ "tile_manifest_path": str(manifest_path),
+ },
+ ),
+ (Dataset, reference_dataset_id): reference_dataset,
+ },
+ query_rows={Detection: [detection], VectorFeature: [inside_reference, outside_reference]},
+ )
+
+ result = DetectionService.compare_detections_with_reference(
+ db=db,
+ analysis_run_id=analysis_run_id,
+ reference_dataset_id=reference_dataset_id,
+ iou_threshold=0.5,
+ )
+
+ quality_check = next(item for item in db.added if isinstance(item, QualityCheck))
+ assert result["matches"] == 1
+ assert result["false_negatives"] == 0
+ assert result["reference_feature_count_raw"] == 2
+ assert result["reference_feature_count"] == 1
+ assert result["coverage"]["applied"] is True
+ assert result["coverage"]["reference_excluded_outside_count"] == 1
+ assert quality_check.parameters_json["coverage_policy"] == "persisted_tile_manifest_union"
+ assert quality_check.findings_json["coverage"] == result["coverage"]
+
+
+def test_detection_qa_reports_box_to_footprint_diagnostic_without_changing_strict_metrics(tmp_path) -> None:
+ project_id = uuid4()
+ dataset_id = uuid4()
+ reference_dataset_id = uuid4()
+ analysis_run_id = uuid4()
+ manifest_path = _coverage_manifest(tmp_path, dataset_id)
+ detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0.0, 0.0, 2.0, 2.0))
+ l_shaped_footprint = Polygon(
+ [(0.0, 0.0), (2.0, 0.0), (2.0, 0.4), (0.4, 0.4), (0.4, 2.0), (0.0, 2.0), (0.0, 0.0)]
+ )
+ reference_dataset = Dataset(
+ id=reference_dataset_id,
+ project_id=project_id,
+ name="reference.geojson",
+ dataset_type="vector",
+ source="manual",
+ dataset_role="reference",
+ )
+ reference_feature = VectorFeature(
+ id=uuid4(),
+ dataset_id=reference_dataset_id,
+ feature_class="building",
+ geometry=from_shape(l_shaped_footprint, srid=4326),
+ )
+ db = FakeSession(
+ objects={
+ (AnalysisRun, analysis_run_id): AnalysisRun(
+ id=analysis_run_id,
+ project_id=project_id,
+ dataset_id=dataset_id,
+ analysis_type="detection",
+ status="success",
+ model_name="yolo-configured",
+ parameters_json={
+ "model_id": "yolo-configured",
+ "tile_manifest_path": str(manifest_path),
+ },
+ ),
+ (Dataset, reference_dataset_id): reference_dataset,
+ },
+ query_rows={Detection: [detection], VectorFeature: [reference_feature]},
+ )
+
+ result = DetectionService.compare_detections_with_reference(
+ db=db,
+ analysis_run_id=analysis_run_id,
+ reference_dataset_id=reference_dataset_id,
+ iou_threshold=0.5,
+ )
+
+ diagnostics = result["box_to_footprint_diagnostics"]
+ quality_check = next(item for item in db.added if isinstance(item, QualityCheck))
+ assert result["matches"] == 0
+ assert result["false_positives"] == 1
+ assert result["false_negatives"] == 1
+ assert diagnostics["diagnostic_only"] is True
+ assert diagnostics["envelope_matches"] == 1
+ assert diagnostics["possible_box_to_footprint_mismatch_count"] == 1
+ assert quality_check.findings_json["box_to_footprint_diagnostics"] == diagnostics
diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md
index 858111af..c10e769c 100644
--- a/docs/AI_PIPELINES.md
+++ b/docs/AI_PIPELINES.md
@@ -544,6 +544,15 @@ Sprint 8C makes persisted detections reviewable:
- Persisted detection geometries can be returned as GeoJSON FeatureCollections for MapLibre display.
- Detection QA compares candidate detection geometries against persisted reference `vector_features`.
- QA results reuse `quality_checks` and `metrics`; no parallel QA persistence system is introduced.
+- Configured-YOLO QA derives its evaluation extent from the persisted tile
+ manifest. Tile bounds are transformed from their explicit source CRS to
+ EPSG:4326, unioned, and used to clip candidate/reference populations before
+ canonical footprint-IoU matching. Reference features wholly outside the
+ imagery presented to the model no longer count as false negatives.
+- A separate reference-envelope IoU pass is persisted as
+ `box_to_footprint_diagnostics`. It quantifies possible matching artifacts from
+ comparing rectangular detections with irregular building footprints, but is
+ diagnostic only and never changes canonical QA metrics.
- Segmentation remains out of scope for Sprint 8C.
## 2. Tile Metadata
diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md
index 8e5448a8..77259450 100644
--- a/docs/API_CONTRACTS.md
+++ b/docs/API_CONTRACTS.md
@@ -872,6 +872,29 @@ Response persists a `quality_check` and `metrics` rows through the existing QA/Q
- `false_negatives`
- `quality_check_id`
+Configured-YOLO QA automatically reads `tile_manifest_path` from the persisted
+`AnalysisRun.parameters_json`. Candidate and reference geometries are clipped
+to the union of the manifest's tile bounds after explicit CRS transformation to
+EPSG:4326. The response additionally returns:
+
+- `candidate_feature_count_raw` and `reference_feature_count_raw`;
+- `coverage`, including raw/evaluated/excluded/boundary-clipped population
+ counts, tile count, source CRS values and coverage mode;
+- `box_to_footprint_diagnostics`, which compares candidate boxes with reference
+ envelopes at the same IoU threshold.
+
+The canonical precision, recall, F1 and mean IoU always remain based on
+candidate geometry versus the persisted reference footprint. Envelope results
+are explicitly `diagnostic_only` and are persisted in
+`quality_checks.findings_json`; they never replace or inflate canonical metrics.
+
+Configured-YOLO QA fails closed with `DETECTION_QA_COVERAGE_UNAVAILABLE` when
+manifest provenance is absent, `DETECTION_QA_COVERAGE_MISMATCH` when it belongs
+to another raster, `DETECTION_QA_COVERAGE_INVALID` when bounds/CRS are invalid,
+or `REFERENCE_FEATURES_OUTSIDE_COVERAGE` when no reference polygons overlap the
+actual inference coverage. Explicit fixture/legacy runs without a manifest keep
+the documented unbounded comparison behavior.
+
If the reference dataset has no persisted vector features, the endpoint returns `REFERENCE_FEATURES_NOT_FOUND`. It does not calculate fake QA metrics.
#### Future analysis route: `/api/v1/analysis/building-stats`
diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md
index 3d518fb7..f97bcfd0 100644
--- a/docs/CODEX_EXECUTION_LOG.md
+++ b/docs/CODEX_EXECUTION_LOG.md
@@ -7556,3 +7556,46 @@ Open:
- Clip detection QA reference populations to actual raster/tile coverage and
add box-to-building-footprint matching diagnostics before any further model
training or promotion decision.
+
+# Sprint 184 - Detection QA coverage and matching diagnostics
+
+## Implementation
+
+- Added a focused detection-QA coverage service that resolves the persisted
+ tile manifest from the analysis run, validates dataset ownership and CRS,
+ transforms each tile footprint to EPSG:4326 and unions the exact inference
+ coverage.
+- Configured-YOLO QA now fails closed when persisted manifest provenance is
+ absent, invalid or belongs to another raster. Explicit fixture and legacy
+ runs without a manifest retain their existing unbounded QA behavior.
+- Candidate Detection geometries and persisted reference VectorFeature
+ geometries are clipped to inference coverage before the existing one-to-one
+ IoU matcher runs. Raw, evaluated, excluded-outside and boundary-clipped
+ counts are persisted in `quality_checks.findings_json`.
+- Canonical precision, recall, F1, mean IoU and metric rows remain strict
+ candidate-polygon versus reference-footprint results. A second
+ candidate-polygon versus reference-envelope pass is persisted and displayed
+ as diagnostic-only evidence; it never replaces or promotes canonical
+ metrics.
+- Detection Lab now explains which reference population was evaluated and
+ clearly separates possible box-to-footprint artifacts from the canonical
+ scorecard.
+- Updated the real-data operator smoke assertions, API/AI/database contracts
+ and backend/frontend operator documentation. No migration, request contract,
+ provider fetch or model dependency changed.
+
+## Local validation
+
+- `bash scripts/run_readiness_check.sh` passed end to end.
+- Full backend suite passed: `506` tests.
+- API contract audit passed with `81` implemented routes and the two documented
+ non-envelope endpoints.
+- Alembic reports one head: `202606120900`.
+- Frontend TypeScript checking and production build passed with `85` modules;
+ the dedicated MapLibre chunk remains intact.
+
+## Next pass
+
+- Deploy Sprint 184 to Tower, rerun QA for the persisted Mol-center analysis
+ run and verify both persisted coverage evidence and Detection Lab rendering
+ against live PostGIS before making any model-training decision.
diff --git a/docs/DATABASE_IMPLEMENTATION_PLAN.md b/docs/DATABASE_IMPLEMENTATION_PLAN.md
index 3df1c1fd..2ffaeade 100644
--- a/docs/DATABASE_IMPLEMENTATION_PLAN.md
+++ b/docs/DATABASE_IMPLEMENTATION_PLAN.md
@@ -189,6 +189,13 @@ Metrics may belong to a quality check, an analysis run, or both. Sprint 7A persi
Quality checks are domain records. Jobs track execution state; quality checks track the persisted QA/QC result; metrics track individual measurements.
+Detection QA coverage and box-to-footprint diagnostics require no schema
+change. The canonical metric rows remain precision, recall, F1, mean IoU and
+false-positive/negative counts. Tile coverage population counts and the
+diagnostic reference-envelope comparison are persisted in the existing
+`quality_checks.findings_json`; `parameters_json.coverage_policy` records the
+evaluation policy used for reproducibility.
+
### exports
- `id uuid primary key`
diff --git a/docs/TODO.md b/docs/TODO.md
index f5fc1d18..a0558bc3 100644
--- a/docs/TODO.md
+++ b/docs/TODO.md
@@ -26,7 +26,7 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Separate persisted database layers from available analysis-result overlays with an explicit Map content mode.
- [x] Persist combined Mol operator evidence under the Unraid storage mount so reports survive all-in-one container replacement.
- [x] Visually review Mol Postel and Donk false-positive/false-negative evidence, classify the dominant error modes and only then decide whether another model-training pass is justified.
-- [ ] Clip detection QA populations to persisted raster/tile coverage and add box-to-footprint matching diagnostics before reconsidering model training.
+- [x] Clip detection QA populations to persisted raster/tile coverage and add box-to-footprint matching diagnostics before reconsidering model training.
- [x] Backend FastAPI foundation, health endpoint and service structure.
- [x] React/TypeScript frontend foundation and MapLibre workbench.
- [x] Map layer visibility, opacity and feature property inspection.
@@ -411,7 +411,7 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Bound populated Project, Dataset, QA, AI and Export panels so long histories do not push core actions thousands of pixels down-page.
- [x] Move Data create/upload forms and Map/AI diagnostics into explicit progressive disclosures.
- [x] Reorder AI Labs around run controls and Map around layer selection plus the MapLibre canvas.
-- [ ] Harden detection QA coverage and matching diagnostics before any further model training.
+- [x] Harden detection QA coverage and matching diagnostics before any further model training.
- [x] Replace the one-page workflow panel stack with a task-based workbench shell.
- [x] Add persistent project/AOI/dataset/layer context.
diff --git a/frontend/README.md b/frontend/README.md
index 0e7f84b9..a7227a4b 100644
--- a/frontend/README.md
+++ b/frontend/README.md
@@ -142,6 +142,10 @@ AI Lab run controls explicitly explain when no raster dataset is available, inst
- Users can filter detections by class and minimum confidence.
- Selected detection GeoJSON is rendered on the existing MapLibre workbench map.
- Detection QA compares a selected detection run against a reference dataset and displays persisted QA metrics.
+- Detection QA also shows whether persisted inference-tile coverage was
+ applied, how many reference features were raw/evaluated/excluded, and a
+ clearly labeled diagnostic-only box-to-footprint envelope comparison. The
+ main precision, recall and F1 display remains canonical footprint-IoU output.
- No segmentation UI is introduced in Sprint 8C.
## Sprint 9 additions
diff --git a/frontend/src/components/detection/DetectionLab.tsx b/frontend/src/components/detection/DetectionLab.tsx
index ad33cfc0..5497e3c0 100644
--- a/frontend/src/components/detection/DetectionLab.tsx
+++ b/frontend/src/components/detection/DetectionLab.tsx
@@ -933,6 +933,33 @@ export function DetectionLab({
Mean IoU: {detectionQaResult.mean_iou?.toFixed(3) ?? 'n/a'}
False positives: {detectionQaResult.false_positives}
False negatives: {detectionQaResult.false_negatives}
+ {detectionQaResult.coverage ? (
+
+
Inference coverage
+
+ {detectionQaResult.coverage.applied
+ ? `${detectionQaResult.coverage.reference_evaluated_count} of ${detectionQaResult.coverage.reference_raw_count} reference features evaluated`
+ : 'No tile manifest coverage applied'}
+
+
+ {detectionQaResult.coverage.applied
+ ? `${detectionQaResult.coverage.reference_excluded_outside_count} outside coverage, ${detectionQaResult.coverage.reference_clipped_boundary_count} clipped at the boundary, ${detectionQaResult.coverage.tile_count} tiles.`
+ : 'This run uses the complete selected reference population.'}
+
+
+
Box-to-footprint diagnostic only
+
+ {detectionQaResult.box_to_footprint_diagnostics.envelope_matches} envelope matches versus{' '}
+ {detectionQaResult.box_to_footprint_diagnostics.strict_matches} canonical matches
+
+
+ {detectionQaResult.box_to_footprint_diagnostics.possible_box_to_footprint_mismatch_count} possible matching artifacts. Canonical precision, recall and F1 above remain footprint-IoU based.
+
+