Persist QA feature evidence

backend/app/services/qa_service.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from typing import Any
 from uuid import UUID
@@ -16,6 +17,19 @@ from app.schemas.qa import QaProviderComparisonResult
 from app.services.vector_operations_service import VectorOperationsService
 
 
+@dataclass
+class QaMatchEvidence:
+    matches: int = 0
+    false_positives: int = 0
+    false_negatives: int = 0
+    match_iou_values: list[float] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+    unsupported: bool = False
+    match_evidence: list[dict[str, Any]] = field(default_factory=list)
+    false_positive_evidence: list[dict[str, Any]] = field(default_factory=list)
+    false_negative_evidence: list[dict[str, Any]] = field(default_factory=list)
+
+
 def _extract_crs_warnings(source_dataset: Dataset, reference_dataset: Dataset) -> list[str]:
     warnings: list[str] = []
     for dataset, label in ((source_dataset, "candidate"), (reference_dataset, "reference")):
@@ -92,17 +106,34 @@ class QaService:
         return area_geometry
 
     @staticmethod
-    def _match_io_u_metrics(
+    def _feature_identifier(feature: dict[str, Any], fallback_prefix: str, index: int) -> str:
+        feature_id = feature.get("id")
+        if feature_id is not None:
+            return str(feature_id)
+        properties = feature.get("properties")
+        if isinstance(properties, dict):
+            for key in ("vector_feature_id", "source_feature_id", "detection_id", "segmentation_id", "id", "name"):
+                value = properties.get(key)
+                if value is not None:
+                    return str(value)
+        for key in ("vector_feature_id", "source_feature_id", "detection_id", "segmentation_id", "class_name", "feature_class"):
+            value = feature.get(key)
+            if value is not None:
+                return str(value)
+        return f"{fallback_prefix}-{index + 1}"
+
+    @staticmethod
+    def _match_io_u_evidence(
         source_geometries: list[tuple[dict[str, Any], BaseGeometry]],
         reference_geometries: list[tuple[dict[str, Any], BaseGeometry]],
         iou_threshold: float,
-    ) -> tuple[int, int, int, list[float], list[str], bool]:
+    ) -> QaMatchEvidence:
         source_supported = [
-            (feature, geom) for feature, geom in source_geometries if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES
+            (index, feature, geom) for index, (feature, geom) in enumerate(source_geometries) if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES
         ]
         reference_supported = [
-            (feature, geom)
-            for feature, geom in reference_geometries
+            (index, feature, geom)
+            for index, (feature, geom) in enumerate(reference_geometries)
             if geom.geom_type in QaService.SUPPORTED_GEOMETRY_TYPES
         ]
 
@@ -114,29 +145,35 @@ class QaService:
             }
         )
         if not source_supported or not reference_supported:
-            return (
-                0,
-                len(source_supported),
-                len(reference_supported),
-                [],
-                [f"Unsupported geometry types: {unsupported}"] if unsupported else [],
-                True,
+            return QaMatchEvidence(
+                false_positives=len(source_supported),
+                false_negatives=len(reference_supported),
+                warnings=[f"Unsupported geometry types: {unsupported}"] if unsupported else [],
+                unsupported=True,
+                false_positive_evidence=[
+                    {"candidate_feature_id": QaService._feature_identifier(feature, "candidate", source_index)}
+                    for source_index, feature, _ in source_supported
+                ],
+                false_negative_evidence=[
+                    {"reference_feature_id": QaService._feature_identifier(feature, "reference", reference_index)}
+                    for reference_index, feature, _ in reference_supported
+                ],
             )
 
         unmatched_reference_indices = set(range(len(reference_supported)))
-        matches = 0
-        match_iou_values: list[float] = []
-        false_positives = 0
+        evidence = QaMatchEvidence(warnings=[f"Unsupported geometry types: {unsupported}"] if unsupported else [], unsupported=bool(unsupported))
 
-        for _, source_geom in source_supported:
+        for source_index, source_feature, source_geom in source_supported:
+            source_feature_id = QaService._feature_identifier(source_feature, "candidate", source_index)
             if source_geom.area <= 0:
-                false_positives += 1
+                evidence.false_positives += 1
+                evidence.false_positive_evidence.append({"candidate_feature_id": source_feature_id})
                 continue
 
             best_iou = 0.0
             best_index = None
             for reference_index in list(unmatched_reference_indices):
-                _, reference_geom = reference_supported[reference_index]
+                _, _, reference_geom = reference_supported[reference_index]
                 if reference_geom.area <= 0:
                     unmatched_reference_indices.discard(reference_index)
                     continue
@@ -161,16 +198,45 @@ class QaService:
                     best_index = reference_index
 
             if best_index is not None and best_iou >= iou_threshold:
-                matches += 1
-                match_iou_values.append(best_iou)
+                reference_original_index, reference_feature, _ = reference_supported[best_index]
+                evidence.matches += 1
+                evidence.match_iou_values.append(best_iou)
+                evidence.match_evidence.append(
+                    {
+                        "candidate_feature_id": source_feature_id,
+                        "reference_feature_id": QaService._feature_identifier(reference_feature, "reference", reference_original_index),
+                        "iou": best_iou,
+                    }
+                )
                 unmatched_reference_indices.discard(best_index)
             else:
-                false_positives += 1
+                evidence.false_positives += 1
+                evidence.false_positive_evidence.append({"candidate_feature_id": source_feature_id})
 
-        false_negatives = len(unmatched_reference_indices)
-        warnings: list[str] = [f"Unsupported geometry types: {unsupported}"] if unsupported else []
+        evidence.false_negatives = len(unmatched_reference_indices)
+        for reference_index in sorted(unmatched_reference_indices):
+            reference_original_index, reference_feature, _ = reference_supported[reference_index]
+            evidence.false_negative_evidence.append(
+                {"reference_feature_id": QaService._feature_identifier(reference_feature, "reference", reference_original_index)}
+            )
 
-        return matches, false_positives, false_negatives, match_iou_values, warnings, bool(unsupported)
+        return evidence
+
+    @staticmethod
+    def _match_io_u_metrics(
+        source_geometries: list[tuple[dict[str, Any], BaseGeometry]],
+        reference_geometries: list[tuple[dict[str, Any], BaseGeometry]],
+        iou_threshold: float,
+    ) -> tuple[int, int, int, list[float], list[str], bool]:
+        evidence = QaService._match_io_u_evidence(source_geometries, reference_geometries, iou_threshold)
+        return (
+            evidence.matches,
+            evidence.false_positives,
+            evidence.false_negatives,
+            evidence.match_iou_values,
+            evidence.warnings,
+            evidence.unsupported,
+        )
 
     @staticmethod
     def compare_candidate_with_reference(
@@ -206,7 +272,7 @@ class QaService:
             candidate_geometries = QaService._apply_area_filter(candidate_geometries, area_geometry, dataset_id=candidate_dataset.id)
             reference_geometries = QaService._apply_area_filter(reference_geometries, area_geometry, dataset_id=reference_dataset.id)
 
-        matches, false_positives, false_negatives, match_iou_values, warnings, unsupported = QaService._match_io_u_metrics(
+        evidence = QaService._match_io_u_evidence(
             candidate_geometries,
             reference_geometries,
             iou_threshold,
@@ -214,35 +280,38 @@ class QaService:
 
         candidate_feature_count = len(candidate_payload.get("features", [])) if isinstance(candidate_payload, dict) else 0
         reference_feature_count = len(reference_payload.get("features", [])) if isinstance(reference_payload, dict) else 0
-        mean_iou = None if not match_iou_values else sum(match_iou_values) / len(match_iou_values)
+        mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
 
         precision = None
-        if matches + false_positives > 0:
-            precision = matches / (matches + false_positives)
+        if evidence.matches + evidence.false_positives > 0:
+            precision = evidence.matches / (evidence.matches + evidence.false_positives)
 
         recall = None
-        if matches + false_negatives > 0:
-            recall = matches / (matches + false_negatives)
+        if evidence.matches + evidence.false_negatives > 0:
+            recall = evidence.matches / (evidence.matches + evidence.false_negatives)
 
         f1_score = None
         if precision is not None and recall is not None and precision + recall > 0:
             f1_score = (2 * precision * recall) / (precision + recall)
 
-        status = "unsupported" if unsupported else "ok"
+        status = "unsupported" if evidence.unsupported else "ok"
         return QaProviderComparisonResult(
             status=status,
-            warnings=_extract_crs_warnings(candidate_dataset, reference_dataset) + warnings,
+            warnings=_extract_crs_warnings(candidate_dataset, reference_dataset) + evidence.warnings,
             candidate_feature_count=candidate_feature_count,
             reference_feature_count=reference_feature_count,
-            matches=matches,
-            false_positives=false_positives,
-            false_negatives=false_negatives,
+            matches=evidence.matches,
+            false_positives=evidence.false_positives,
+            false_negatives=evidence.false_negatives,
             precision=precision,
             recall=recall,
             f1_score=f1_score,
             mean_iou=mean_iou,
             iou_threshold=iou_threshold,
-            unsupported_geometry=unsupported,
-            unsupported_geometries=warnings,
+            unsupported_geometry=evidence.unsupported,
+            unsupported_geometries=evidence.warnings,
+            match_evidence=evidence.match_evidence,
+            false_positive_evidence=evidence.false_positive_evidence,
+            false_negative_evidence=evidence.false_negative_evidence,
             generated_at=datetime.now(timezone.utc),
         )